diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 3060660d47..5c474aa826 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -107,9 +107,9 @@ jobs: - name: Test run: | if ${{ env.USE_NX_AFFECTED }}; then - yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro --since=${{ env.NX_BASE_BRANCH }} + yarn test --ignore=@budibase/worker --ignore=@budibase/server --since=${{ env.NX_BASE_BRANCH }} else - yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro + yarn test --ignore=@budibase/worker --ignore=@budibase/server fi test-worker: @@ -160,31 +160,6 @@ jobs: yarn test --scope=@budibase/server fi - test-pro: - runs-on: ubuntu-latest - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' - steps: - - name: Checkout repo and submodules - uses: actions/checkout@v4 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - fetch-depth: 0 - - - name: Use Node.js 20.x - uses: actions/setup-node@v4 - with: - node-version: 20.x - cache: yarn - - run: yarn --frozen-lockfile - - name: Test - run: | - if ${{ env.USE_NX_AFFECTED }}; then - yarn test --scope=@budibase/pro --since=${{ env.NX_BASE_BRANCH }} - else - yarn test --scope=@budibase/pro - fi - integration-test: runs-on: ubuntu-latest steps: diff --git a/eslint-local-rules/index.js b/eslint-local-rules/index.js index 177b0a129c..202e52e70e 100644 --- a/eslint-local-rules/index.js +++ b/eslint-local-rules/index.js @@ -7,11 +7,12 @@ module.exports = { if ( /^@budibase\/[^/]+\/.*$/.test(importPath) && - importPath !== "@budibase/backend-core/tests" + importPath !== "@budibase/backend-core/tests" && + importPath !== "@budibase/string-templates/test/utils" ) { context.report({ node, - message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests.`, + message: `Importing from @budibase is not allowed, except for @budibase/backend-core/tests and @budibase/string-templates/test/utils.`, }) } }, diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index ee98b0729d..be01056b53 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -12,8 +12,6 @@ COPY .yarnrc . COPY packages/server/package.json packages/server/package.json COPY packages/worker/package.json packages/worker/package.json -# string-templates does not get bundled during the esbuild process, so we want to use the local version -COPY packages/string-templates/package.json packages/string-templates/package.json COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh @@ -26,7 +24,7 @@ RUN ./scripts/removeWorkspaceDependencies.sh packages/worker/package.json RUN echo '' > scripts/syncProPackage.js RUN jq 'del(.scripts.postinstall)' package.json > temp.json && mv temp.json package.json RUN ./scripts/removeWorkspaceDependencies.sh package.json -RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production +RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production --frozen-lockfile # copy the actual code COPY packages/server/dist packages/server/dist @@ -35,7 +33,6 @@ COPY packages/server/client packages/server/client COPY packages/server/builder packages/server/builder COPY packages/worker/dist packages/worker/dist COPY packages/worker/pm2.config.js packages/worker/pm2.config.js -COPY packages/string-templates packages/string-templates FROM budibase/couchdb:v3.3.3 as runner @@ -52,11 +49,11 @@ RUN apt-get update && \ # Install postgres client for pg_dump utils RUN apt install -y software-properties-common apt-transport-https ca-certificates gnupg \ - && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \ - && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \ - && apt update -y \ - && apt install postgresql-client-15 -y \ - && apt remove software-properties-common apt-transport-https gpg -y + && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \ + && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \ + && apt update -y \ + && apt install postgresql-client-15 -y \ + && apt remove software-properties-common apt-transport-https gpg -y # We use pm2 in order to run multiple node processes in a single container RUN npm install --global pm2 @@ -100,9 +97,6 @@ COPY --from=build /app/node_modules /node_modules COPY --from=build /app/package.json /package.json COPY --from=build /app/packages/server /app COPY --from=build /app/packages/worker /worker -COPY --from=build /app/packages/string-templates /string-templates - -RUN cd /string-templates && yarn link && cd ../app && yarn link @budibase/string-templates && cd ../worker && yarn link @budibase/string-templates EXPOSE 80 diff --git a/lerna.json b/lerna.json index 0f6121bb18..d191854fac 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.21.9", + "version": "2.22.1", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/account-portal b/packages/account-portal index 0c050591c2..23a1219732 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 0c050591c21d3b67dc0c9225d60cc9e2324c8dac +Subproject commit 23a1219732bd778654c0bcc4f49910c511e2d51f diff --git a/packages/cli/tsconfig.build.json b/packages/cli/tsconfig.build.json index 9fbccadd31..f89ad95c28 100644 --- a/packages/cli/tsconfig.build.json +++ b/packages/cli/tsconfig.build.json @@ -15,7 +15,8 @@ "@budibase/types": ["../types/src"], "@budibase/backend-core": ["../backend-core/src"], "@budibase/backend-core/*": ["../backend-core/*"], - "@budibase/shared-core": ["../shared-core/src"] + "@budibase/shared-core": ["../shared-core/src"], + "@budibase/string-templates": ["../string-templates/src"] } }, "include": ["src/**/*"], diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json index 6ca641e214..bf51d7ed76 100644 --- a/packages/cli/tsconfig.json +++ b/packages/cli/tsconfig.json @@ -1,16 +1,8 @@ { "extends": "./tsconfig.build.json", - "compilerOptions": { - "composite": true, - "declaration": true, - "sourceMap": true, - "baseUrl": ".", - "resolveJsonModule": true - }, "ts-node": { "require": ["tsconfig-paths/register"], "swc": true }, - "include": ["src/**/*", "package.json"], "exclude": ["node_modules", "dist"] } diff --git a/packages/pro b/packages/pro index c4c98ae70f..65ac3fc8a2 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit c4c98ae70f2e936009250893898ecf11f4ddf2c3 +Subproject commit 65ac3fc8a20a5244fbe47629cf79678db2d9ae8a diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index 3aae395bc5..7c0e6e59bc 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -41,17 +41,9 @@ COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies. RUN chmod +x ./scripts/removeWorkspaceDependencies.sh -WORKDIR /string-templates -COPY packages/string-templates/package.json package.json -RUN ../scripts/removeWorkspaceDependencies.sh package.json -RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true --network-timeout 1000000 -COPY packages/string-templates . - - WORKDIR /app COPY packages/server/package.json . COPY packages/server/dist/yarn.lock . -RUN cd ../string-templates && yarn link && cd - && yarn link @budibase/string-templates COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh RUN chmod +x ./scripts/removeWorkspaceDependencies.sh diff --git a/packages/server/jest.config.ts b/packages/server/jest.config.ts index 80b187966c..6c6d6a20d3 100644 --- a/packages/server/jest.config.ts +++ b/packages/server/jest.config.ts @@ -30,6 +30,8 @@ const baseConfig: Config.InitialProjectOptions = { "@budibase/backend-core": "/../backend-core/src", "@budibase/shared-core": "/../shared-core/src", "@budibase/types": "/../types/src", + "@budibase/string-templates/(.*)": ["/../string-templates/$1"], + "@budibase/string-templates": ["/../string-templates/src"], }, } diff --git a/packages/server/src/api/routes/tests/permissions.spec.ts b/packages/server/src/api/routes/tests/permissions.spec.ts index 1eabf6edbb..af7a2a578e 100644 --- a/packages/server/src/api/routes/tests/permissions.spec.ts +++ b/packages/server/src/api/routes/tests/permissions.spec.ts @@ -16,7 +16,7 @@ import { ViewV2, } from "@budibase/types" import * as setup from "./utilities" -import { mocks } from "@budibase/backend-core/tests" +import { generator, mocks } from "@budibase/backend-core/tests" const { basicRow } = setup.structures const { BUILTIN_ROLE_IDS } = roles @@ -44,7 +44,10 @@ describe("/permission", () => { table = (await config.createTable()) as typeof table row = await config.createRow() - view = await config.api.viewV2.create({ tableId: table._id }) + view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + }) perms = await config.api.permission.add({ roleId: STD_ROLE_ID, resourceId: table._id, diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 371045687b..ee9af34965 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -42,6 +42,7 @@ tk.freeze(timestamp) jest.unmock("mysql2") jest.unmock("mysql2/promise") jest.unmock("mssql") +jest.unmock("pg") describe.each([ ["internal", undefined], @@ -152,8 +153,8 @@ describe.each([ table = await config.api.table.save(defaultTable()) }) - describe("save, load, update", () => { - it("returns a success message when the row is created", async () => { + describe("create", () => { + it("creates a new row successfully", async () => { const rowUsage = await getRowUsage() const row = await config.api.row.save(table._id!, { name: "Test Contact", @@ -163,7 +164,44 @@ describe.each([ await assertRowUsage(rowUsage + 1) }) - it("Increment row autoId per create row request", async () => { + it("fails to create a row for a table that does not exist", async () => { + const rowUsage = await getRowUsage() + await config.api.row.save("1234567", {}, { status: 404 }) + await assertRowUsage(rowUsage) + }) + + it("fails to create a row if required fields are missing", async () => { + const rowUsage = await getRowUsage() + const table = await config.api.table.save( + saveTableRequest({ + schema: { + required: { + type: FieldType.STRING, + name: "required", + constraints: { + type: "string", + presence: true, + }, + }, + }, + }) + ) + await config.api.row.save( + table._id!, + {}, + { + status: 500, + body: { + validationErrors: { + required: ["can't be blank"], + }, + }, + } + ) + await assertRowUsage(rowUsage) + }) + + it("increment row autoId per create row request", async () => { const rowUsage = await getRowUsage() const newTable = await config.api.table.save( @@ -198,52 +236,6 @@ describe.each([ await assertRowUsage(rowUsage + 10) }) - it("updates a row successfully", async () => { - const existing = await config.api.row.save(table._id!, {}) - const rowUsage = await getRowUsage() - - const res = await config.api.row.save(table._id!, { - _id: existing._id, - _rev: existing._rev, - name: "Updated Name", - }) - - expect(res.name).toEqual("Updated Name") - await assertRowUsage(rowUsage) - }) - - it("should load a row", async () => { - const existing = await config.api.row.save(table._id!, {}) - - const res = await config.api.row.get(table._id!, existing._id!) - - expect(res).toEqual({ - ...existing, - ...defaultRowFields, - }) - }) - - it("should list all rows for given tableId", async () => { - const table = await config.api.table.save(defaultTable()) - const rows = await Promise.all([ - config.api.row.save(table._id!, {}), - config.api.row.save(table._id!, {}), - ]) - - const res = await config.api.row.fetch(table._id!) - expect(res.map(r => r._id)).toEqual( - expect.arrayContaining(rows.map(r => r._id)) - ) - }) - - it("load should return 404 when row does not exist", async () => { - const table = await config.api.table.save(defaultTable()) - await config.api.row.save(table._id!, {}) - await config.api.row.get(table._id!, "1234567", { - status: 404, - }) - }) - isInternal && it("row values are coerced", async () => { const str: FieldSchema = { @@ -296,8 +288,6 @@ describe.each([ } const table = await config.api.table.save( saveTableRequest({ - name: "TestTable2", - type: "table", schema: { name: str, stringUndefined: str, @@ -404,53 +394,60 @@ describe.each([ }) }) - describe("view save", () => { - it("views have extra data trimmed", async () => { - const table = await config.api.table.save( - saveTableRequest({ - name: "orders", - schema: { - Country: { - type: FieldType.STRING, - name: "Country", - }, - Story: { - type: FieldType.STRING, - name: "Story", - }, - }, - }) - ) + describe("get", () => { + it("reads an existing row successfully", async () => { + const existing = await config.api.row.save(table._id!, {}) - const createViewResponse = await config.api.viewV2.create({ - tableId: table._id!, - name: uuid.v4(), - schema: { - Country: { - visible: true, - }, - }, - }) + const res = await config.api.row.get(table._id!, existing._id!) - const createRowResponse = await config.api.row.save( - createViewResponse.id, - { - Country: "Aussy", - Story: "aaaaa", - } - ) - - const row = await config.api.row.get(table._id!, createRowResponse._id!) - expect(row.Story).toBeUndefined() - expect(row).toEqual({ + expect(res).toEqual({ + ...existing, ...defaultRowFields, - Country: "Aussy", - id: createRowResponse.id, - _id: createRowResponse._id, - _rev: createRowResponse._rev, - tableId: table._id, }) }) + + it("returns 404 when row does not exist", async () => { + const table = await config.api.table.save(defaultTable()) + await config.api.row.save(table._id!, {}) + await config.api.row.get(table._id!, "1234567", { + status: 404, + }) + }) + }) + + describe("fetch", () => { + it("fetches all rows for given tableId", async () => { + const table = await config.api.table.save(defaultTable()) + const rows = await Promise.all([ + config.api.row.save(table._id!, {}), + config.api.row.save(table._id!, {}), + ]) + + const res = await config.api.row.fetch(table._id!) + expect(res.map(r => r._id)).toEqual( + expect.arrayContaining(rows.map(r => r._id)) + ) + }) + + it("returns 404 when table does not exist", async () => { + await config.api.row.fetch("1234567", { status: 404 }) + }) + }) + + describe("update", () => { + it("updates an existing row successfully", async () => { + const existing = await config.api.row.save(table._id!, {}) + const rowUsage = await getRowUsage() + + const res = await config.api.row.save(table._id!, { + _id: existing._id, + _rev: existing._rev, + name: "Updated Name", + }) + + expect(res.name).toEqual("Updated Name") + await assertRowUsage(rowUsage) + }) }) describe("patch", () => { @@ -722,50 +719,7 @@ describe.each([ }) }) - // Legacy views are not available for external - isInternal && - describe("fetchView", () => { - beforeEach(async () => { - table = await config.api.table.save(defaultTable()) - }) - - it("should be able to fetch tables contents via 'view'", async () => { - const row = await config.api.row.save(table._id!, {}) - const rowUsage = await getRowUsage() - - const rows = await config.api.legacyView.get(table._id!) - expect(rows.length).toEqual(1) - expect(rows[0]._id).toEqual(row._id) - await assertRowUsage(rowUsage) - }) - - it("should throw an error if view doesn't exist", async () => { - const rowUsage = await getRowUsage() - - await config.api.legacyView.get("derp", undefined, { status: 404 }) - - await assertRowUsage(rowUsage) - }) - - it("should be able to run on a view", async () => { - const view = await config.createLegacyView({ - tableId: table._id!, - name: "ViewTest", - filters: [], - schema: {}, - }) - const row = await config.api.row.save(table._id!, {}) - const rowUsage = await getRowUsage() - - const rows = await config.api.legacyView.get(view.name) - expect(rows.length).toEqual(1) - expect(rows[0]._id).toEqual(row._id) - - await assertRowUsage(rowUsage) - }) - }) - - describe("fetchEnrichedRows", () => { + describe("enrich", () => { beforeAll(async () => { table = await config.api.table.save(defaultTable()) }) @@ -827,10 +781,6 @@ describe.each([ isInternal && describe("attachments", () => { - beforeAll(async () => { - table = await config.api.table.save(defaultTable()) - }) - it("should allow enriching attachment rows", async () => { const table = await config.api.table.save( defaultTable({ @@ -865,7 +815,7 @@ describe.each([ }) }) - describe("exportData", () => { + describe("exportRows", () => { beforeAll(async () => { table = await config.api.table.save(defaultTable()) }) @@ -947,6 +897,7 @@ describe.each([ const table = await config.api.table.save(await userTable()) const view = await config.api.viewV2.create({ tableId: table._id!, + name: generator.guid(), schema: { name: { visible: true }, surname: { visible: true }, @@ -984,6 +935,7 @@ describe.each([ const tableId = table._id! const view = await config.api.viewV2.create({ tableId: tableId, + name: generator.guid(), schema: { name: { visible: true }, address: { visible: true }, @@ -1026,6 +978,7 @@ describe.each([ const tableId = table._id! const view = await config.api.viewV2.create({ tableId: tableId, + name: generator.guid(), schema: { name: { visible: true }, address: { visible: true }, @@ -1049,6 +1002,7 @@ describe.each([ const tableId = table._id! const view = await config.api.viewV2.create({ tableId: tableId, + name: generator.guid(), schema: { name: { visible: true }, address: { visible: true }, @@ -1109,6 +1063,7 @@ describe.each([ const createViewResponse = await config.api.viewV2.create({ tableId: table._id!, + name: generator.guid(), }) const response = await config.api.viewV2.search(createViewResponse.id) @@ -1155,6 +1110,7 @@ describe.each([ const createViewResponse = await config.api.viewV2.create({ tableId: table._id!, + name: generator.guid(), query: [ { operator: SearchQueryOperators.EQUAL, field: "age", value: 40 }, ], @@ -1279,6 +1235,7 @@ describe.each([ async (sortParams, expected) => { const createViewResponse = await config.api.viewV2.create({ tableId: table._id!, + name: generator.guid(), sort: sortParams, schema: viewSchema, }) @@ -1299,6 +1256,7 @@ describe.each([ async (sortParams, expected) => { const createViewResponse = await config.api.viewV2.create({ tableId: table._id!, + name: generator.guid(), sort: { field: "name", order: SortOrder.ASCENDING, @@ -1339,6 +1297,7 @@ describe.each([ const view = await config.api.viewV2.create({ tableId: table._id!, + name: generator.guid(), schema: { name: { visible: true } }, }) const response = await config.api.viewV2.search(view.id) @@ -1361,6 +1320,7 @@ describe.each([ const table = await config.api.table.save(await userTable()) const createViewResponse = await config.api.viewV2.create({ tableId: table._id!, + name: generator.guid(), }) const response = await config.api.viewV2.search(createViewResponse.id) expect(response.rows).toHaveLength(0) @@ -1376,6 +1336,7 @@ describe.each([ const createViewResponse = await config.api.viewV2.create({ tableId: table._id!, + name: generator.guid(), }) const response = await config.api.viewV2.search(createViewResponse.id, { limit, @@ -1392,6 +1353,7 @@ describe.each([ ) const view = await config.api.viewV2.create({ tableId: table._id!, + name: generator.guid(), }) const rows = (await config.api.viewV2.search(view.id)).rows @@ -1466,6 +1428,7 @@ describe.each([ view = await config.api.viewV2.create({ tableId: table._id!, + name: generator.guid(), }) }) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 77704a0408..4321f012aa 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -20,7 +20,7 @@ import sdk from "../../../sdk" import * as uuid from "uuid" import tk from "timekeeper" -import { mocks } from "@budibase/backend-core/tests" +import { generator, mocks } from "@budibase/backend-core/tests" import { TableToBuild } from "../../../tests/utilities/TestConfiguration" tk.freeze(mocks.date.MOCK_DATE) @@ -417,8 +417,8 @@ describe("/tables", () => { it("should fetch views", async () => { const tableId = config.table!._id! const views = [ - await config.api.viewV2.create({ tableId }), - await config.api.viewV2.create({ tableId }), + await config.api.viewV2.create({ tableId, name: generator.guid() }), + await config.api.viewV2.create({ tableId, name: generator.guid() }), ] const res = await request @@ -455,7 +455,7 @@ describe("/tables", () => { }, })) - await config.api.viewV2.create({ tableId }) + await config.api.viewV2.create({ tableId, name: generator.guid() }) await config.createLegacyView() const res = await config.api.table.fetch() diff --git a/packages/server/src/api/routes/tests/view.spec.ts b/packages/server/src/api/routes/tests/view.spec.ts index 2e8c71b812..893df61fdc 100644 --- a/packages/server/src/api/routes/tests/view.spec.ts +++ b/packages/server/src/api/routes/tests/view.spec.ts @@ -3,12 +3,15 @@ import * as setup from "./utilities" import { FieldType, INTERNAL_TABLE_SOURCE_ID, + QuotaUsageType, SaveTableRequest, + StaticQuotaName, Table, TableSourceType, View, ViewCalculation, } from "@budibase/types" +import { quotas } from "@budibase/pro" const priceTable: SaveTableRequest = { name: "table", @@ -57,6 +60,18 @@ describe("/views", () => { return config.api.legacyView.save(viewToSave) } + const getRowUsage = async () => { + const { total } = await config.doInContext(undefined, () => + quotas.getCurrentUsageValues(QuotaUsageType.STATIC, StaticQuotaName.ROWS) + ) + return total + } + + const assertRowUsage = async (expected: number) => { + const usage = await getRowUsage() + expect(usage).toBe(expected) + } + describe("create", () => { it("returns a success message when the view is successfully created", async () => { const res = await saveView() @@ -265,6 +280,41 @@ describe("/views", () => { expect(views.length).toBe(1) expect(views.find(({ name }) => name === "TestView")).toBeDefined() }) + + it("should be able to fetch tables contents via 'view'", async () => { + const row = await config.api.row.save(table._id!, {}) + const rowUsage = await getRowUsage() + + const rows = await config.api.legacyView.get(table._id!) + expect(rows.length).toEqual(1) + expect(rows[0]._id).toEqual(row._id) + await assertRowUsage(rowUsage) + }) + + it("should throw an error if view doesn't exist", async () => { + const rowUsage = await getRowUsage() + + await config.api.legacyView.get("derp", undefined, { status: 404 }) + + await assertRowUsage(rowUsage) + }) + + it("should be able to run on a view", async () => { + const view = await config.api.legacyView.save({ + tableId: table._id!, + name: "ViewTest", + filters: [], + schema: {}, + }) + const row = await config.api.row.save(table._id!, {}) + const rowUsage = await getRowUsage() + + const rows = await config.api.legacyView.get(view.name!) + expect(rows.length).toEqual(1) + expect(rows[0]._id).toEqual(row._id) + + await assertRowUsage(rowUsage) + }) }) describe("query", () => { diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 5198e63338..ded5e08d29 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -1,9 +1,11 @@ import * as setup from "./utilities" import { CreateViewRequest, + Datasource, FieldSchema, FieldType, INTERNAL_TABLE_SOURCE_ID, + SaveTableRequest, SearchQueryOperators, SortOrder, SortType, @@ -14,65 +16,88 @@ import { ViewV2, } from "@budibase/types" import { generator } from "@budibase/backend-core/tests" -import { generateDatasourceID } from "../../../db/utils" +import * as uuid from "uuid" +import { databaseTestProviders } from "../../../integrations/tests/utils" +import merge from "lodash/merge" -function priceTable(): Table { - return { - name: "table", - type: "table", - sourceId: INTERNAL_TABLE_SOURCE_ID, - sourceType: TableSourceType.INTERNAL, - schema: { - Price: { - type: FieldType.NUMBER, - name: "Price", - constraints: {}, - }, - Category: { - type: FieldType.STRING, - name: "Category", - constraints: { - type: "string", - }, - }, - }, - } -} - -const config = setup.getConfig() - -beforeAll(async () => { - await config.init() -}) +jest.unmock("mysql2") +jest.unmock("mysql2/promise") +jest.unmock("mssql") +jest.unmock("pg") describe.each([ - ["internal ds", () => config.createTable(priceTable())], - [ - "external ds", - async () => { - const datasource = await config.createDatasource({ - datasource: { - ...setup.structures.basicDatasource().datasource, - plus: true, - _id: generateDatasourceID({ plus: true }), - }, - }) + ["internal", undefined], + ["postgres", databaseTestProviders.postgres], + ["mysql", databaseTestProviders.mysql], + ["mssql", databaseTestProviders.mssql], + ["mariadb", databaseTestProviders.mariadb], +])("/v2/views (%s)", (_, dsProvider) => { + const config = setup.getConfig() - return config.createExternalTable({ - ...priceTable(), - sourceId: datasource._id, - sourceType: TableSourceType.EXTERNAL, - }) - }, - ], -])("/v2/views (%s)", (_, tableBuilder) => { let table: Table + let datasource: Datasource + + function saveTableRequest( + ...overrides: Partial[] + ): SaveTableRequest { + const req: SaveTableRequest = { + name: uuid.v4().substring(0, 16), + type: "table", + sourceType: datasource + ? TableSourceType.EXTERNAL + : TableSourceType.INTERNAL, + sourceId: datasource ? datasource._id! : INTERNAL_TABLE_SOURCE_ID, + primary: ["id"], + schema: { + id: { + type: FieldType.AUTO, + name: "id", + autocolumn: true, + constraints: { + presence: true, + }, + }, + }, + } + return merge(req, ...overrides) + } + + function priceTable(): SaveTableRequest { + return saveTableRequest({ + schema: { + Price: { + type: FieldType.NUMBER, + name: "Price", + constraints: {}, + }, + Category: { + type: FieldType.STRING, + name: "Category", + constraints: { + type: "string", + }, + }, + }, + }) + } beforeAll(async () => { - table = await tableBuilder() + await config.init() + + if (dsProvider) { + datasource = await config.createDatasource({ + datasource: await dsProvider.datasource(), + }) + } + table = await config.api.table.save(priceTable()) }) - afterAll(setup.afterAll) + afterAll(async () => { + if (dsProvider) { + await dsProvider.stop() + } + setup.afterAll() + }) describe("create", () => { it("persist the view when the view is successfully created", async () => { @@ -186,9 +211,12 @@ describe.each([ let view: ViewV2 beforeEach(async () => { - table = await tableBuilder() + table = await config.api.table.save(priceTable()) - view = await config.api.viewV2.create({ name: "View A" }) + view = await config.api.viewV2.create({ + tableId: table._id!, + name: "View A", + }) }) it("can update an existing view data", async () => { @@ -247,6 +275,9 @@ describe.each([ ...updatedData, schema: { ...table.schema, + id: expect.objectContaining({ + visible: false, + }), Category: expect.objectContaining({ visible: false, }), @@ -320,23 +351,27 @@ describe.each([ }) it("cannot update views v1", async () => { - const viewV1 = await config.createLegacyView() - await config.api.viewV2.update( - { - ...viewV1, - }, - { + const viewV1 = await config.api.legacyView.save({ + tableId: table._id!, + name: generator.guid(), + filters: [], + schema: {}, + }) + + await config.api.viewV2.update(viewV1 as unknown as ViewV2, { + status: 400, + body: { + message: "Only views V2 can be updated", status: 400, - body: { - message: "Only views V2 can be updated", - status: 400, - }, - } - ) + }, + }) }) it("cannot update the a view with unmatching ids between url and body", async () => { - const anotherView = await config.api.viewV2.create() + const anotherView = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + }) const result = await config .request!.put(`/api/v2/views/${anotherView.id}`) .send(view) @@ -411,7 +446,10 @@ describe.each([ let view: ViewV2 beforeAll(async () => { - view = await config.api.viewV2.create() + view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + }) }) it("can delete an existing view", async () => { @@ -448,4 +486,43 @@ describe.each([ expect(viewSchema.Price?.visible).toEqual(false) }) }) + + describe("read", () => { + it("views have extra data trimmed", async () => { + const table = await config.api.table.save( + saveTableRequest({ + name: "orders", + schema: { + Country: { + type: FieldType.STRING, + name: "Country", + }, + Story: { + type: FieldType.STRING, + name: "Story", + }, + }, + }) + ) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: uuid.v4(), + schema: { + Country: { + visible: true, + }, + }, + }) + + let row = await config.api.row.save(view.id, { + Country: "Aussy", + Story: "aaaaa", + }) + + row = await config.api.row.get(table._id!, row._id!) + expect(row.Story).toBeUndefined() + expect(row.Country).toEqual("Aussy") + }) + }) }) diff --git a/packages/server/src/jsRunner/tests/jsRunner.spec.ts b/packages/server/src/jsRunner/tests/jsRunner.spec.ts index 54983aa470..dc6e32f52c 100644 --- a/packages/server/src/jsRunner/tests/jsRunner.spec.ts +++ b/packages/server/src/jsRunner/tests/jsRunner.spec.ts @@ -1,7 +1,7 @@ import { validate as isValidUUID } from "uuid" import { processStringSync, encodeJSBinding } from "@budibase/string-templates" -const { runJsHelpersTests } = require("@budibase/string-templates/test/utils") +import { runJsHelpersTests } from "@budibase/string-templates/test/utils" import tk from "timekeeper" import { init } from ".." diff --git a/packages/server/src/sdk/tests/rows/search.spec.ts b/packages/server/src/sdk/tests/rows/search.spec.ts deleted file mode 100644 index feae5e7ee8..0000000000 --- a/packages/server/src/sdk/tests/rows/search.spec.ts +++ /dev/null @@ -1,21 +0,0 @@ -import * as search from "../../app/rows/search" - -describe("removeEmptyFilters", () => { - it("0 should not be removed", () => { - const filters = search.removeEmptyFilters({ - equal: { - column: 0, - }, - }) - expect((filters.equal as any).column).toBe(0) - }) - - it("empty string should be removed", () => { - const filters = search.removeEmptyFilters({ - equal: { - column: "", - }, - }) - expect(Object.values(filters.equal as any).length).toBe(0) - }) -}) diff --git a/packages/server/src/tests/utilities/api/viewV2.ts b/packages/server/src/tests/utilities/api/viewV2.ts index d4539e00b1..bd6241b7cd 100644 --- a/packages/server/src/tests/utilities/api/viewV2.ts +++ b/packages/server/src/tests/utilities/api/viewV2.ts @@ -11,21 +11,9 @@ import sdk from "../../../sdk" export class ViewV2API extends TestAPI { create = async ( - viewData?: Partial, + view: CreateViewRequest, expectations?: Expectations ): Promise => { - let tableId = viewData?.tableId - if (!tableId && !this.config.table) { - throw "Test requires table to be configured." - } - - tableId = tableId || this.config.table!._id! - const view = { - tableId, - name: generator.guid(), - ...viewData, - } - const exp: Expectations = { status: 201, ...expectations, diff --git a/packages/string-templates/jest.config.js b/packages/string-templates/jest.config.js index c6391cdb92..4916ebf894 100644 --- a/packages/string-templates/jest.config.js +++ b/packages/string-templates/jest.config.js @@ -4,6 +4,7 @@ */ module.exports = { + preset: "ts-jest", // All imported modules in your tests should be mocked automatically // automock: false, diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 340d74ef8a..aa3aa5adec 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -2,29 +2,28 @@ "name": "@budibase/string-templates", "version": "0.0.0", "description": "Handlebars wrapper for Budibase templating.", - "main": "src/index.js", + "main": "dist/bundle.cjs", "module": "dist/bundle.mjs", + "types": "src/index.ts", "license": "MPL-2.0", - "types": "dist/index.d.ts", "exports": { ".": { - "require": "./src/index.js", + "require": "./dist/bundle.cjs", "import": "./dist/bundle.mjs" }, "./package.json": "./package.json", - "./test/utils": "./test/utils.js", "./iife": "./src/iife.js" }, "files": [ "dist", - "src", - "manifest.json" + "src" ], "scripts": { - "build": "tsc && rollup -c", - "dev": "concurrently \"tsc --watch\" \"rollup -cw\"", + "build": "tsc --emitDeclarationOnly && rollup -c", + "dev": "rollup -cw", + "check:types": "tsc -p tsconfig.json --noEmit --paths null", "test": "jest", - "manifest": "node ./scripts/gen-collection-info.js" + "manifest": "ts-node ./scripts/gen-collection-info.ts" }, "dependencies": { "@budibase/handlebars-helpers": "^0.13.1", @@ -34,8 +33,7 @@ }, "devDependencies": { "@rollup/plugin-commonjs": "^17.1.0", - "@rollup/plugin-json": "^4.1.0", - "concurrently": "^8.2.2", + "@rollup/plugin-typescript": "8.3.0", "doctrine": "^3.0.0", "jest": "29.7.0", "marked": "^4.0.10", diff --git a/packages/string-templates/rollup.config.js b/packages/string-templates/rollup.config.js index d7f45489c9..e7aa2ced99 100644 --- a/packages/string-templates/rollup.config.js +++ b/packages/string-templates/rollup.config.js @@ -4,31 +4,36 @@ import json from "@rollup/plugin-json" import { terser } from "rollup-plugin-terser" import builtins from "rollup-plugin-node-builtins" import globals from "rollup-plugin-node-globals" +import typescript from "@rollup/plugin-typescript" import injectProcessEnv from "rollup-plugin-inject-process-env" const production = !process.env.ROLLUP_WATCH -export default [ - { - input: "src/index.mjs", - output: { - sourcemap: !production, - format: "esm", - file: "./dist/bundle.mjs", - }, - plugins: [ - resolve({ - preferBuiltins: true, - browser: true, - }), - commonjs(), - globals(), - builtins(), - json(), - injectProcessEnv({ - NO_JS: process.env.NO_JS, - }), - production && terser(), - ], +const config = (format, outputFile) => ({ + input: "src/index.ts", + output: { + sourcemap: !production, + format, + file: outputFile, }, + plugins: [ + typescript(), + resolve({ + preferBuiltins: true, + browser: true, + }), + commonjs(), + globals(), + builtins(), + json(), + injectProcessEnv({ + NO_JS: process.env.NO_JS, + }), + production && terser(), + ], +}) + +export default [ + config("cjs", "./dist/bundle.cjs"), + config("esm", "./dist/bundle.mjs"), ] diff --git a/packages/string-templates/scripts/gen-collection-info.js b/packages/string-templates/scripts/gen-collection-info.ts similarity index 95% rename from packages/string-templates/scripts/gen-collection-info.js rename to packages/string-templates/scripts/gen-collection-info.ts index ed57fe7fae..ae2a726661 100644 --- a/packages/string-templates/scripts/gen-collection-info.js +++ b/packages/string-templates/scripts/gen-collection-info.ts @@ -22,7 +22,7 @@ const COLLECTIONS = [ "object", "uuid", ] -const FILENAME = join(__dirname, "..", "manifest.json") +const FILENAME = join(__dirname, "..", "src", "manifest.json") const outputJSON = {} const ADDED_HELPERS = { date: { @@ -126,7 +126,7 @@ const excludeFunctions = { string: ["raw"] } * This script is very specific to purpose, parsing the handlebars-helpers files to attempt to get information about them. */ function run() { - const foundNames = [] + const foundNames: string[] = [] for (let collection of COLLECTIONS) { const collectionFile = fs.readFileSync( `${path.dirname(require.resolve(HELPER_LIBRARY))}/lib/${collection}.js`, @@ -147,7 +147,7 @@ function run() { } foundNames.push(name) // this is ridiculous, but it parse the function header - const fnc = entry[1].toString() + const fnc = entry[1]!.toString() const jsDocInfo = getCommentInfo(collectionFile, fnc) let args = jsDocInfo.tags .filter(tag => tag.title === "param") @@ -176,8 +176,8 @@ function run() { } // convert all markdown to HTML - for (let collection of Object.values(outputJSON)) { - for (let helper of Object.values(collection)) { + for (let collection of Object.values(outputJSON)) { + for (let helper of Object.values(collection)) { helper.description = marked.parse(helper.description) } } diff --git a/packages/string-templates/src/conversion/index.js b/packages/string-templates/src/conversion/index.ts similarity index 85% rename from packages/string-templates/src/conversion/index.js rename to packages/string-templates/src/conversion/index.ts index 10aaef0d2f..7eb5ea71af 100644 --- a/packages/string-templates/src/conversion/index.js +++ b/packages/string-templates/src/conversion/index.ts @@ -1,11 +1,11 @@ -const { getJsHelperList } = require("../helpers") +import { getJsHelperList } from "../helpers" -function getLayers(fullBlock) { +function getLayers(fullBlock: string): string[] { let layers = [] while (fullBlock.length) { const start = fullBlock.lastIndexOf("("), end = fullBlock.indexOf(")") - let layer + let layer: string if (start === -1 || end === -1) { layer = fullBlock.trim() fullBlock = "" @@ -21,7 +21,7 @@ function getLayers(fullBlock) { return layers } -function getVariable(variableName) { +function getVariable(variableName: string) { if (!variableName || typeof variableName !== "string") { return variableName } @@ -47,10 +47,12 @@ function getVariable(variableName) { return `$("${variableName}")` } -function buildList(parts, value) { +function buildList(parts: string[], value: any) { function build() { return parts - .map(part => (part.startsWith("helper") ? part : getVariable(part))) + .map((part: string) => + part.startsWith("helper") ? part : getVariable(part) + ) .join(", ") } if (!value) { @@ -60,12 +62,12 @@ function buildList(parts, value) { } } -function splitBySpace(layer) { - const parts = [] +function splitBySpace(layer: string) { + const parts: string[] = [] let started = null, endChar = null, last = 0 - function add(str) { + function add(str: string) { const startsWith = ["]"] while (startsWith.indexOf(str.substring(0, 1)) !== -1) { str = str.substring(1, str.length) @@ -103,7 +105,7 @@ function splitBySpace(layer) { return parts } -module.exports.convertHBSBlock = (block, blockNumber) => { +export function convertHBSBlock(block: string, blockNumber: number) { const braceLength = block[2] === "{" ? 3 : 2 block = block.substring(braceLength, block.length - braceLength).trim() const layers = getLayers(block) @@ -114,7 +116,7 @@ module.exports.convertHBSBlock = (block, blockNumber) => { const parts = splitBySpace(layer) if (value || parts.length > 1 || list[parts[0]]) { // first of layer should always be the helper - const helper = parts.splice(0, 1) + const [helper] = parts.splice(0, 1) if (list[helper]) { value = `helpers.${helper}(${buildList(parts, value)})` } diff --git a/packages/string-templates/src/errors.js b/packages/string-templates/src/errors.js deleted file mode 100644 index e0dcc3de1c..0000000000 --- a/packages/string-templates/src/errors.js +++ /dev/null @@ -1,11 +0,0 @@ -class JsErrorTimeout extends Error { - code = "ERR_SCRIPT_EXECUTION_TIMEOUT" - - constructor() { - super() - } -} - -module.exports = { - JsErrorTimeout, -} diff --git a/packages/string-templates/src/errors.ts b/packages/string-templates/src/errors.ts new file mode 100644 index 0000000000..79a8a525ef --- /dev/null +++ b/packages/string-templates/src/errors.ts @@ -0,0 +1,3 @@ +export class JsErrorTimeout extends Error { + code = "ERR_SCRIPT_EXECUTION_TIMEOUT" +} diff --git a/packages/string-templates/src/helpers/Helper.js b/packages/string-templates/src/helpers/Helper.js deleted file mode 100644 index d963c87158..0000000000 --- a/packages/string-templates/src/helpers/Helper.js +++ /dev/null @@ -1,29 +0,0 @@ -class Helper { - constructor(name, fn, useValueFallback = true) { - this.name = name - this.fn = fn - this.useValueFallback = useValueFallback - } - - register(handlebars) { - // wrap the function so that no helper can cause handlebars to break - handlebars.registerHelper(this.name, (value, info) => { - let context = {} - if (info && info.data && info.data.root) { - context = info.data.root - } - const result = this.fn(value, context) - if (result == null) { - return this.useValueFallback ? value : null - } else { - return result - } - }) - } - - unregister(handlebars) { - handlebars.unregisterHelper(this.name) - } -} - -module.exports = Helper diff --git a/packages/string-templates/src/helpers/Helper.ts b/packages/string-templates/src/helpers/Helper.ts new file mode 100644 index 0000000000..a1722ac2c8 --- /dev/null +++ b/packages/string-templates/src/helpers/Helper.ts @@ -0,0 +1,34 @@ +export default class Helper { + private name: any + private fn: any + private useValueFallback: boolean + + constructor(name: string, fn: any, useValueFallback = true) { + this.name = name + this.fn = fn + this.useValueFallback = useValueFallback + } + + register(handlebars: typeof Handlebars) { + // wrap the function so that no helper can cause handlebars to break + handlebars.registerHelper( + this.name, + (value: any, info: { data: { root: {} } }) => { + let context = {} + if (info && info.data && info.data.root) { + context = info.data.root + } + const result = this.fn(value, context) + if (result == null) { + return this.useValueFallback ? value : null + } else { + return result + } + } + ) + } + + unregister(handlebars: { unregisterHelper: any }) { + handlebars.unregisterHelper(this.name) + } +} diff --git a/packages/string-templates/src/helpers/constants.js b/packages/string-templates/src/helpers/constants.ts similarity index 66% rename from packages/string-templates/src/helpers/constants.js rename to packages/string-templates/src/helpers/constants.ts index 1f8cf7a59d..ffccd36ab0 100644 --- a/packages/string-templates/src/helpers/constants.js +++ b/packages/string-templates/src/helpers/constants.ts @@ -1,4 +1,4 @@ -module.exports.HelperFunctionBuiltin = [ +export const HelperFunctionBuiltin = [ "#if", "#unless", "#each", @@ -15,11 +15,11 @@ module.exports.HelperFunctionBuiltin = [ "with", ] -module.exports.HelperFunctionNames = { +export const HelperFunctionNames = { OBJECT: "object", ALL: "all", LITERAL: "literal", JS: "js", } -module.exports.LITERAL_MARKER = "%LITERAL%" +export const LITERAL_MARKER = "%LITERAL%" diff --git a/packages/string-templates/src/helpers/date.js b/packages/string-templates/src/helpers/date.ts similarity index 72% rename from packages/string-templates/src/helpers/date.js rename to packages/string-templates/src/helpers/date.ts index 6fe8b288d6..589cb3d978 100644 --- a/packages/string-templates/src/helpers/date.js +++ b/packages/string-templates/src/helpers/date.ts @@ -1,12 +1,22 @@ -const dayjs = require("dayjs") -dayjs.extend(require("dayjs/plugin/duration")) -dayjs.extend(require("dayjs/plugin/advancedFormat")) -dayjs.extend(require("dayjs/plugin/isoWeek")) -dayjs.extend(require("dayjs/plugin/weekYear")) -dayjs.extend(require("dayjs/plugin/weekOfYear")) -dayjs.extend(require("dayjs/plugin/relativeTime")) -dayjs.extend(require("dayjs/plugin/utc")) -dayjs.extend(require("dayjs/plugin/timezone")) +import dayjs from "dayjs" + +import dayjsDurationPlugin from "dayjs/plugin/duration" +import dayjsAdvancedFormatPlugin from "dayjs/plugin/advancedFormat" +import dayjsIsoWeekPlugin from "dayjs/plugin/isoWeek" +import dayjsWeekYearPlugin from "dayjs/plugin/weekYear" +import dayjsWeekOfYearPlugin from "dayjs/plugin/weekOfYear" +import dayjsRelativeTimePlugin from "dayjs/plugin/relativeTime" +import dayjsUtcPlugin from "dayjs/plugin/utc" +import dayjsTimezonePlugin from "dayjs/plugin/timezone" + +dayjs.extend(dayjsDurationPlugin) +dayjs.extend(dayjsAdvancedFormatPlugin) +dayjs.extend(dayjsIsoWeekPlugin) +dayjs.extend(dayjsWeekYearPlugin) +dayjs.extend(dayjsWeekOfYearPlugin) +dayjs.extend(dayjsRelativeTimePlugin) +dayjs.extend(dayjsUtcPlugin) +dayjs.extend(dayjsTimezonePlugin) /** * This file was largely taken from the helper-date package - we did this for two reasons: @@ -17,11 +27,11 @@ dayjs.extend(require("dayjs/plugin/timezone")) * https://github.com/helpers/helper-date */ -function isOptions(val) { +function isOptions(val: any) { return typeof val === "object" && typeof val.hash === "object" } -function isApp(thisArg) { +function isApp(thisArg: any) { return ( typeof thisArg === "object" && typeof thisArg.options === "object" && @@ -29,7 +39,7 @@ function isApp(thisArg) { ) } -function getContext(thisArg, locals, options) { +function getContext(thisArg: any, locals: any, options: any) { if (isOptions(thisArg)) { return getContext({}, locals, thisArg) } @@ -58,7 +68,7 @@ function getContext(thisArg, locals, options) { return context } -function initialConfig(str, pattern, options) { +function initialConfig(str: any, pattern: any, options?: any) { if (isOptions(pattern)) { options = pattern pattern = null @@ -72,7 +82,7 @@ function initialConfig(str, pattern, options) { return { str, pattern, options } } -function setLocale(str, pattern, options) { +function setLocale(this: any, str: any, pattern: any, options?: any) { // if options is null then it'll get updated here const config = initialConfig(str, pattern, options) const defaults = { lang: "en", date: new Date(config.str) } @@ -83,7 +93,7 @@ function setLocale(str, pattern, options) { dayjs.locale(opts.lang || opts.language) } -module.exports.date = (str, pattern, options) => { +export const date = (str: any, pattern: any, options: any) => { const config = initialConfig(str, pattern, options) // if no args are passed, return a formatted date @@ -109,7 +119,7 @@ module.exports.date = (str, pattern, options) => { return date.format(config.pattern) } -module.exports.duration = (str, pattern, format) => { +export const duration = (str: any, pattern: any, format: any) => { const config = initialConfig(str, pattern) setLocale(config.str, config.pattern) diff --git a/packages/string-templates/src/helpers/external.js b/packages/string-templates/src/helpers/external.ts similarity index 66% rename from packages/string-templates/src/helpers/external.js rename to packages/string-templates/src/helpers/external.ts index 2487fd91df..3a95406549 100644 --- a/packages/string-templates/src/helpers/external.js +++ b/packages/string-templates/src/helpers/external.ts @@ -1,6 +1,8 @@ -const helpers = require("@budibase/handlebars-helpers") -const { date, duration } = require("./date") -const { HelperFunctionBuiltin } = require("./constants") +// @ts-ignore we don't have types for it +import helpers from "@budibase/handlebars-helpers" + +import { date, duration } from "./date" +import { HelperFunctionBuiltin } from "./constants" /** * full list of supported helpers can be found here: @@ -24,10 +26,10 @@ const ADDED_HELPERS = { duration: duration, } -exports.externalCollections = EXTERNAL_FUNCTION_COLLECTIONS -exports.addedHelpers = ADDED_HELPERS +export const externalCollections = EXTERNAL_FUNCTION_COLLECTIONS +export const addedHelpers = ADDED_HELPERS -exports.registerAll = handlebars => { +export function registerAll(handlebars: typeof Handlebars) { for (let [name, helper] of Object.entries(ADDED_HELPERS)) { handlebars.registerHelper(name, helper) } @@ -52,17 +54,17 @@ exports.registerAll = handlebars => { }) } // add date external functionality - exports.externalHelperNames = externalNames.concat(Object.keys(ADDED_HELPERS)) + externalHelperNames = externalNames.concat(Object.keys(ADDED_HELPERS)) } -exports.unregisterAll = handlebars => { +export function unregisterAll(handlebars: typeof Handlebars) { for (let name of Object.keys(ADDED_HELPERS)) { handlebars.unregisterHelper(name) } - for (let name of module.exports.externalHelperNames) { + for (let name of externalHelperNames) { handlebars.unregisterHelper(name) } - exports.externalHelperNames = [] + externalHelperNames = [] } -exports.externalHelperNames = [] +export let externalHelperNames: any[] = [] diff --git a/packages/string-templates/src/helpers/index.js b/packages/string-templates/src/helpers/index.js deleted file mode 100644 index 5e6dcbd2b9..0000000000 --- a/packages/string-templates/src/helpers/index.js +++ /dev/null @@ -1,100 +0,0 @@ -const Helper = require("./Helper") -const { SafeString } = require("handlebars") -const externalHandlebars = require("./external") -const { processJS } = require("./javascript") -const { - HelperFunctionNames, - HelperFunctionBuiltin, - LITERAL_MARKER, -} = require("./constants") -const { getJsHelperList } = require("./list") - -const HTML_SWAPS = { - "<": "<", - ">": ">", -} - -function isObject(value) { - if (value == null || typeof value !== "object") { - return false - } - return ( - value.toString() === "[object Object]" || - (value.length > 0 && typeof value[0] === "object") - ) -} - -const HELPERS = [ - // external helpers - new Helper(HelperFunctionNames.OBJECT, value => { - return new SafeString(JSON.stringify(value)) - }), - // javascript helper - new Helper(HelperFunctionNames.JS, processJS, false), - // this help is applied to all statements - new Helper(HelperFunctionNames.ALL, (value, inputs) => { - const { __opts } = inputs - if (isObject(value)) { - return new SafeString(JSON.stringify(value)) - } - // null/undefined values produce bad results - if (__opts && __opts.onlyFound && value == null) { - return __opts.input - } - if (value == null || typeof value !== "string") { - return value == null ? "" : value - } - if (value && value.string) { - value = value.string - } - let text = value - if (__opts && __opts.escapeNewlines) { - text = value.replace(/\n/g, "\\n") - } - text = new SafeString(text.replace(/&/g, "&")) - if (text == null || typeof text !== "string") { - return text - } - return text.replace(/[<>]/g, tag => { - return HTML_SWAPS[tag] || tag - }) - }), - // adds a note for post-processor - new Helper(HelperFunctionNames.LITERAL, value => { - if (value === undefined) { - return "" - } - const type = typeof value - const outputVal = type === "object" ? JSON.stringify(value) : value - return `{{${LITERAL_MARKER} ${type}-${outputVal}}}` - }), -] - -module.exports.HelperNames = () => { - return Object.values(HelperFunctionNames).concat( - HelperFunctionBuiltin, - externalHandlebars.externalHelperNames - ) -} - -module.exports.registerMinimum = handlebars => { - for (let helper of HELPERS) { - helper.register(handlebars) - } -} - -module.exports.registerAll = handlebars => { - module.exports.registerMinimum(handlebars) - // register imported helpers - externalHandlebars.registerAll(handlebars) -} - -module.exports.unregisterAll = handlebars => { - for (let helper of HELPERS) { - helper.unregister(handlebars) - } - // unregister all imported helpers - externalHandlebars.unregisterAll(handlebars) -} - -module.exports.getJsHelperList = getJsHelperList diff --git a/packages/string-templates/src/helpers/index.ts b/packages/string-templates/src/helpers/index.ts new file mode 100644 index 0000000000..595440ad55 --- /dev/null +++ b/packages/string-templates/src/helpers/index.ts @@ -0,0 +1,103 @@ +import Helper from "./Helper" +import { SafeString } from "handlebars" +import * as externalHandlebars from "./external" +import { processJS } from "./javascript" +import { + HelperFunctionNames, + HelperFunctionBuiltin, + LITERAL_MARKER, +} from "./constants" + +export { getJsHelperList } from "./list" + +const HTML_SWAPS = { + "<": "<", + ">": ">", +} + +function isObject(value: string | any[]) { + if (value == null || typeof value !== "object") { + return false + } + return ( + value.toString() === "[object Object]" || + (value.length > 0 && typeof value[0] === "object") + ) +} + +const HELPERS = [ + // external helpers + new Helper(HelperFunctionNames.OBJECT, (value: any) => { + return new SafeString(JSON.stringify(value)) + }), + // javascript helper + new Helper(HelperFunctionNames.JS, processJS, false), + // this help is applied to all statements + new Helper( + HelperFunctionNames.ALL, + (value: string, inputs: { __opts: any }) => { + const { __opts } = inputs + if (isObject(value)) { + return new SafeString(JSON.stringify(value)) + } + // null/undefined values produce bad results + if (__opts && __opts.onlyFound && value == null) { + return __opts.input + } + if (value == null || typeof value !== "string") { + return value == null ? "" : value + } + // TODO: check, this should always be false + if (value && (value as any).string) { + value = (value as any).string + } + let text: any = value + if (__opts && __opts.escapeNewlines) { + text = value.replace(/\n/g, "\\n") + } + text = new SafeString(text.replace(/&/g, "&")) + if (text == null || typeof text !== "string") { + return text + } + return text.replace(/[<>]/g, (tag: string) => { + return HTML_SWAPS[tag as keyof typeof HTML_SWAPS] || tag + }) + } + ), + // adds a note for post-processor + new Helper(HelperFunctionNames.LITERAL, (value: any) => { + if (value === undefined) { + return "" + } + const type = typeof value + const outputVal = type === "object" ? JSON.stringify(value) : value + return `{{${LITERAL_MARKER} ${type}-${outputVal}}}` + }), +] + +export function HelperNames() { + return Object.values(HelperFunctionNames).concat( + HelperFunctionBuiltin, + externalHandlebars.externalHelperNames + ) +} + +export function registerMinimum(handlebars: typeof Handlebars) { + for (let helper of HELPERS) { + helper.register(handlebars) + } +} + +export function registerAll(handlebars: typeof Handlebars) { + registerMinimum(handlebars) + // register imported helpers + externalHandlebars.registerAll(handlebars) +} + +export function unregisterAll(handlebars: any) { + for (let helper of HELPERS) { + helper.unregister(handlebars) + } + // unregister all imported helpers + externalHandlebars.unregisterAll(handlebars) +} diff --git a/packages/string-templates/src/helpers/javascript.js b/packages/string-templates/src/helpers/javascript.ts similarity index 72% rename from packages/string-templates/src/helpers/javascript.js rename to packages/string-templates/src/helpers/javascript.ts index 5be2619463..931cc46dc7 100644 --- a/packages/string-templates/src/helpers/javascript.js +++ b/packages/string-templates/src/helpers/javascript.ts @@ -1,22 +1,24 @@ -const { atob, isBackendService, isJSAllowed } = require("../utilities") -const cloneDeep = require("lodash.clonedeep") -const { LITERAL_MARKER } = require("../helpers/constants") -const { getJsHelperList } = require("./list") -const { iifeWrapper } = require("../iife") +import { atob, isJSAllowed } from "../utilities" +import cloneDeep from "lodash/fp/cloneDeep" +import { LITERAL_MARKER } from "../helpers/constants" +import { getJsHelperList } from "./list" +import { iifeWrapper } from "../iife" // The method of executing JS scripts depends on the bundle being built. // This setter is used in the entrypoint (either index.js or index.mjs). -let runJS -module.exports.setJSRunner = runner => (runJS = runner) -module.exports.removeJSRunner = () => { +let runJS: ((js: string, context: any) => any) | undefined = undefined +export const setJSRunner = (runner: typeof runJS) => (runJS = runner) + +export const removeJSRunner = () => { runJS = undefined } -let onErrorLog -module.exports.setOnErrorLog = delegate => (onErrorLog = delegate) +let onErrorLog: (message: Error) => void +export const setOnErrorLog = (delegate: typeof onErrorLog) => + (onErrorLog = delegate) // Helper utility to strip square brackets from a value -const removeSquareBrackets = value => { +const removeSquareBrackets = (value: string) => { if (!value || typeof value !== "string") { return value } @@ -30,7 +32,7 @@ const removeSquareBrackets = value => { // Our context getter function provided to JS code as $. // Extracts a value from context. -const getContextValue = (path, context) => { +const getContextValue = (path: string, context: any) => { let data = context path.split(".").forEach(key => { if (data == null || typeof data !== "object") { @@ -42,8 +44,8 @@ const getContextValue = (path, context) => { } // Evaluates JS code against a certain context -module.exports.processJS = (handlebars, context) => { - if (!isJSAllowed() || (isBackendService() && !runJS)) { +export function processJS(handlebars: string, context: any) { + if (!isJSAllowed() || !runJS) { throw new Error("JS disabled in environment.") } try { @@ -53,8 +55,8 @@ module.exports.processJS = (handlebars, context) => { // Transform snippets into an object for faster access, and cache previously // evaluated snippets - let snippetMap = {} - let snippetCache = {} + let snippetMap: any = {} + let snippetCache: any = {} for (let snippet of context.snippets || []) { snippetMap[snippet.name] = snippet.code } @@ -64,7 +66,7 @@ module.exports.processJS = (handlebars, context) => { // app context. const clonedContext = cloneDeep({ ...context, snippets: null }) const sandboxContext = { - $: path => getContextValue(path, clonedContext), + $: (path: string) => getContextValue(path, clonedContext), helpers: getJsHelperList(), // Proxy to evaluate snippets when running in the browser @@ -84,7 +86,7 @@ module.exports.processJS = (handlebars, context) => { // Create a sandbox with our context and run the JS const res = { data: runJS(js, sandboxContext) } return `{{${LITERAL_MARKER} js_result-${JSON.stringify(res)}}}` - } catch (error) { + } catch (error: any) { onErrorLog && onErrorLog(error) if (error.code === "ERR_SCRIPT_EXECUTION_TIMEOUT") { diff --git a/packages/string-templates/src/helpers/list.js b/packages/string-templates/src/helpers/list.ts similarity index 70% rename from packages/string-templates/src/helpers/list.js rename to packages/string-templates/src/helpers/list.ts index 883ab5678e..361558e04d 100644 --- a/packages/string-templates/src/helpers/list.js +++ b/packages/string-templates/src/helpers/list.ts @@ -1,7 +1,7 @@ -const { date, duration } = require("./date") +import { date, duration } from "./date" // https://github.com/evanw/esbuild/issues/56 -const externalCollections = { +const getExternalCollections = (): Record any> => ({ math: require("@budibase/handlebars-helpers/lib/math"), array: require("@budibase/handlebars-helpers/lib/array"), number: require("@budibase/handlebars-helpers/lib/number"), @@ -11,32 +11,32 @@ const externalCollections = { object: require("@budibase/handlebars-helpers/lib/object"), regex: require("@budibase/handlebars-helpers/lib/regex"), uuid: require("@budibase/handlebars-helpers/lib/uuid"), -} +}) -const helpersToRemoveForJs = ["sortBy"] -module.exports.helpersToRemoveForJs = helpersToRemoveForJs +export const helpersToRemoveForJs = ["sortBy"] const addedHelpers = { date: date, duration: duration, } -let helpers = undefined +let helpers: Record -module.exports.getJsHelperList = () => { +export function getJsHelperList() { if (helpers) { return helpers } helpers = {} - for (let collection of Object.values(externalCollections)) { + for (let collection of Object.values(getExternalCollections())) { for (let [key, func] of Object.entries(collection)) { // Handlebars injects the hbs options to the helpers by default. We are adding an empty {} as a last parameter to simulate it - helpers[key] = (...props) => func(...props, {}) + helpers[key] = (...props: any) => func(...props, {}) } } - for (let key of Object.keys(addedHelpers)) { - helpers[key] = addedHelpers[key] + helpers = { + ...helpers, + addedHelpers, } for (const toRemove of helpersToRemoveForJs) { diff --git a/packages/string-templates/src/iife.js b/packages/string-templates/src/iife.js deleted file mode 100644 index d043c14565..0000000000 --- a/packages/string-templates/src/iife.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports.iifeWrapper = script => { - return `(function(){\n${script}\n})();` -} diff --git a/packages/string-templates/src/iife.ts b/packages/string-templates/src/iife.ts new file mode 100644 index 0000000000..0ce3ad8f8e --- /dev/null +++ b/packages/string-templates/src/iife.ts @@ -0,0 +1,3 @@ +export const iifeWrapper = (script: string) => { + return `(function(){\n${script}\n})();` +} diff --git a/packages/string-templates/src/index.mjs b/packages/string-templates/src/index.mjs deleted file mode 100644 index f54ca7e23e..0000000000 --- a/packages/string-templates/src/index.mjs +++ /dev/null @@ -1,26 +0,0 @@ -import templates from "./index.js" - -/** - * ES6 entrypoint for rollup - */ -export const isValid = templates.isValid -export const makePropSafe = templates.makePropSafe -export const getManifest = templates.getManifest -export const isJSBinding = templates.isJSBinding -export const encodeJSBinding = templates.encodeJSBinding -export const decodeJSBinding = templates.decodeJSBinding -export const processStringSync = templates.processStringSync -export const processObjectSync = templates.processObjectSync -export const processString = templates.processString -export const processObject = templates.processObject -export const doesContainStrings = templates.doesContainStrings -export const doesContainString = templates.doesContainString -export const disableEscaping = templates.disableEscaping -export const findHBSBlocks = templates.findHBSBlocks -export const convertToJS = templates.convertToJS -export const setJSRunner = templates.setJSRunner -export const setOnErrorLog = templates.setOnErrorLog -export const FIND_ANY_HBS_REGEX = templates.FIND_ANY_HBS_REGEX -export const helpersToRemoveForJs = templates.helpersToRemoveForJs - -export * from "./errors.js" diff --git a/packages/string-templates/src/index.js b/packages/string-templates/src/index.ts similarity index 73% rename from packages/string-templates/src/index.js rename to packages/string-templates/src/index.ts index 5ae773516f..3b0d7c0115 100644 --- a/packages/string-templates/src/index.js +++ b/packages/string-templates/src/index.ts @@ -1,24 +1,30 @@ -const vm = require("vm") -const handlebars = require("handlebars") -const { registerAll, registerMinimum } = require("./helpers/index") -const processors = require("./processors") -const { atob, btoa, isBackendService } = require("./utilities") -const { iifeWrapper } = require("./iife") -const manifest = require("../manifest.json") -const { +import { Context, createContext, runInNewContext } from "vm" +import { create } from "handlebars" +import { registerAll, registerMinimum } from "./helpers/index" +import { preprocess, postprocess } from "./processors" +import { + atob, + btoa, + isBackendService, FIND_HBS_REGEX, FIND_ANY_HBS_REGEX, findDoubleHbsInstances, -} = require("./utilities") -const { convertHBSBlock } = require("./conversion") -const javascript = require("./helpers/javascript") -const { helpersToRemoveForJs } = require("./helpers/list") +} from "./utilities" +import { convertHBSBlock } from "./conversion" +import { setJSRunner, removeJSRunner } from "./helpers/javascript" +import { helpersToRemoveForJs } from "./helpers/list" -const hbsInstance = handlebars.create() +import manifest from "./manifest.json" +import { ProcessOptions } from "./types" + +export { setJSRunner, setOnErrorLog } from "./helpers/javascript" +export { iifeWrapper } from "./iife" + +const hbsInstance = create() registerAll(hbsInstance) -const hbsInstanceNoHelpers = handlebars.create() +const hbsInstanceNoHelpers = create() registerMinimum(hbsInstanceNoHelpers) -const defaultOpts = { +const defaultOpts: ProcessOptions = { noHelpers: false, cacheTemplates: false, noEscaping: false, @@ -29,7 +35,7 @@ const defaultOpts = { /** * Utility function to check if the object is valid. */ -function testObject(object) { +function testObject(object: any) { // JSON stringify will fail if there are any cycles, stops infinite recursion try { JSON.stringify(object) @@ -41,8 +47,8 @@ function testObject(object) { /** * Creates a HBS template function for a given string, and optionally caches it. */ -let templateCache = {} -function createTemplate(string, opts) { +const templateCache: Record> = {} +function createTemplate(string: string, opts?: ProcessOptions) { opts = { ...defaultOpts, ...opts } // Finalising adds a helper, can't do this with no helpers @@ -53,11 +59,11 @@ function createTemplate(string, opts) { return templateCache[key] } - string = processors.preprocess(string, opts) + string = preprocess(string, opts) // Optionally disable built in HBS escaping if (opts.noEscaping) { - string = exports.disableEscaping(string) + string = disableEscaping(string) } // This does not throw an error when template can't be fulfilled, @@ -78,24 +84,25 @@ function createTemplate(string, opts) { * @param {object|undefined} [opts] optional - specify some options for processing. * @returns {Promise} The structure input, as fully updated as possible. */ -module.exports.processObject = async (object, context, opts) => { +export async function processObject>( + object: T, + context: object, + opts?: { noHelpers?: boolean; escapeNewlines?: boolean; onlyFound?: boolean } +): Promise { testObject(object) - for (let key of Object.keys(object || {})) { + + for (const key of Object.keys(object || {})) { if (object[key] != null) { - let val = object[key] + const val = object[key] + let parsedValue if (typeof val === "string") { - object[key] = await module.exports.processString( - object[key], - context, - opts - ) + parsedValue = await processString(object[key], context, opts) } else if (typeof val === "object") { - object[key] = await module.exports.processObject( - object[key], - context, - opts - ) + parsedValue = await processObject(object[key], context, opts) } + + // @ts-ignore + object[key] = parsedValue } } return object @@ -109,9 +116,13 @@ module.exports.processObject = async (object, context, opts) => { * @param {object|undefined} [opts] optional - specify some options for processing. * @returns {Promise} The enriched string, all templates should have been replaced if they can be. */ -module.exports.processString = async (string, context, opts) => { +export async function processString( + string: string, + context: object, + opts?: ProcessOptions +): Promise { // TODO: carry out any async calls before carrying out async call - return module.exports.processStringSync(string, context, opts) + return processStringSync(string, context, opts) } /** @@ -123,14 +134,18 @@ module.exports.processString = async (string, context, opts) => { * @param {object|undefined} [opts] optional - specify some options for processing. * @returns {object|array} The structure input, as fully updated as possible. */ -module.exports.processObjectSync = (object, context, opts) => { +export function processObjectSync( + object: { [x: string]: any }, + context: any, + opts: any +): object | Array { testObject(object) for (let key of Object.keys(object || {})) { let val = object[key] if (typeof val === "string") { - object[key] = module.exports.processStringSync(object[key], context, opts) + object[key] = processStringSync(object[key], context, opts) } else if (typeof val === "object") { - object[key] = module.exports.processObjectSync(object[key], context, opts) + object[key] = processObjectSync(object[key], context, opts) } } return object @@ -144,29 +159,32 @@ module.exports.processObjectSync = (object, context, opts) => { * @param {object|undefined} [opts] optional - specify some options for processing. * @returns {string} The enriched string, all templates should have been replaced if they can be. */ -module.exports.processStringSync = (string, context, opts) => { +export function processStringSync( + string: string, + context?: object, + opts?: ProcessOptions +): string { // Take a copy of input in case of error const input = string if (typeof string !== "string") { throw "Cannot process non-string types." } - function process(stringPart) { + function process(stringPart: string) { const template = createTemplate(stringPart, opts) const now = Math.floor(Date.now() / 1000) * 1000 - return processors.postprocess( - template({ - now: new Date(now).toISOString(), - __opts: { - ...opts, - input: stringPart, - }, - ...context, - }) - ) + const processedString = template({ + now: new Date(now).toISOString(), + __opts: { + ...opts, + input: stringPart, + }, + ...context, + }) + return postprocess(processedString) } try { if (opts && opts.onlyFound) { - const blocks = exports.findHBSBlocks(string) + const blocks = findHBSBlocks(string) for (let block of blocks) { const outcome = process(block) string = string.replace(block, outcome) @@ -186,7 +204,7 @@ module.exports.processStringSync = (string, context, opts) => { * this function will find any double braces and switch to triple. * @param string the string to have double HBS statements converted to triple. */ -module.exports.disableEscaping = string => { +export function disableEscaping(string: string) { const matches = findDoubleHbsInstances(string) if (matches == null) { return string @@ -207,7 +225,7 @@ module.exports.disableEscaping = string => { * @param {string} property The property which is to be wrapped. * @returns {string} The wrapped property ready to be added to a templating string. */ -module.exports.makePropSafe = property => { +export function makePropSafe(property: any): string { return `[${property}]`.replace("[[", "[").replace("]]", "]") } @@ -217,7 +235,7 @@ module.exports.makePropSafe = property => { * @param [opts] optional - specify some options for processing. * @returns {boolean} Whether or not the input string is valid. */ -module.exports.isValid = (string, opts) => { +export function isValid(string: any, opts?: any): boolean { const validCases = [ "string", "number", @@ -238,7 +256,7 @@ module.exports.isValid = (string, opts) => { }) template(context) return true - } catch (err) { + } catch (err: any) { const msg = err && err.message ? err.message : err if (!msg) { return false @@ -259,7 +277,7 @@ module.exports.isValid = (string, opts) => { * This manifest provides information about each of the helpers and how it can be used. * @returns The manifest JSON which has been generated from the helpers. */ -module.exports.getManifest = () => { +export function getManifest() { return manifest } @@ -268,8 +286,8 @@ module.exports.getManifest = () => { * @param handlebars the HBS expression to check * @returns {boolean} whether the expression is JS or not */ -module.exports.isJSBinding = handlebars => { - return module.exports.decodeJSBinding(handlebars) != null +export function isJSBinding(handlebars: any): boolean { + return decodeJSBinding(handlebars) != null } /** @@ -277,7 +295,7 @@ module.exports.isJSBinding = handlebars => { * @param javascript the JS code to encode * @returns {string} the JS HBS expression */ -module.exports.encodeJSBinding = javascript => { +export function encodeJSBinding(javascript: string): string { return `{{ js "${btoa(javascript)}" }}` } @@ -286,7 +304,7 @@ module.exports.encodeJSBinding = javascript => { * @param handlebars the JS HBS expression * @returns {string|null} the raw JS code */ -module.exports.decodeJSBinding = handlebars => { +export function decodeJSBinding(handlebars: string): string | null { if (!handlebars || typeof handlebars !== "string") { return null } @@ -311,7 +329,7 @@ module.exports.decodeJSBinding = handlebars => { * @param {string[]} strings The strings to look for. * @returns {boolean} Will return true if all strings found in HBS statement. */ -module.exports.doesContainStrings = (template, strings) => { +export function doesContainStrings(template: string, strings: any[]): boolean { let regexp = new RegExp(FIND_HBS_REGEX) let matches = template.match(regexp) if (matches == null) { @@ -319,8 +337,8 @@ module.exports.doesContainStrings = (template, strings) => { } for (let match of matches) { let hbs = match - if (exports.isJSBinding(match)) { - hbs = exports.decodeJSBinding(match) + if (isJSBinding(match)) { + hbs = decodeJSBinding(match)! } let allFound = true for (let string of strings) { @@ -341,7 +359,7 @@ module.exports.doesContainStrings = (template, strings) => { * @param {string} string The string to search within. * @return {string[]} The found HBS blocks. */ -module.exports.findHBSBlocks = string => { +export function findHBSBlocks(string: string): string[] { if (!string || typeof string !== "string") { return [] } @@ -362,18 +380,15 @@ module.exports.findHBSBlocks = string => { * @param {string} string The word or sentence to search for. * @returns {boolean} The this return true if the string is found, false if not. */ -module.exports.doesContainString = (template, string) => { - return exports.doesContainStrings(template, [string]) +export function doesContainString(template: any, string: any): boolean { + return doesContainStrings(template, [string]) } -module.exports.setJSRunner = javascript.setJSRunner -module.exports.setOnErrorLog = javascript.setOnErrorLog - -module.exports.convertToJS = hbs => { - const blocks = exports.findHBSBlocks(hbs) +export function convertToJS(hbs: string) { + const blocks = findHBSBlocks(hbs) let js = "return `", - prevBlock = null - const variables = {} + prevBlock: string | null = null + const variables: Record = {} if (blocks.length === 0) { js += hbs } @@ -387,7 +402,7 @@ module.exports.convertToJS = hbs => { prevBlock = block const { variable, value } = convertHBSBlock(block, count++) variables[variable] = value - js += `${stringPart.split()}\${${variable}}` + js += `${[stringPart]}\${${variable}}` } let varBlock = "" for (let [variable, value] of Object.entries(variables)) { @@ -397,34 +412,34 @@ module.exports.convertToJS = hbs => { return `${varBlock}${js}` } -module.exports.FIND_ANY_HBS_REGEX = FIND_ANY_HBS_REGEX +const _FIND_ANY_HBS_REGEX = FIND_ANY_HBS_REGEX +export { _FIND_ANY_HBS_REGEX as FIND_ANY_HBS_REGEX } -const errors = require("./errors") -// We cannot use dynamic exports, otherwise the typescript file will not be generating it -module.exports.JsErrorTimeout = errors.JsErrorTimeout +export { JsErrorTimeout } from "./errors" -module.exports.helpersToRemoveForJs = helpersToRemoveForJs +const _helpersToRemoveForJs = helpersToRemoveForJs +export { _helpersToRemoveForJs as helpersToRemoveForJs } function defaultJSSetup() { if (!isBackendService()) { /** * Use polyfilled vm to run JS scripts in a browser Env */ - javascript.setJSRunner((js, context) => { + setJSRunner((js: string, context: Context) => { context = { ...context, alert: undefined, setInterval: undefined, setTimeout: undefined, } - vm.createContext(context) - return vm.runInNewContext(js, context, { timeout: 1000 }) + createContext(context) + return runInNewContext(js, context, { timeout: 1000 }) }) } else { - javascript.removeJSRunner() + removeJSRunner() } } defaultJSSetup() -module.exports.defaultJSSetup = defaultJSSetup -module.exports.iifeWrapper = iifeWrapper +const _defaultJSSetup = defaultJSSetup +export { _defaultJSSetup as defaultJSSetup } diff --git a/packages/string-templates/manifest.json b/packages/string-templates/src/manifest.json similarity index 100% rename from packages/string-templates/manifest.json rename to packages/string-templates/src/manifest.json diff --git a/packages/string-templates/src/processors/index.js b/packages/string-templates/src/processors/index.ts similarity index 67% rename from packages/string-templates/src/processors/index.js rename to packages/string-templates/src/processors/index.ts index aae18aed8b..308ac5adf4 100644 --- a/packages/string-templates/src/processors/index.js +++ b/packages/string-templates/src/processors/index.ts @@ -1,8 +1,9 @@ -const { FIND_HBS_REGEX } = require("../utilities") -const preprocessor = require("./preprocessor") -const postprocessor = require("./postprocessor") +import { FIND_HBS_REGEX } from "../utilities" +import * as preprocessor from "./preprocessor" +import * as postprocessor from "./postprocessor" +import { ProcessOptions } from "../types" -function process(output, processors, opts) { +function process(output: string, processors: any[], opts?: ProcessOptions) { for (let processor of processors) { // if a literal statement has occurred stop if (typeof output !== "string") { @@ -21,7 +22,7 @@ function process(output, processors, opts) { return output } -module.exports.preprocess = (string, opts) => { +export function preprocess(string: string, opts: ProcessOptions) { let processors = preprocessor.processors if (opts.noFinalise) { processors = processors.filter( @@ -30,7 +31,7 @@ module.exports.preprocess = (string, opts) => { } return process(string, processors, opts) } -module.exports.postprocess = string => { +export function postprocess(string: string) { let processors = postprocessor.processors return process(string, processors) } diff --git a/packages/string-templates/src/processors/postprocessor.js b/packages/string-templates/src/processors/postprocessor.js deleted file mode 100644 index f78a572d07..0000000000 --- a/packages/string-templates/src/processors/postprocessor.js +++ /dev/null @@ -1,49 +0,0 @@ -const { LITERAL_MARKER } = require("../helpers/constants") - -const PostProcessorNames = { - CONVERT_LITERALS: "convert-literals", -} - -/* eslint-disable no-unused-vars */ -class Postprocessor { - constructor(name, fn) { - this.name = name - this.fn = fn - } - - process(statement) { - return this.fn(statement) - } -} - -module.exports.PostProcessorNames = PostProcessorNames - -module.exports.processors = [ - new Postprocessor(PostProcessorNames.CONVERT_LITERALS, statement => { - if (typeof statement !== "string" || !statement.includes(LITERAL_MARKER)) { - return statement - } - const splitMarkerIndex = statement.indexOf("-") - const type = statement.substring(12, splitMarkerIndex) - const value = statement.substring( - splitMarkerIndex + 1, - statement.length - 2 - ) - switch (type) { - case "string": - return value - case "number": - return parseFloat(value) - case "boolean": - return value === "true" - case "object": - return JSON.parse(value) - case "js_result": - // We use the literal helper to process the result of JS expressions - // as we want to be able to return any types. - // We wrap the value in an abject to be able to use undefined properly. - return JSON.parse(value).data - } - return value - }), -] diff --git a/packages/string-templates/src/processors/postprocessor.ts b/packages/string-templates/src/processors/postprocessor.ts new file mode 100644 index 0000000000..6f7260718b --- /dev/null +++ b/packages/string-templates/src/processors/postprocessor.ts @@ -0,0 +1,56 @@ +import { LITERAL_MARKER } from "../helpers/constants" + +export const PostProcessorNames = { + CONVERT_LITERALS: "convert-literals", +} + +/* eslint-disable no-unused-vars */ +class Postprocessor { + name: string + private fn: any + + constructor(name: string, fn: any) { + this.name = name + this.fn = fn + } + + process(statement: any) { + return this.fn(statement) + } +} + +export const processors = [ + new Postprocessor( + PostProcessorNames.CONVERT_LITERALS, + (statement: string) => { + if ( + typeof statement !== "string" || + !statement.includes(LITERAL_MARKER) + ) { + return statement + } + const splitMarkerIndex = statement.indexOf("-") + const type = statement.substring(12, splitMarkerIndex) + const value = statement.substring( + splitMarkerIndex + 1, + statement.length - 2 + ) + switch (type) { + case "string": + return value + case "number": + return parseFloat(value) + case "boolean": + return value === "true" + case "object": + return JSON.parse(value) + case "js_result": + // We use the literal helper to process the result of JS expressions + // as we want to be able to return any types. + // We wrap the value in an abject to be able to use undefined properly. + return JSON.parse(value).data + } + return value + } + ), +] diff --git a/packages/string-templates/src/processors/preprocessor.js b/packages/string-templates/src/processors/preprocessor.js deleted file mode 100644 index 185a3ab38a..0000000000 --- a/packages/string-templates/src/processors/preprocessor.js +++ /dev/null @@ -1,78 +0,0 @@ -const { HelperNames } = require("../helpers") -const { swapStrings, isAlphaNumeric } = require("../utilities") - -const FUNCTION_CASES = ["#", "else", "/"] - -const PreprocessorNames = { - SWAP_TO_DOT: "swap-to-dot-notation", - FIX_FUNCTIONS: "fix-functions", - FINALISE: "finalise", -} - -/* eslint-disable no-unused-vars */ -class Preprocessor { - constructor(name, fn) { - this.name = name - this.fn = fn - } - - process(fullString, statement, opts) { - const output = this.fn(statement, opts) - const idx = fullString.indexOf(statement) - return swapStrings(fullString, idx, statement.length, output) - } -} - -module.exports.processors = [ - new Preprocessor(PreprocessorNames.SWAP_TO_DOT, statement => { - let startBraceIdx = statement.indexOf("[") - let lastIdx = 0 - while (startBraceIdx !== -1) { - // if the character previous to the literal specifier is alphanumeric this should happen - if (isAlphaNumeric(statement.charAt(startBraceIdx - 1))) { - statement = swapStrings(statement, startBraceIdx + lastIdx, 1, ".[") - } - lastIdx = startBraceIdx + 1 - const nextBraceIdx = statement.substring(lastIdx + 1).indexOf("[") - startBraceIdx = nextBraceIdx > 0 ? lastIdx + 1 + nextBraceIdx : -1 - } - return statement - }), - - new Preprocessor(PreprocessorNames.FIX_FUNCTIONS, statement => { - for (let specialCase of FUNCTION_CASES) { - const toFind = `{ ${specialCase}`, - replacement = `{${specialCase}` - statement = statement.replace(new RegExp(toFind, "g"), replacement) - } - return statement - }), - - new Preprocessor(PreprocessorNames.FINALISE, (statement, opts) => { - const noHelpers = opts && opts.noHelpers - let insideStatement = statement.slice(2, statement.length - 2) - if (insideStatement.charAt(0) === " ") { - insideStatement = insideStatement.slice(1) - } - if (insideStatement.charAt(insideStatement.length - 1) === " ") { - insideStatement = insideStatement.slice(0, insideStatement.length - 1) - } - const possibleHelper = insideStatement.split(" ")[0] - // function helpers can't be wrapped - for (let specialCase of FUNCTION_CASES) { - if (possibleHelper.includes(specialCase)) { - return statement - } - } - const testHelper = possibleHelper.trim().toLowerCase() - if ( - !noHelpers && - HelperNames().some(option => testHelper === option.toLowerCase()) - ) { - insideStatement = `(${insideStatement})` - } - return `{{ all ${insideStatement} }}` - }), -] - -module.exports.PreprocessorNames = PreprocessorNames diff --git a/packages/string-templates/src/processors/preprocessor.ts b/packages/string-templates/src/processors/preprocessor.ts new file mode 100644 index 0000000000..141b2be3a9 --- /dev/null +++ b/packages/string-templates/src/processors/preprocessor.ts @@ -0,0 +1,82 @@ +import { HelperNames } from "../helpers" +import { swapStrings, isAlphaNumeric } from "../utilities" + +const FUNCTION_CASES = ["#", "else", "/"] + +export const PreprocessorNames = { + SWAP_TO_DOT: "swap-to-dot-notation", + FIX_FUNCTIONS: "fix-functions", + FINALISE: "finalise", +} + +/* eslint-disable no-unused-vars */ +class Preprocessor { + name: string + private fn: any + + constructor(name: string, fn: any) { + this.name = name + this.fn = fn + } + + process(fullString: string, statement: string, opts: Object) { + const output = this.fn(statement, opts) + const idx = fullString.indexOf(statement) + return swapStrings(fullString, idx, statement.length, output) + } +} + +export const processors = [ + new Preprocessor(PreprocessorNames.SWAP_TO_DOT, (statement: string) => { + let startBraceIdx = statement.indexOf("[") + let lastIdx = 0 + while (startBraceIdx !== -1) { + // if the character previous to the literal specifier is alphanumeric this should happen + if (isAlphaNumeric(statement.charAt(startBraceIdx - 1))) { + statement = swapStrings(statement, startBraceIdx + lastIdx, 1, ".[") + } + lastIdx = startBraceIdx + 1 + const nextBraceIdx = statement.substring(lastIdx + 1).indexOf("[") + startBraceIdx = nextBraceIdx > 0 ? lastIdx + 1 + nextBraceIdx : -1 + } + return statement + }), + + new Preprocessor(PreprocessorNames.FIX_FUNCTIONS, (statement: string) => { + for (let specialCase of FUNCTION_CASES) { + const toFind = `{ ${specialCase}`, + replacement = `{${specialCase}` + statement = statement.replace(new RegExp(toFind, "g"), replacement) + } + return statement + }), + + new Preprocessor( + PreprocessorNames.FINALISE, + (statement: string, opts: { noHelpers: any }) => { + const noHelpers = opts && opts.noHelpers + let insideStatement = statement.slice(2, statement.length - 2) + if (insideStatement.charAt(0) === " ") { + insideStatement = insideStatement.slice(1) + } + if (insideStatement.charAt(insideStatement.length - 1) === " ") { + insideStatement = insideStatement.slice(0, insideStatement.length - 1) + } + const possibleHelper = insideStatement.split(" ")[0] + // function helpers can't be wrapped + for (let specialCase of FUNCTION_CASES) { + if (possibleHelper.includes(specialCase)) { + return statement + } + } + const testHelper = possibleHelper.trim().toLowerCase() + if ( + !noHelpers && + HelperNames().some(option => testHelper === option.toLowerCase()) + ) { + insideStatement = `(${insideStatement})` + } + return `{{ all ${insideStatement} }}` + } + ), +] diff --git a/packages/string-templates/src/types.ts b/packages/string-templates/src/types.ts new file mode 100644 index 0000000000..1e1ef048a4 --- /dev/null +++ b/packages/string-templates/src/types.ts @@ -0,0 +1,8 @@ +export interface ProcessOptions { + cacheTemplates?: boolean + noEscaping?: boolean + noHelpers?: boolean + noFinalise?: boolean + escapeNewlines?: boolean + onlyFound?: boolean +} diff --git a/packages/string-templates/src/utilities.js b/packages/string-templates/src/utilities.ts similarity index 54% rename from packages/string-templates/src/utilities.js rename to packages/string-templates/src/utilities.ts index 00b2d7d855..bcb9987c89 100644 --- a/packages/string-templates/src/utilities.js +++ b/packages/string-templates/src/utilities.ts @@ -1,28 +1,28 @@ const ALPHA_NUMERIC_REGEX = /^[A-Za-z0-9]+$/g -module.exports.FIND_HBS_REGEX = /{{([^{].*?)}}/g -module.exports.FIND_ANY_HBS_REGEX = /{?{{([^{].*?)}}}?/g -module.exports.FIND_TRIPLE_HBS_REGEX = /{{{([^{].*?)}}}/g +export const FIND_HBS_REGEX = /{{([^{].*?)}}/g +export const FIND_ANY_HBS_REGEX = /{?{{([^{].*?)}}}?/g +export const FIND_TRIPLE_HBS_REGEX = /{{{([^{].*?)}}}/g -module.exports.isBackendService = () => { +export const isBackendService = () => { return typeof window === "undefined" } -module.exports.isJSAllowed = () => { +export const isJSAllowed = () => { return process && !process.env.NO_JS } // originally this could be done with a single regex using look behinds // but safari does not support this feature // original regex: /(? { +export const findDoubleHbsInstances = (string: string): string[] => { let copied = string - const doubleRegex = new RegExp(exports.FIND_HBS_REGEX) - const regex = new RegExp(exports.FIND_TRIPLE_HBS_REGEX) + const doubleRegex = new RegExp(FIND_HBS_REGEX) + const regex = new RegExp(FIND_TRIPLE_HBS_REGEX) const tripleMatches = copied.match(regex) // remove triple braces if (tripleMatches) { - tripleMatches.forEach(match => { + tripleMatches.forEach((match: string) => { copied = copied.replace(match, "") }) } @@ -30,34 +30,39 @@ module.exports.findDoubleHbsInstances = string => { return doubleMatches ? doubleMatches : [] } -module.exports.isAlphaNumeric = char => { +export const isAlphaNumeric = (char: string) => { return char.match(ALPHA_NUMERIC_REGEX) } -module.exports.swapStrings = (string, start, length, swap) => { +export const swapStrings = ( + string: string, + start: number, + length: number, + swap: string +) => { return string.slice(0, start) + swap + string.slice(start + length) } -module.exports.removeHandlebarsStatements = ( - string, +export const removeHandlebarsStatements = ( + string: string, replacement = "Invalid binding" ) => { - let regexp = new RegExp(exports.FIND_HBS_REGEX) + let regexp = new RegExp(FIND_HBS_REGEX) let matches = string.match(regexp) if (matches == null) { return string } for (let match of matches) { const idx = string.indexOf(match) - string = exports.swapStrings(string, idx, match.length, replacement) + string = swapStrings(string, idx, match.length, replacement) } return string } -module.exports.btoa = plainText => { +export const btoa = (plainText: string) => { return Buffer.from(plainText, "utf-8").toString("base64") } -module.exports.atob = base64 => { +export const atob = (base64: string) => { return Buffer.from(base64, "base64").toString("utf-8") } diff --git a/packages/string-templates/test/basic.spec.js b/packages/string-templates/test/basic.spec.ts similarity index 97% rename from packages/string-templates/test/basic.spec.js rename to packages/string-templates/test/basic.spec.ts index 0b78e3cafd..ae006f06f9 100644 --- a/packages/string-templates/test/basic.spec.js +++ b/packages/string-templates/test/basic.spec.ts @@ -1,4 +1,4 @@ -const { +import { processObject, processString, isValid, @@ -8,7 +8,7 @@ const { doesContainString, disableEscaping, findHBSBlocks, -} = require("../src/index.js") +} from "../src/index" describe("Test that the string processing works correctly", () => { it("should process a basic template string", async () => { @@ -28,7 +28,7 @@ describe("Test that the string processing works correctly", () => { it("should fail gracefully when wrong type passed in", async () => { let error = null try { - await processString(null, null) + await processString(null as any, null as any) } catch (err) { error = err } @@ -76,7 +76,7 @@ describe("Test that the object processing works correctly", () => { it("should fail gracefully when object passed in has cycles", async () => { let error = null try { - const innerObj = { a: "thing {{ a }}" } + const innerObj: any = { a: "thing {{ a }}" } innerObj.b = innerObj await processObject(innerObj, { a: 1 }) } catch (err) { @@ -98,7 +98,7 @@ describe("Test that the object processing works correctly", () => { it("should be able to handle null objects", async () => { let error = null try { - await processObject(null, null) + await processObject(null as any, null as any) } catch (err) { error = err } diff --git a/packages/string-templates/test/constants.js b/packages/string-templates/test/constants.ts similarity index 72% rename from packages/string-templates/test/constants.js rename to packages/string-templates/test/constants.ts index aac0291636..b9e31bcfcc 100644 --- a/packages/string-templates/test/constants.js +++ b/packages/string-templates/test/constants.ts @@ -1,2 +1,2 @@ -module.exports.UUID_REGEX = +export const UUID_REGEX = /^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}$/i diff --git a/packages/string-templates/test/escapes.spec.js b/packages/string-templates/test/escapes.spec.ts similarity index 98% rename from packages/string-templates/test/escapes.spec.js rename to packages/string-templates/test/escapes.spec.ts index caa2f7d0c1..ffc72a6b92 100644 --- a/packages/string-templates/test/escapes.spec.js +++ b/packages/string-templates/test/escapes.spec.ts @@ -1,4 +1,4 @@ -const { processString } = require("../src/index.js") +import { processString } from "../src/index" describe("Handling context properties with spaces in their name", () => { it("should allow through literal specifiers", async () => { diff --git a/packages/string-templates/test/hbsToJs.spec.js b/packages/string-templates/test/hbsToJs.spec.ts similarity index 97% rename from packages/string-templates/test/hbsToJs.spec.js rename to packages/string-templates/test/hbsToJs.spec.ts index 08d8ff5f67..ea375064a1 100644 --- a/packages/string-templates/test/hbsToJs.spec.js +++ b/packages/string-templates/test/hbsToJs.spec.ts @@ -1,6 +1,6 @@ -const { convertToJS } = require("../src/index.js") +import { convertToJS } from "../src/index" -function checkLines(response, lines) { +function checkLines(response: string, lines: string[]) { const toCheck = response.split("\n") let count = 0 for (let line of lines) { diff --git a/packages/string-templates/test/helpers.spec.js b/packages/string-templates/test/helpers.spec.ts similarity index 96% rename from packages/string-templates/test/helpers.spec.js rename to packages/string-templates/test/helpers.spec.ts index 86fef538d3..5f1855535d 100644 --- a/packages/string-templates/test/helpers.spec.js +++ b/packages/string-templates/test/helpers.spec.ts @@ -1,7 +1,7 @@ -const { processString, processObject, isValid } = require("../src/index.js") -const tableJson = require("./examples/table.json") -const dayjs = require("dayjs") -const { UUID_REGEX } = require("./constants") +import { processString, processObject, isValid } from "../src/index" +import tableJson from "./examples/table.json" +import dayjs from "dayjs" +import { UUID_REGEX } from "./constants" describe("test the custom helpers we have applied", () => { it("should be able to use the object helper", async () => { @@ -188,9 +188,7 @@ describe("test the date helpers", () => { time: date.toUTCString(), } ) - const formatted = new dayjs(date) - .tz("America/New_York") - .format("HH-mm-ss Z") + const formatted = dayjs(date).tz("America/New_York").format("HH-mm-ss Z") expect(output).toBe(formatted) }) @@ -200,7 +198,7 @@ describe("test the date helpers", () => { time: date.toUTCString(), }) const timezone = dayjs.tz.guess() - const offset = new dayjs(date).tz(timezone).format("Z") + const offset = dayjs(date).tz(timezone).format("Z") expect(output).toBe(offset) }) }) @@ -273,7 +271,7 @@ describe("test the string helpers", () => { }) describe("test the comparison helpers", () => { - async function compare(func, a, b) { + async function compare(func: string, a: any, b: any) { const output = await processString( `{{ #${func} a b }}Success{{ else }}Fail{{ /${func} }}`, { @@ -344,14 +342,14 @@ describe("Test the literal helper", () => { }) it("should allow use of the literal specifier for an object", async () => { - const output = await processString(`{{literal a}}`, { + const output: any = await processString(`{{literal a}}`, { a: { b: 1 }, }) expect(output.b).toBe(1) }) it("should allow use of the literal specifier for an object with dashes", async () => { - const output = await processString(`{{literal a}}`, { + const output: any = await processString(`{{literal a}}`, { a: { b: "i-have-dashes" }, }) expect(output.b).toBe("i-have-dashes") diff --git a/packages/string-templates/test/javascript.spec.js b/packages/string-templates/test/javascript.spec.ts similarity index 94% rename from packages/string-templates/test/javascript.spec.js rename to packages/string-templates/test/javascript.spec.ts index 0e9f196da6..cb2f765007 100644 --- a/packages/string-templates/test/javascript.spec.js +++ b/packages/string-templates/test/javascript.spec.ts @@ -1,13 +1,9 @@ -const vm = require("vm") +import vm from "vm" -const { - processStringSync, - encodeJSBinding, - setJSRunner, -} = require("../src/index.js") -const { UUID_REGEX } = require("./constants") +import { processStringSync, encodeJSBinding, setJSRunner } from "../src/index" +import { UUID_REGEX } from "./constants" -const processJS = (js, context) => { +const processJS = (js: string, context?: object): any => { return processStringSync(encodeJSBinding(js), context) } diff --git a/packages/string-templates/test/manifest.spec.js b/packages/string-templates/test/manifest.spec.ts similarity index 80% rename from packages/string-templates/test/manifest.spec.js rename to packages/string-templates/test/manifest.spec.ts index 81183f13c9..d8fee0fb1a 100644 --- a/packages/string-templates/test/manifest.spec.js +++ b/packages/string-templates/test/manifest.spec.ts @@ -1,4 +1,4 @@ -const vm = require("vm") +import vm from "vm" jest.mock("@budibase/handlebars-helpers/lib/math", () => { const actual = jest.requireActual("@budibase/handlebars-helpers/lib/math") @@ -17,14 +17,14 @@ jest.mock("@budibase/handlebars-helpers/lib/uuid", () => { } }) -const { processString, setJSRunner } = require("../src/index.js") +import { processString, setJSRunner } from "../src/index" -const tk = require("timekeeper") -const { getParsedManifest, runJsHelpersTests } = require("./utils") +import tk from "timekeeper" +import { getParsedManifest, runJsHelpersTests } from "./utils" tk.freeze("2021-01-21T12:00:00") -function escapeRegExp(string) { +function escapeRegExp(string: string) { return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") // $& means the whole matched string } @@ -40,9 +40,9 @@ describe("manifest", () => { describe("examples are valid", () => { describe.each(Object.keys(manifest))("%s", collection => { it.each(manifest[collection])("%s", async (_, { hbs, js }) => { - const context = { - double: i => i * 2, - isString: x => typeof x === "string", + const context: any = { + double: (i: number) => i * 2, + isString: (x: any) => typeof x === "string", } const arrays = hbs.match(/\[[^/\]]+\]/) diff --git a/packages/string-templates/test/renderApp.spec.js b/packages/string-templates/test/renderApp.spec.ts similarity index 94% rename from packages/string-templates/test/renderApp.spec.js rename to packages/string-templates/test/renderApp.spec.ts index 582e70701f..7570d1a8ec 100644 --- a/packages/string-templates/test/renderApp.spec.js +++ b/packages/string-templates/test/renderApp.spec.ts @@ -1,4 +1,4 @@ -const { processString } = require("../src/index.js") +import { processString } from "../src/index" describe("specific test case for whether or not full app template can still be rendered", () => { it("should be able to render the app template", async () => { diff --git a/packages/string-templates/test/utils.js b/packages/string-templates/test/utils.ts similarity index 63% rename from packages/string-templates/test/utils.js rename to packages/string-templates/test/utils.ts index 927a6e3aeb..c683acf5b3 100644 --- a/packages/string-templates/test/utils.js +++ b/packages/string-templates/test/utils.ts @@ -1,13 +1,9 @@ -const { getManifest } = require("../src") -const { getJsHelperList } = require("../src/helpers") +import { getManifest } from "../src" +import { getJsHelperList } from "../src/helpers" -const { - convertToJS, - processStringSync, - encodeJSBinding, -} = require("../src/index.js") +import { convertToJS, processStringSync, encodeJSBinding } from "../src/index" -function tryParseJson(str) { +function tryParseJson(str: string) { if (typeof str !== "string") { return } @@ -19,23 +15,35 @@ function tryParseJson(str) { } } -const getParsedManifest = () => { - const manifest = getManifest() +type ExampleType = [ + string, + { + hbs: string + js: string + requiresHbsBody: boolean + } +] + +export const getParsedManifest = () => { + const manifest: any = getManifest() const collections = Object.keys(manifest) + const examples = collections.reduce((acc, collection) => { - const functions = Object.entries(manifest[collection]) - .filter(([_, details]) => details.example) - .map(([name, details]) => { + const functions = Object.entries<{ + example: string + requiresBlock: boolean + }>(manifest[collection]) + .filter( + ([_, details]) => + details.example?.split("->").map(x => x.trim()).length > 1 + ) + .map(([name, details]): ExampleType => { const example = details.example let [hbs, js] = example.split("->").map(x => x.trim()) - if (!js) { - // The function has no return value - return - } // Trim 's js = js.replace(/^'|'$/g, "") - let parsedExpected + let parsedExpected: string if ((parsedExpected = tryParseJson(js))) { if (Array.isArray(parsedExpected)) { if (typeof parsedExpected[0] === "object") { @@ -48,36 +56,40 @@ const getParsedManifest = () => { const requiresHbsBody = details.requiresBlock return [name, { hbs, js, requiresHbsBody }] }) - .filter(x => !!x) - if (Object.keys(functions).length) { + if (functions.length) { acc[collection] = functions } return acc - }, {}) + }, {} as Record) return examples } -module.exports.getParsedManifest = getParsedManifest -module.exports.runJsHelpersTests = ({ funcWrap, testsToSkip } = {}) => { - funcWrap = funcWrap || (delegate => delegate()) +export const runJsHelpersTests = ({ + funcWrap, + testsToSkip, +}: { + funcWrap?: any + testsToSkip?: any +} = {}) => { + funcWrap = funcWrap || ((delegate: () => any) => delegate()) const manifest = getParsedManifest() - const processJS = (js, context) => { + const processJS = (js: string, context: object | undefined) => { return funcWrap(() => processStringSync(encodeJSBinding(js), context)) } - function escapeRegExp(string) { + function escapeRegExp(string: string) { return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") // $& means the whole matched string } describe("can be parsed and run as js", () => { - const jsHelpers = getJsHelperList() + const jsHelpers = getJsHelperList()! const jsExamples = Object.keys(manifest).reduce((acc, v) => { acc[v] = manifest[v].filter(([key]) => jsHelpers[key]) return acc - }, {}) + }, {} as typeof manifest) describe.each(Object.keys(jsExamples))("%s", collection => { const examplesToRun = jsExamples[collection] @@ -86,9 +98,9 @@ module.exports.runJsHelpersTests = ({ funcWrap, testsToSkip } = {}) => { examplesToRun.length && it.each(examplesToRun)("%s", async (_, { hbs, js }) => { - const context = { - double: i => i * 2, - isString: x => typeof x === "string", + const context: any = { + double: (i: number) => i * 2, + isString: (x: any) => typeof x === "string", } const arrays = hbs.match(/\[[^/\]]+\]/) diff --git a/packages/string-templates/test/vm.spec.js b/packages/string-templates/test/vm.spec.ts similarity index 85% rename from packages/string-templates/test/vm.spec.js rename to packages/string-templates/test/vm.spec.ts index 7bf1612386..0581736af2 100644 --- a/packages/string-templates/test/vm.spec.js +++ b/packages/string-templates/test/vm.spec.ts @@ -5,8 +5,10 @@ jest.mock("../src/utilities", () => { isBackendService: jest.fn().mockReturnValue(true), } }) -const { defaultJSSetup, processStringSync, encodeJSBinding } = require("../src") -const { isBackendService } = require("../src/utilities") + +import { defaultJSSetup, processStringSync, encodeJSBinding } from "../src" +import { isBackendService } from "../src/utilities" + const mockedBackendService = jest.mocked(isBackendService) const binding = encodeJSBinding("return 1") diff --git a/packages/string-templates/tsconfig.json b/packages/string-templates/tsconfig.json index a7c7a0df67..7fc13ace8e 100644 --- a/packages/string-templates/tsconfig.json +++ b/packages/string-templates/tsconfig.json @@ -1,11 +1,15 @@ { "include": ["src/**/*"], "compilerOptions": { - "allowJs": true, "declaration": true, - "emitDeclarationOnly": true, + "target": "es6", + "moduleResolution": "node", + "noImplicitAny": true, + "incremental": true, + "lib": ["dom"], "outDir": "dist", "esModuleInterop": true, - "types": ["node", "jest"] + "types": ["node", "jest"], + "resolveJsonModule": true } } diff --git a/packages/worker/Dockerfile b/packages/worker/Dockerfile index 77869c48f1..28796ea666 100644 --- a/packages/worker/Dockerfile +++ b/packages/worker/Dockerfile @@ -16,17 +16,11 @@ COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies. RUN chmod +x ./scripts/removeWorkspaceDependencies.sh -WORKDIR /string-templates -COPY packages/string-templates/package.json package.json -RUN ../scripts/removeWorkspaceDependencies.sh package.json -RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true --network-timeout 1000000 -COPY packages/string-templates . WORKDIR /app COPY packages/worker/package.json . COPY packages/worker/dist/yarn.lock . -RUN cd ../string-templates && yarn link && cd - && yarn link @budibase/string-templates RUN ../scripts/removeWorkspaceDependencies.sh package.json diff --git a/packages/worker/jest.config.ts b/packages/worker/jest.config.ts index 0a640766f4..1eb17f100f 100644 --- a/packages/worker/jest.config.ts +++ b/packages/worker/jest.config.ts @@ -15,6 +15,7 @@ const config: Config.InitialOptions = { "@budibase/backend-core": "/../backend-core/src", "@budibase/types": "/../types/src", "@budibase/shared-core": ["/../shared-core/src"], + "@budibase/string-templates": ["/../string-templates/src"], }, } diff --git a/packages/worker/src/api/controllers/global/self.ts b/packages/worker/src/api/controllers/global/self.ts index d2741116e5..d762f5168a 100644 --- a/packages/worker/src/api/controllers/global/self.ts +++ b/packages/worker/src/api/controllers/global/self.ts @@ -114,9 +114,16 @@ export const syncAppFavourites = async (processedAppIds: string[]) => { if (processedAppIds.length === 0) { return [] } - const apps = await fetchAppsByIds(processedAppIds) + + const tenantId = tenancy.getTenantId() + const appPrefix = + tenantId === tenancy.DEFAULT_TENANT_ID + ? dbCore.APP_DEV_PREFIX + : `${dbCore.APP_DEV_PREFIX}${tenantId}_` + + const apps = await fetchAppsByIds(processedAppIds, appPrefix) return apps?.reduce((acc: string[], app) => { - const id = app.appId.replace(dbCore.APP_DEV_PREFIX, "") + const id = app.appId.replace(appPrefix, "") if (processedAppIds.includes(id)) { acc.push(id) } @@ -124,9 +131,14 @@ export const syncAppFavourites = async (processedAppIds: string[]) => { }, []) } -export const fetchAppsByIds = async (processedAppIds: string[]) => { +export const fetchAppsByIds = async ( + processedAppIds: string[], + appPrefix: string +) => { return await dbCore.getAppsByIDs( - processedAppIds.map(appId => `${dbCore.APP_DEV_PREFIX}${appId}`) + processedAppIds.map(appId => { + return `${appPrefix}${appId}` + }) ) } diff --git a/packages/worker/tsconfig.build.json b/packages/worker/tsconfig.build.json index bc477abe4d..f38f31fe67 100644 --- a/packages/worker/tsconfig.build.json +++ b/packages/worker/tsconfig.build.json @@ -16,7 +16,8 @@ "@budibase/backend-core": ["../backend-core/src"], "@budibase/backend-core/*": ["../backend-core/*"], "@budibase/shared-core": ["../shared-core/src"], - "@budibase/pro": ["../pro/src"] + "@budibase/pro": ["../pro/src"], + "@budibase/string-templates": ["../string-templates/src"] } }, "include": ["src/**/*"], diff --git a/yarn.lock b/yarn.lock index 53bf1ac017..979f5a94e0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1988,7 +1988,7 @@ resolved "https://registry.yarnpkg.com/@babel/regjsgen/-/regjsgen-0.8.0.tgz#f0ba69b075e1f05fb2825b7fad991e7adbb18310" integrity sha512-x/rqGMdzj+fWZvCOYForTghzbtqPDZ5gPwaoNGHdgDfF2QA/XZbCBp4Moo5scrkAMPhB7z26XM/AaHuIJdgauA== -"@babel/runtime@^7.10.5", "@babel/runtime@^7.13.10", "@babel/runtime@^7.21.0": +"@babel/runtime@^7.10.5", "@babel/runtime@^7.13.10": version "7.23.9" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.23.9.tgz#47791a15e4603bb5f905bc0753801cf21d6345f7" integrity sha512-0CX6F+BI2s9dkUqr08KFrAIZgNFj75rdBU/DjCyYLIaV/quFjkk6T+EJ2LkZHyZTbEV4L5p97mNkUsHl2wLFAw== @@ -8467,21 +8467,6 @@ concat-with-sourcemaps@^1.1.0: dependencies: source-map "^0.6.1" -concurrently@^8.2.2: - version "8.2.2" - resolved "https://registry.yarnpkg.com/concurrently/-/concurrently-8.2.2.tgz#353141985c198cfa5e4a3ef90082c336b5851784" - integrity sha512-1dP4gpXFhei8IOtlXRE/T/4H88ElHgTiUzh71YUmtjTEHMSRS2Z/fgOxHSxxusGHogsRfxNq1vyAwxSC+EVyDg== - dependencies: - chalk "^4.1.2" - date-fns "^2.30.0" - lodash "^4.17.21" - rxjs "^7.8.1" - shell-quote "^1.8.1" - spawn-command "0.0.2" - supports-color "^8.1.1" - tree-kill "^1.2.2" - yargs "^17.7.2" - condense-newlines@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/condense-newlines/-/condense-newlines-0.2.1.tgz#3de985553139475d32502c83b02f60684d24c55f" @@ -9049,13 +9034,6 @@ data-urls@^4.0.0: whatwg-mimetype "^3.0.0" whatwg-url "^12.0.0" -date-fns@^2.30.0: - version "2.30.0" - resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.30.0.tgz#f367e644839ff57894ec6ac480de40cae4b0f4d0" - integrity sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw== - dependencies: - "@babel/runtime" "^7.21.0" - dateformat@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-3.0.3.tgz#a6e37499a4d9a9cf85ef5872044d62901c9889ae" @@ -19400,13 +19378,6 @@ rxjs@^7.5.5: dependencies: tslib "^2.1.0" -rxjs@^7.8.1: - version "7.8.1" - resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-7.8.1.tgz#6f6f3d99ea8044291efd92e7c7fcf562c4057543" - integrity sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg== - dependencies: - tslib "^2.1.0" - safe-array-concat@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/safe-array-concat/-/safe-array-concat-1.0.1.tgz#91686a63ce3adbea14d61b14c99572a8ff84754c" @@ -19682,11 +19653,6 @@ shell-exec@1.0.2: resolved "https://registry.yarnpkg.com/shell-exec/-/shell-exec-1.0.2.tgz#2e9361b0fde1d73f476c4b6671fa17785f696756" integrity sha512-jyVd+kU2X+mWKMmGhx4fpWbPsjvD53k9ivqetutVW/BQ+WIZoDoP4d8vUMGezV6saZsiNoW2f9GIhg9Dondohg== -shell-quote@^1.8.1: - version "1.8.1" - resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.8.1.tgz#6dbf4db75515ad5bac63b4f1894c3a154c766680" - integrity sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA== - shortid@2.2.15: version "2.2.15" resolved "https://registry.yarnpkg.com/shortid/-/shortid-2.2.15.tgz#2b902eaa93a69b11120373cd42a1f1fe4437c122" @@ -20005,11 +19971,6 @@ sparse-bitfield@^3.0.3: dependencies: memory-pager "^1.0.2" -spawn-command@0.0.2: - version "0.0.2" - resolved "https://registry.yarnpkg.com/spawn-command/-/spawn-command-0.0.2.tgz#9544e1a43ca045f8531aac1a48cb29bdae62338e" - integrity sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ== - spdx-correct@^3.0.0: version "3.1.1" resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-3.1.1.tgz#dece81ac9c1e6713e5f7d1b6f17d468fa53d89a9" @@ -20582,7 +20543,7 @@ supports-color@^7.0.0, supports-color@^7.1.0: dependencies: has-flag "^4.0.0" -supports-color@^8.0.0, supports-color@^8.1.1: +supports-color@^8.0.0: version "8.1.1" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c" integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== @@ -21209,11 +21170,6 @@ tr46@~0.0.3: resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== -tree-kill@^1.2.2: - version "1.2.2" - resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.2.2.tgz#4ca09a9092c88b73a7cdc5e8a01b507b0790a0cc" - integrity sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A== - trim-newlines@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-3.0.1.tgz#260a5d962d8b752425b32f3a7db0dcacd176c144"