diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 2e7851b338..c8bdfe9655 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -165,6 +165,7 @@ jobs: oracle, sqs, elasticsearch, + dynamodb, none, ] steps: @@ -205,6 +206,8 @@ jobs: docker pull postgres:9.5.25 elif [ "${{ matrix.datasource }}" == "elasticsearch" ]; then docker pull elasticsearch@${{ steps.dotenv.outputs.ELASTICSEARCH_SHA }} + elif [ "${{ matrix.datasource }}" == "dynamodb" ]; then + docker pull amazon/dynamodb-local@${{ steps.dotenv.outputs.DYNAMODB_SHA }} fi docker pull minio/minio & docker pull redis & diff --git a/globalSetup.ts b/globalSetup.ts index 0b0e276b49..7396540936 100644 --- a/globalSetup.ts +++ b/globalSetup.ts @@ -88,6 +88,16 @@ export default async function setup() { content: ` [log] level = warn + + [httpd] + socket_options = [{nodelay, true}] + + [couchdb] + single_node = true + + [cluster] + n = 1 + q = 1 `, target: "/opt/couchdb/etc/local.d/test-couchdb.ini", }, diff --git a/lerna.json b/lerna.json index d20097b6e5..10e7b8cdee 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.4.22", + "version": "3.4.24", "npmClient": "yarn", "concurrency": 20, "command": { diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index 842d3243bc..dc6890e655 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -3,7 +3,6 @@ import { newid } from "../utils" import { Queue, QueueOptions, JobOptions } from "./queue" import { helpers } from "@budibase/shared-core" import { Job, JobId, JobInformation } from "bull" -import { cloneDeep } from "lodash" function jobToJobInformation(job: Job): JobInformation { let cron = "" @@ -88,9 +87,7 @@ export class InMemoryQueue implements Partial> { */ async process(concurrencyOrFunc: number | any, func?: any) { func = typeof concurrencyOrFunc === "number" ? func : concurrencyOrFunc - this._emitter.on("message", async msg => { - const message = cloneDeep(msg) - + this._emitter.on("message", async message => { // For the purpose of testing, don't trigger cron jobs immediately. // Require the test to trigger them manually with timestamps. if (!message.manualTrigger && message.opts?.repeat != null) { @@ -165,6 +162,9 @@ export class InMemoryQueue implements Partial> { opts, } this._messages.push(message) + if (this._messages.length > 1000) { + this._messages.shift() + } this._addCount++ this._emitter.emit("message", message) } diff --git a/packages/builder/src/analytics/PosthogClient.js b/packages/builder/src/analytics/PosthogClient.ts similarity index 63% rename from packages/builder/src/analytics/PosthogClient.js rename to packages/builder/src/analytics/PosthogClient.ts index f541b69b13..fe41989a66 100644 --- a/packages/builder/src/analytics/PosthogClient.js +++ b/packages/builder/src/analytics/PosthogClient.ts @@ -1,9 +1,12 @@ import posthog from "posthog-js" -import { Events } from "./constants" export default class PosthogClient { - constructor(token) { + token: string + initialised: boolean + + constructor(token: string) { this.token = token + this.initialised = false } init() { @@ -12,6 +15,8 @@ export default class PosthogClient { posthog.init(this.token, { autocapture: false, capture_pageview: false, + // disable by default + disable_session_recording: true, }) posthog.set_config({ persistence: "cookie" }) @@ -22,7 +27,7 @@ export default class PosthogClient { * Set the posthog context to the current user * @param {String} id - unique user id */ - identify(id) { + identify(id: string) { if (!this.initialised) return posthog.identify(id) @@ -32,7 +37,7 @@ export default class PosthogClient { * Update user metadata associated with current user in posthog * @param {Object} meta - user fields */ - updateUser(meta) { + updateUser(meta: Record) { if (!this.initialised) return posthog.people.set(meta) @@ -43,28 +48,22 @@ export default class PosthogClient { * @param {String} event - event identifier * @param {Object} props - properties for the event */ - captureEvent(eventName, props) { - if (!this.initialised) return - - props.sourceApp = "builder" - posthog.capture(eventName, props) - } - - /** - * Submit NPS feedback to posthog. - * @param {Object} values - NPS Values - */ - npsFeedback(values) { - if (!this.initialised) return - - localStorage.setItem(Events.NPS.SUBMITTED, Date.now()) - - const prefixedFeedback = {} - for (let key in values) { - prefixedFeedback[`feedback_${key}`] = values[key] + captureEvent(event: string, props: Record) { + if (!this.initialised) { + return } - posthog.capture(Events.NPS.SUBMITTED, prefixedFeedback) + props.sourceApp = "builder" + posthog.capture(event, props) + } + + enableSessionRecording() { + if (!this.initialised) { + return + } + posthog.set_config({ + disable_session_recording: false, + }) } /** diff --git a/packages/builder/src/analytics/index.js b/packages/builder/src/analytics/index.js index aa83f3c7ab..12bd548e9b 100644 --- a/packages/builder/src/analytics/index.js +++ b/packages/builder/src/analytics/index.js @@ -31,6 +31,10 @@ class AnalyticsHub { posthog.captureEvent(eventName, props) } + enableSessionRecording() { + posthog.enableSessionRecording() + } + async logout() { posthog.logout() } diff --git a/packages/builder/src/components/common/bindings/BindingSidePanel.svelte b/packages/builder/src/components/common/bindings/BindingSidePanel.svelte index 251bf19b49..513be45692 100644 --- a/packages/builder/src/components/common/bindings/BindingSidePanel.svelte +++ b/packages/builder/src/components/common/bindings/BindingSidePanel.svelte @@ -145,9 +145,11 @@ return } popoverAnchor = target + + const doc = new DOMParser().parseFromString(helper.description, "text/html") hoverTarget = { type: "helper", - description: helper.description, + description: doc.body.textContent || "", code: getHelperExample(helper, mode === BindingMode.JavaScript), } popover.show() @@ -241,20 +243,19 @@ > {#if hoverTarget.description}
- - {@html hoverTarget.description} + {hoverTarget.description}
{/if} {#if hoverTarget.code} - {#if mode === BindingMode.JavaScript} + {#if mode === BindingMode.Text || (mode === BindingMode.JavaScript && hoverTarget.type === "binding")} + + 
{@html hoverTarget.code}
+ {:else} - {:else if mode === BindingMode.Text} - - 
{@html hoverTarget.code}
{/if} {/if} diff --git a/packages/builder/src/stores/portal/admin.ts b/packages/builder/src/stores/portal/admin.ts index 90e3a5cdc9..6ac8b00b73 100644 --- a/packages/builder/src/stores/portal/admin.ts +++ b/packages/builder/src/stores/portal/admin.ts @@ -8,6 +8,7 @@ import { SystemStatusResponse, } from "@budibase/types" import { BudiStore } from "../BudiStore" +import Analytics from "../../analytics" interface AdminState extends GetEnvironmentResponse { loaded: boolean @@ -33,6 +34,8 @@ export class AdminStore extends BudiStore { await this.getEnvironment() // enable system status checks in the cloud if (get(this.store).cloud) { + // in cloud allow this + Analytics.enableSessionRecording() await this.getSystemStatus() this.checkStatus() } diff --git a/packages/client/src/components/app/forms/RelationshipField.svelte b/packages/client/src/components/app/forms/RelationshipField.svelte index 831ada9a2d..d70cb7b194 100644 --- a/packages/client/src/components/app/forms/RelationshipField.svelte +++ b/packages/client/src/components/app/forms/RelationshipField.svelte @@ -108,17 +108,14 @@ filter: SearchFilter[], linkedTableId?: string ) => { - if (!linkedTableId) { - return undefined - } const datasource = - datasourceType === "table" + dsType === "table" ? { - type: datasourceType, - tableId: fieldSchema?.tableId!, + type: dsType, + tableId: linkedTableId!, } : { - type: datasourceType, + type: dsType, tableId: InternalTable.USER_METADATA, } return fetchData({ diff --git a/packages/server/datasource-sha.env b/packages/server/datasource-sha.env index 69750793ce..13413dcb59 100644 --- a/packages/server/datasource-sha.env +++ b/packages/server/datasource-sha.env @@ -3,4 +3,5 @@ MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588eb POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d MARIADB_SHA=sha256:e59ba8783bf7bc02a4779f103bb0d8751ac0e10f9471089709608377eded7aa8 -ELASTICSEARCH_SHA=sha256:9a6443f55243f6acbfeb4a112d15eb3b9aac74bf25e0e39fa19b3ddd3a6879d0 \ No newline at end of file +ELASTICSEARCH_SHA=sha256:9a6443f55243f6acbfeb4a112d15eb3b9aac74bf25e0e39fa19b3ddd3a6879d0 +DYNAMODB_SHA=sha256:cf8cebd061f988628c02daff10fdb950a54478feff9c52f6ddf84710fe3c3906 \ No newline at end of file diff --git a/packages/server/src/api/routes/tests/automation.spec.ts b/packages/server/src/api/routes/tests/automation.spec.ts index 1591412735..c9bc940ff3 100644 --- a/packages/server/src/api/routes/tests/automation.spec.ts +++ b/packages/server/src/api/routes/tests/automation.spec.ts @@ -290,8 +290,7 @@ describe("/automations", () => { await setup.delay(500) let elements = await getAllTableRows(config) // don't test it unless there are values to test - if (elements.length > 1) { - expect(elements.length).toBeGreaterThanOrEqual(MAX_RETRIES) + if (elements.length >= 1) { expect(elements[0].name).toEqual("Test") expect(elements[0].description).toEqual("TEST") return diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 3a732cc662..1ee0e168a1 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -166,18 +166,6 @@ if (descriptions.length) { ) } - const resetRowUsage = async () => { - await config.doInContext( - undefined, - async () => - await quotas.setUsage( - 0, - StaticQuotaName.ROWS, - QuotaUsageType.STATIC - ) - ) - } - const getRowUsage = async () => { const { total } = await config.doInContext(undefined, () => quotas.getCurrentUsageValues( @@ -188,19 +176,27 @@ if (descriptions.length) { return total } - const assertRowUsage = async (expected: number) => { - const usage = await getRowUsage() + async function expectRowUsage(expected: number, f: () => Promise) { + const before = await getRowUsage() + await f() + const after = await getRowUsage() + const usage = after - before // Because our quota tracking is not perfect, we allow a 10% margin of - // error. This is to account for the fact that parallel writes can result - // in some quota updates getting lost. We don't have any need to solve this - // right now, so we just allow for some error. + // error. This is to account for the fact that parallel writes can + // result in some quota updates getting lost. We don't have any need + // to solve this right now, so we just allow for some error. if (expected === 0) { expect(usage).toEqual(0) return } - expect(usage).toBeGreaterThan(expected * 0.9) - expect(usage).toBeLessThan(expected * 1.1) + if (usage < 0) { + expect(usage).toBeGreaterThan(expected * 1.1) + expect(usage).toBeLessThan(expected * 0.9) + } else { + expect(usage).toBeGreaterThan(expected * 0.9) + expect(usage).toBeLessThan(expected * 1.1) + } } const defaultRowFields = isInternal @@ -215,91 +211,86 @@ if (descriptions.length) { table = await config.api.table.save(defaultTable()) }) - beforeEach(async () => { - await resetRowUsage() - }) - describe("create", () => { it("creates a new row successfully", async () => { - const rowUsage = await getRowUsage() - const row = await config.api.row.save(table._id!, { - name: "Test Contact", + await expectRowUsage(isInternal ? 1 : 0, async () => { + const row = await config.api.row.save(table._id!, { + name: "Test Contact", + }) + expect(row.name).toEqual("Test Contact") + expect(row._rev).toBeDefined() }) - expect(row.name).toEqual("Test Contact") - expect(row._rev).toBeDefined() - await assertRowUsage(isInternal ? rowUsage + 1 : rowUsage) }) it("fails to create a row for a table that does not exist", async () => { - const rowUsage = await getRowUsage() - await config.api.row.save("1234567", {}, { status: 404 }) - await assertRowUsage(rowUsage) + await expectRowUsage(0, async () => { + await config.api.row.save("1234567", {}, { status: 404 }) + }) }) it("fails to create a row if required fields are missing", async () => { - const rowUsage = await getRowUsage() - const table = await config.api.table.save( - saveTableRequest({ - schema: { - required: { - type: FieldType.STRING, - name: "required", - constraints: { - type: "string", - presence: true, - }, - }, - }, - }) - ) - await config.api.row.save( - table._id!, - {}, - { - status: 500, - body: { - validationErrors: { - required: ["can't be blank"], - }, - }, - } - ) - await assertRowUsage(rowUsage) - }) - - isInternal && - it("increment row autoId per create row request", async () => { - const rowUsage = await getRowUsage() - - const newTable = await config.api.table.save( + await expectRowUsage(0, async () => { + const table = await config.api.table.save( saveTableRequest({ schema: { - "Row ID": { - name: "Row ID", - type: FieldType.NUMBER, - subtype: AutoFieldSubType.AUTO_ID, - icon: "ri-magic-line", - autocolumn: true, + required: { + type: FieldType.STRING, + name: "required", constraints: { - type: "number", + type: "string", presence: true, - numericality: { - greaterThanOrEqualTo: "", - lessThanOrEqualTo: "", - }, }, }, }, }) ) + await config.api.row.save( + table._id!, + {}, + { + status: 500, + body: { + validationErrors: { + required: ["can't be blank"], + }, + }, + } + ) + }) + }) - let previousId = 0 - for (let i = 0; i < 10; i++) { - const row = await config.api.row.save(newTable._id!, {}) - expect(row["Row ID"]).toBeGreaterThan(previousId) - previousId = row["Row ID"] - } - await assertRowUsage(isInternal ? rowUsage + 10 : rowUsage) + isInternal && + it("increment row autoId per create row request", async () => { + await expectRowUsage(isInternal ? 10 : 0, async () => { + const newTable = await config.api.table.save( + saveTableRequest({ + schema: { + "Row ID": { + name: "Row ID", + type: FieldType.NUMBER, + subtype: AutoFieldSubType.AUTO_ID, + icon: "ri-magic-line", + autocolumn: true, + constraints: { + type: "number", + presence: true, + numericality: { + greaterThanOrEqualTo: "", + lessThanOrEqualTo: "", + }, + }, + }, + }, + }) + ) + + let previousId = 0 + for (let i = 0; i < 10; i++) { + const row = await config.api.row.save(newTable._id!, {}) + expect(row["Row ID"]).toBeGreaterThan(previousId) + previousId = row["Row ID"] + } + }) }) isInternal && @@ -985,16 +976,16 @@ if (descriptions.length) { describe("update", () => { it("updates an existing row successfully", async () => { const existing = await config.api.row.save(table._id!, {}) - const rowUsage = await getRowUsage() - const res = await config.api.row.save(table._id!, { - _id: existing._id, - _rev: existing._rev, - name: "Updated Name", + await expectRowUsage(0, async () => { + const res = await config.api.row.save(table._id!, { + _id: existing._id, + _rev: existing._rev, + name: "Updated Name", + }) + + expect(res.name).toEqual("Updated Name") }) - - expect(res.name).toEqual("Updated Name") - await assertRowUsage(rowUsage) }) !isInternal && @@ -1177,23 +1168,22 @@ if (descriptions.length) { it("should update only the fields that are supplied", async () => { const existing = await config.api.row.save(table._id!, {}) - const rowUsage = await getRowUsage() + await expectRowUsage(0, async () => { + const row = await config.api.row.patch(table._id!, { + _id: existing._id!, + _rev: existing._rev!, + tableId: table._id!, + name: "Updated Name", + }) - const row = await config.api.row.patch(table._id!, { - _id: existing._id!, - _rev: existing._rev!, - tableId: table._id!, - name: "Updated Name", + expect(row.name).toEqual("Updated Name") + expect(row.description).toEqual(existing.description) + + const savedRow = await config.api.row.get(table._id!, row._id!) + + expect(savedRow.description).toEqual(existing.description) + expect(savedRow.name).toEqual("Updated Name") }) - - expect(row.name).toEqual("Updated Name") - expect(row.description).toEqual(existing.description) - - const savedRow = await config.api.row.get(table._id!, row._id!) - - expect(savedRow.description).toEqual(existing.description) - expect(savedRow.name).toEqual("Updated Name") - await assertRowUsage(rowUsage) }) it("should update only the fields that are supplied and emit the correct oldRow", async () => { @@ -1224,20 +1214,19 @@ if (descriptions.length) { it("should throw an error when given improper types", async () => { const existing = await config.api.row.save(table._id!, {}) - const rowUsage = await getRowUsage() - await config.api.row.patch( - table._id!, - { - _id: existing._id!, - _rev: existing._rev!, - tableId: table._id!, - name: 1, - }, - { status: 400 } - ) - - await assertRowUsage(rowUsage) + await expectRowUsage(0, async () => { + await config.api.row.patch( + table._id!, + { + _id: existing._id!, + _rev: existing._rev!, + tableId: table._id!, + name: 1, + }, + { status: 400 } + ) + }) }) it("should not overwrite links if those links are not set", async () => { @@ -1452,25 +1441,25 @@ if (descriptions.length) { it("should be able to delete a row", async () => { const createdRow = await config.api.row.save(table._id!, {}) - const rowUsage = await getRowUsage() - const res = await config.api.row.bulkDelete(table._id!, { - rows: [createdRow], + await expectRowUsage(isInternal ? -1 : 0, async () => { + const res = await config.api.row.bulkDelete(table._id!, { + rows: [createdRow], + }) + expect(res[0]._id).toEqual(createdRow._id) }) - expect(res[0]._id).toEqual(createdRow._id) - await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage) }) it("should be able to delete a row with ID only", async () => { const createdRow = await config.api.row.save(table._id!, {}) - const rowUsage = await getRowUsage() - const res = await config.api.row.bulkDelete(table._id!, { - rows: [createdRow._id!], + await expectRowUsage(isInternal ? -1 : 0, async () => { + const res = await config.api.row.bulkDelete(table._id!, { + rows: [createdRow._id!], + }) + expect(res[0]._id).toEqual(createdRow._id) + expect(res[0].tableId).toEqual(table._id!) }) - expect(res[0]._id).toEqual(createdRow._id) - expect(res[0].tableId).toEqual(table._id!) - await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage) }) it("should be able to bulk delete rows, including a row that doesn't exist", async () => { @@ -1560,31 +1549,29 @@ if (descriptions.length) { }) it("should return no errors on valid row", async () => { - const rowUsage = await getRowUsage() + await expectRowUsage(0, async () => { + const res = await config.api.row.validate(table._id!, { + name: "ivan", + }) - const res = await config.api.row.validate(table._id!, { - name: "ivan", + expect(res.valid).toBe(true) + expect(Object.keys(res.errors)).toEqual([]) }) - - expect(res.valid).toBe(true) - expect(Object.keys(res.errors)).toEqual([]) - await assertRowUsage(rowUsage) }) it("should errors on invalid row", async () => { - const rowUsage = await getRowUsage() + await expectRowUsage(0, async () => { + const res = await config.api.row.validate(table._id!, { name: 1 }) - const res = await config.api.row.validate(table._id!, { name: 1 }) - - if (isInternal) { - expect(res.valid).toBe(false) - expect(Object.keys(res.errors)).toEqual(["name"]) - } else { - // Validation for external is not implemented, so it will always return valid - expect(res.valid).toBe(true) - expect(Object.keys(res.errors)).toEqual([]) - } - await assertRowUsage(rowUsage) + if (isInternal) { + expect(res.valid).toBe(false) + expect(Object.keys(res.errors)).toEqual(["name"]) + } else { + // Validation for external is not implemented, so it will always return valid + expect(res.valid).toBe(true) + expect(Object.keys(res.errors)).toEqual([]) + } + }) }) }) @@ -1596,15 +1583,15 @@ if (descriptions.length) { it("should be able to delete a bulk set of rows", async () => { const row1 = await config.api.row.save(table._id!, {}) const row2 = await config.api.row.save(table._id!, {}) - const rowUsage = await getRowUsage() - const res = await config.api.row.bulkDelete(table._id!, { - rows: [row1, row2], + await expectRowUsage(isInternal ? -2 : 0, async () => { + const res = await config.api.row.bulkDelete(table._id!, { + rows: [row1, row2], + }) + + expect(res.length).toEqual(2) + await config.api.row.get(table._id!, row1._id!, { status: 404 }) }) - - expect(res.length).toEqual(2) - await config.api.row.get(table._id!, row1._id!, { status: 404 }) - await assertRowUsage(isInternal ? rowUsage - 2 : rowUsage) }) it("should be able to delete a variety of row set types", async () => { @@ -1613,41 +1600,42 @@ if (descriptions.length) { config.api.row.save(table._id!, {}), config.api.row.save(table._id!, {}), ]) - const rowUsage = await getRowUsage() - const res = await config.api.row.bulkDelete(table._id!, { - rows: [row1, row2._id!, { _id: row3._id }], + await expectRowUsage(isInternal ? -3 : 0, async () => { + const res = await config.api.row.bulkDelete(table._id!, { + rows: [row1, row2._id!, { _id: row3._id }], + }) + + expect(res.length).toEqual(3) + await config.api.row.get(table._id!, row1._id!, { status: 404 }) }) - - expect(res.length).toEqual(3) - await config.api.row.get(table._id!, row1._id!, { status: 404 }) - await assertRowUsage(isInternal ? rowUsage - 3 : rowUsage) }) it("should accept a valid row object and delete the row", async () => { const row1 = await config.api.row.save(table._id!, {}) - const rowUsage = await getRowUsage() - const res = await config.api.row.delete(table._id!, row1 as DeleteRow) + await expectRowUsage(isInternal ? -1 : 0, async () => { + const res = await config.api.row.delete( + table._id!, + row1 as DeleteRow + ) - expect(res.id).toEqual(row1._id) - await config.api.row.get(table._id!, row1._id!, { status: 404 }) - await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage) + expect(res.id).toEqual(row1._id) + await config.api.row.get(table._id!, row1._id!, { status: 404 }) + }) }) it.each([{ not: "valid" }, { rows: 123 }, "invalid"])( "should ignore malformed/invalid delete request: %s", async (request: any) => { - const rowUsage = await getRowUsage() - - await config.api.row.delete(table._id!, request, { - status: 400, - body: { - message: "Invalid delete rows request", - }, + await expectRowUsage(0, async () => { + await config.api.row.delete(table._id!, request, { + status: 400, + body: { + message: "Invalid delete rows request", + }, + }) }) - - await assertRowUsage(rowUsage) } ) }) @@ -1733,31 +1721,29 @@ if (descriptions.length) { }) ) - const rowUsage = await getRowUsage() + await expectRowUsage(isInternal ? 2 : 0, async () => { + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "Row 1", + description: "Row 1 description", + }, + { + name: "Row 2", + description: "Row 2 description", + }, + ], + }) - await config.api.row.bulkImport(table._id!, { - rows: [ - { - name: "Row 1", - description: "Row 1 description", - }, - { - name: "Row 2", - description: "Row 2 description", - }, - ], + const rows = await config.api.row.fetch(table._id!) + expect(rows.length).toEqual(2) + + rows.sort((a, b) => a.name.localeCompare(b.name)) + expect(rows[0].name).toEqual("Row 1") + expect(rows[0].description).toEqual("Row 1 description") + expect(rows[1].name).toEqual("Row 2") + expect(rows[1].description).toEqual("Row 2 description") }) - - const rows = await config.api.row.fetch(table._id!) - expect(rows.length).toEqual(2) - - rows.sort((a, b) => a.name.localeCompare(b.name)) - expect(rows[0].name).toEqual("Row 1") - expect(rows[0].description).toEqual("Row 1 description") - expect(rows[1].name).toEqual("Row 2") - expect(rows[1].description).toEqual("Row 2 description") - - await assertRowUsage(isInternal ? rowUsage + 2 : rowUsage) }) isInternal && @@ -1782,35 +1768,33 @@ if (descriptions.length) { description: "Existing description", }) - const rowUsage = await getRowUsage() + await expectRowUsage(2, async () => { + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "Row 1", + description: "Row 1 description", + }, + { ...existingRow, name: "Updated existing row" }, + { + name: "Row 2", + description: "Row 2 description", + }, + ], + identifierFields: ["_id"], + }) - await config.api.row.bulkImport(table._id!, { - rows: [ - { - name: "Row 1", - description: "Row 1 description", - }, - { ...existingRow, name: "Updated existing row" }, - { - name: "Row 2", - description: "Row 2 description", - }, - ], - identifierFields: ["_id"], + const rows = await config.api.row.fetch(table._id!) + expect(rows.length).toEqual(3) + + rows.sort((a, b) => a.name.localeCompare(b.name)) + expect(rows[0].name).toEqual("Row 1") + expect(rows[0].description).toEqual("Row 1 description") + expect(rows[1].name).toEqual("Row 2") + expect(rows[1].description).toEqual("Row 2 description") + expect(rows[2].name).toEqual("Updated existing row") + expect(rows[2].description).toEqual("Existing description") }) - - const rows = await config.api.row.fetch(table._id!) - expect(rows.length).toEqual(3) - - rows.sort((a, b) => a.name.localeCompare(b.name)) - expect(rows[0].name).toEqual("Row 1") - expect(rows[0].description).toEqual("Row 1 description") - expect(rows[1].name).toEqual("Row 2") - expect(rows[1].description).toEqual("Row 2 description") - expect(rows[2].name).toEqual("Updated existing row") - expect(rows[2].description).toEqual("Existing description") - - await assertRowUsage(rowUsage + 2) }) isInternal && @@ -1835,36 +1819,34 @@ if (descriptions.length) { description: "Existing description", }) - const rowUsage = await getRowUsage() + await expectRowUsage(3, async () => { + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "Row 1", + description: "Row 1 description", + }, + { ...existingRow, name: "Updated existing row" }, + { + name: "Row 2", + description: "Row 2 description", + }, + ], + }) - await config.api.row.bulkImport(table._id!, { - rows: [ - { - name: "Row 1", - description: "Row 1 description", - }, - { ...existingRow, name: "Updated existing row" }, - { - name: "Row 2", - description: "Row 2 description", - }, - ], + const rows = await config.api.row.fetch(table._id!) + expect(rows.length).toEqual(4) + + rows.sort((a, b) => a.name.localeCompare(b.name)) + expect(rows[0].name).toEqual("Existing row") + expect(rows[0].description).toEqual("Existing description") + expect(rows[1].name).toEqual("Row 1") + expect(rows[1].description).toEqual("Row 1 description") + expect(rows[2].name).toEqual("Row 2") + expect(rows[2].description).toEqual("Row 2 description") + expect(rows[3].name).toEqual("Updated existing row") + expect(rows[3].description).toEqual("Existing description") }) - - const rows = await config.api.row.fetch(table._id!) - expect(rows.length).toEqual(4) - - rows.sort((a, b) => a.name.localeCompare(b.name)) - expect(rows[0].name).toEqual("Existing row") - expect(rows[0].description).toEqual("Existing description") - expect(rows[1].name).toEqual("Row 1") - expect(rows[1].description).toEqual("Row 1 description") - expect(rows[2].name).toEqual("Row 2") - expect(rows[2].description).toEqual("Row 2 description") - expect(rows[3].name).toEqual("Updated existing row") - expect(rows[3].description).toEqual("Existing description") - - await assertRowUsage(rowUsage + 3) }) // Upserting isn't yet supported in MSSQL / Oracle, see: @@ -2187,29 +2169,29 @@ if (descriptions.length) { return { linkedTable, firstRow, secondRow } } ) - const rowUsage = await getRowUsage() - // test basic enrichment - const resBasic = await config.api.row.get( - linkedTable._id!, - secondRow._id! - ) - expect(resBasic.link.length).toBe(1) - expect(resBasic.link[0]).toEqual({ - _id: firstRow._id, - primaryDisplay: firstRow.name, + await expectRowUsage(0, async () => { + // test basic enrichment + const resBasic = await config.api.row.get( + linkedTable._id!, + secondRow._id! + ) + expect(resBasic.link.length).toBe(1) + expect(resBasic.link[0]).toEqual({ + _id: firstRow._id, + primaryDisplay: firstRow.name, + }) + + // test full enrichment + const resEnriched = await config.api.row.getEnriched( + linkedTable._id!, + secondRow._id! + ) + expect(resEnriched.link.length).toBe(1) + expect(resEnriched.link[0]._id).toBe(firstRow._id) + expect(resEnriched.link[0].name).toBe("Test Contact") + expect(resEnriched.link[0].description).toBe("original description") }) - - // test full enrichment - const resEnriched = await config.api.row.getEnriched( - linkedTable._id!, - secondRow._id! - ) - expect(resEnriched.link.length).toBe(1) - expect(resEnriched.link[0]._id).toBe(firstRow._id) - expect(resEnriched.link[0].name).toBe("Test Contact") - expect(resEnriched.link[0].description).toBe("original description") - await assertRowUsage(rowUsage) }) }) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 7eed1811d9..ad41aa618c 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -2826,34 +2826,44 @@ if (descriptions.length) { return total } - const assertRowUsage = async (expected: number) => { - const usage = await getRowUsage() + async function expectRowUsage( + expected: number, + f: () => Promise + ): Promise { + const before = await getRowUsage() + const result = await f() + const after = await getRowUsage() + const usage = after - before expect(usage).toBe(expected) + return result } it("should be able to delete a row", async () => { - const createdRow = await config.api.row.save(table._id!, {}) - const rowUsage = await getRowUsage() - await config.api.row.bulkDelete(view.id, { rows: [createdRow] }) - await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage) + const createdRow = await expectRowUsage(isInternal ? 1 : 0, () => + config.api.row.save(table._id!, {}) + ) + await expectRowUsage(isInternal ? -1 : 0, () => + config.api.row.bulkDelete(view.id, { rows: [createdRow] }) + ) await config.api.row.get(table._id!, createdRow._id!, { status: 404, }) }) it("should be able to delete multiple rows", async () => { - const rows = await Promise.all([ - config.api.row.save(table._id!, {}), - config.api.row.save(table._id!, {}), - config.api.row.save(table._id!, {}), - ]) - const rowUsage = await getRowUsage() - - await config.api.row.bulkDelete(view.id, { - rows: [rows[0], rows[2]], + const rows = await expectRowUsage(isInternal ? 3 : 0, async () => { + return [ + await config.api.row.save(table._id!, {}), + await config.api.row.save(table._id!, {}), + await config.api.row.save(table._id!, {}), + ] }) - await assertRowUsage(isInternal ? rowUsage - 2 : rowUsage) + await expectRowUsage(isInternal ? -2 : 0, async () => { + await config.api.row.bulkDelete(view.id, { + rows: [rows[0], rows[2]], + }) + }) await config.api.row.get(table._id!, rows[0]._id!, { status: 404, diff --git a/packages/server/src/automations/tests/steps/loop.spec.ts b/packages/server/src/automations/tests/steps/loop.spec.ts index 34fc175c71..78ef3a279f 100644 --- a/packages/server/src/automations/tests/steps/loop.spec.ts +++ b/packages/server/src/automations/tests/steps/loop.spec.ts @@ -195,7 +195,34 @@ describe("Attempt to run a basic loop automation", () => { .serverLog({ text: "{{steps.1.iterations}}" }) .test({ fields: {} }) + expect(results.steps[0].outputs.status).toBe( + AutomationStepStatus.MAX_ITERATIONS + ) expect(results.steps[0].outputs.iterations).toBe(2) + expect(results.steps[0].outputs.items).toHaveLength(2) + expect(results.steps[0].outputs.items[0].message).toEndWith("test") + expect(results.steps[0].outputs.items[1].message).toEndWith("test2") + }) + + it("should stop when a failure condition is hit", async () => { + const results = await createAutomationBuilder(config) + .onAppAction() + .loop({ + option: LoopStepType.ARRAY, + binding: ["test", "test2", "test3"], + failure: "test3", + }) + .serverLog({ text: "{{loop.currentItem}}" }) + .serverLog({ text: "{{steps.1.iterations}}" }) + .test({ fields: {} }) + + expect(results.steps[0].outputs.status).toBe( + AutomationStepStatus.FAILURE_CONDITION + ) + expect(results.steps[0].outputs.iterations).toBe(2) + expect(results.steps[0].outputs.items).toHaveLength(2) + expect(results.steps[0].outputs.items[0].message).toEndWith("test") + expect(results.steps[0].outputs.items[1].message).toEndWith("test2") }) it("should run an automation with loop and max iterations to ensure context correctness further down the tree", async () => { diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts index a04b960ca5..5287498e80 100644 --- a/packages/server/src/definitions/automations.ts +++ b/packages/server/src/definitions/automations.ts @@ -20,9 +20,12 @@ export interface TriggerOutput { export interface AutomationContext { trigger: AutomationTriggerResultOutputs - steps: [AutomationTriggerResultOutputs, ...AutomationStepResultOutputs[]] - stepsById: Record + steps: Record< + string, + AutomationStepResultOutputs | AutomationTriggerResultOutputs + > stepsByName: Record + stepsById: Record env?: Record user?: UserBindings settings?: { @@ -31,4 +34,6 @@ export interface AutomationContext { company?: string } loop?: { currentItem: any } + _stepIndex: number + _error: boolean } diff --git a/packages/server/src/events/AutomationEmitter.ts b/packages/server/src/events/AutomationEmitter.ts index a95acd0877..2f0e3d5abb 100644 --- a/packages/server/src/events/AutomationEmitter.ts +++ b/packages/server/src/events/AutomationEmitter.ts @@ -32,6 +32,8 @@ class AutomationEmitter implements ContextEmitter { if (chainAutomations === true) { return MAX_AUTOMATIONS_ALLOWED + } else if (env.isTest()) { + return 0 } else if (chainAutomations === undefined && env.SELF_HOSTED) { return MAX_AUTOMATIONS_ALLOWED } else { diff --git a/packages/server/src/integrations/dynamodb.ts b/packages/server/src/integrations/dynamodb.ts index 96941ebb0e..cfaa851cea 100644 --- a/packages/server/src/integrations/dynamodb.ts +++ b/packages/server/src/integrations/dynamodb.ts @@ -14,15 +14,14 @@ import { UpdateCommandInput, DeleteCommandInput, } from "@aws-sdk/lib-dynamodb" -import { DynamoDB } from "@aws-sdk/client-dynamodb" +import { DynamoDB, DynamoDBClientConfig } from "@aws-sdk/client-dynamodb" import { AWS_REGION } from "../constants" -interface DynamoDBConfig { +export interface DynamoDBConfig { region: string accessKeyId: string secretAccessKey: string endpoint?: string - currentClockSkew?: boolean } const SCHEMA: Integration = { @@ -138,22 +137,16 @@ const SCHEMA: Integration = { }, } -class DynamoDBIntegration implements IntegrationBase { - private config: DynamoDBConfig - private client +export class DynamoDBIntegration implements IntegrationBase { + private config: DynamoDBClientConfig + private client: DynamoDBDocument constructor(config: DynamoDBConfig) { - this.config = config - - // User is using a local dynamoDB endpoint, don't auth with remote - if (this.config?.endpoint?.includes("localhost")) { - // @ts-ignore - this.config = {} - } - this.config = { - ...this.config, - currentClockSkew: true, + credentials: { + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + }, region: config.region || AWS_REGION, endpoint: config.endpoint || undefined, } diff --git a/packages/server/src/integrations/tests/dynamodb.spec.ts b/packages/server/src/integrations/tests/dynamodb.spec.ts index 75fb84ae60..e6b1ed405c 100644 --- a/packages/server/src/integrations/tests/dynamodb.spec.ts +++ b/packages/server/src/integrations/tests/dynamodb.spec.ts @@ -1,167 +1,108 @@ -jest.mock("@aws-sdk/lib-dynamodb", () => ({ - DynamoDBDocument: { - from: jest.fn(() => ({ - update: jest.fn(), - put: jest.fn(), - query: jest.fn(() => ({ - Items: [], - })), - scan: jest.fn(() => ({ - Items: [], - })), - delete: jest.fn(), - get: jest.fn(), - })), - }, -})) -jest.mock("@aws-sdk/client-dynamodb") -import { default as DynamoDBIntegration } from "../dynamodb" +import { Datasource } from "@budibase/types" +import { DynamoDBConfig, DynamoDBIntegration } from "../dynamodb" +import { DatabaseName, datasourceDescribe } from "./utils" +import { + CreateTableCommandInput, + DynamoDB, + DynamoDBClientConfig, +} from "@aws-sdk/client-dynamodb" -class TestConfiguration { - integration: any +const describes = datasourceDescribe({ only: [DatabaseName.DYNAMODB] }) - constructor(config: any = {}) { - this.integration = new DynamoDBIntegration.integration(config) +async function createTable(client: DynamoDB, req: CreateTableCommandInput) { + try { + await client.deleteTable({ TableName: req.TableName }) + } catch (e: any) { + if (e.name !== "ResourceNotFoundException") { + throw e + } } + + return await client.createTable(req) } -describe("DynamoDB Integration", () => { - let config: any - let tableName = "Users" +if (describes.length > 0) { + describe.each(describes)("DynamoDB Integration", ({ dsProvider }) => { + let table = "Users" + let rawDatasource: Datasource + let dynamodb: DynamoDBIntegration - beforeEach(() => { - config = new TestConfiguration() - }) - - it("calls the create method with the correct params", async () => { - await config.integration.create({ - table: tableName, - json: { - Name: "John", - }, - }) - expect(config.integration.client.put).toHaveBeenCalledWith({ - TableName: tableName, - Name: "John", - }) - }) - - it("calls the read method with the correct params", async () => { - const indexName = "Test" - - const response = await config.integration.read({ - table: tableName, - index: indexName, - json: {}, - }) - expect(config.integration.client.query).toHaveBeenCalledWith({ - TableName: tableName, - IndexName: indexName, - }) - expect(response).toEqual([]) - }) - - it("calls the scan method with the correct params", async () => { - const indexName = "Test" - - const response = await config.integration.scan({ - table: tableName, - index: indexName, - json: {}, - }) - expect(config.integration.client.scan).toHaveBeenCalledWith({ - TableName: tableName, - IndexName: indexName, - }) - expect(response).toEqual([]) - }) - - it("calls the get method with the correct params", async () => { - await config.integration.get({ - table: tableName, - json: { - Id: 123, - }, - }) - - expect(config.integration.client.get).toHaveBeenCalledWith({ - TableName: tableName, - Id: 123, - }) - }) - - it("calls the update method with the correct params", async () => { - await config.integration.update({ - table: tableName, - json: { - Name: "John", - }, - }) - expect(config.integration.client.update).toHaveBeenCalledWith({ - TableName: tableName, - Name: "John", - }) - }) - - it("calls the delete method with the correct params", async () => { - await config.integration.delete({ - table: tableName, - json: { - Name: "John", - }, - }) - expect(config.integration.client.delete).toHaveBeenCalledWith({ - TableName: tableName, - Name: "John", - }) - }) - - it("configures the dynamoDB constructor based on an empty endpoint parameter", async () => { - const config = { - region: "us-east-1", - accessKeyId: "test", - secretAccessKey: "test", + function item(json: Record) { + return { table, json: { Item: json } } } - const integration: any = new DynamoDBIntegration.integration(config) - - expect(integration.config).toEqual({ - currentClockSkew: true, - ...config, - }) - }) - - it("configures the dynamoDB constructor based on a localhost endpoint parameter", async () => { - const config = { - region: "us-east-1", - accessKeyId: "test", - secretAccessKey: "test", - endpoint: "localhost:8080", + function key(json: Record) { + return { table, json: { Key: json } } } - const integration: any = new DynamoDBIntegration.integration(config) + beforeEach(async () => { + const ds = await dsProvider() + rawDatasource = ds.rawDatasource! + dynamodb = new DynamoDBIntegration( + rawDatasource.config! as DynamoDBConfig + ) - expect(integration.config).toEqual({ - region: "us-east-1", - currentClockSkew: true, - endpoint: "localhost:8080", + const config: DynamoDBClientConfig = { + credentials: { + accessKeyId: "test", + secretAccessKey: "test", + }, + region: "us-east-1", + endpoint: rawDatasource.config!.endpoint, + } + + const client = new DynamoDB(config) + await createTable(client, { + TableName: table, + KeySchema: [{ AttributeName: "Id", KeyType: "HASH" }], + AttributeDefinitions: [{ AttributeName: "Id", AttributeType: "N" }], + ProvisionedThroughput: { ReadCapacityUnits: 1, WriteCapacityUnits: 1 }, + }) + }) + + it("can create and read a record", async () => { + await dynamodb.create(item({ Id: 1, Name: "John" })) + + const resp = await dynamodb.get(key({ Id: 1 })) + expect(resp.Item).toEqual({ Id: 1, Name: "John" }) + }) + + it("can scan", async () => { + await dynamodb.create(item({ Id: 1, Name: "John" })) + await dynamodb.create(item({ Id: 2, Name: "Jane" })) + await dynamodb.create(item({ Id: 3, Name: "Jack" })) + + const resp = await dynamodb.scan({ table, json: {}, index: null }) + expect(resp).toEqual( + expect.arrayContaining([ + { Id: 1, Name: "John" }, + { Id: 2, Name: "Jane" }, + { Id: 3, Name: "Jack" }, + ]) + ) + }) + + it("can update", async () => { + await dynamodb.create(item({ Id: 1, Foo: "John" })) + await dynamodb.update({ + table, + json: { + Key: { Id: 1 }, + UpdateExpression: "SET Foo = :foo", + ExpressionAttributeValues: { ":foo": "Jane" }, + }, + }) + + const updatedRecord = await dynamodb.get(key({ Id: 1 })) + expect(updatedRecord.Item).toEqual({ Id: 1, Foo: "Jane" }) + }) + + it("can delete", async () => { + await dynamodb.create(item({ Id: 1, Name: "John" })) + await dynamodb.delete(key({ Id: 1 })) + + const deletedRecord = await dynamodb.get(key({ Id: 1 })) + expect(deletedRecord.Item).toBeUndefined() }) }) - - it("configures the dynamoDB constructor based on a remote endpoint parameter", async () => { - const config = { - region: "us-east-1", - accessKeyId: "test", - secretAccessKey: "test", - endpoint: "dynamodb.aws.foo.net", - } - - const integration = new DynamoDBIntegration.integration(config) - - // @ts-ignore - expect(integration.config).toEqual({ - currentClockSkew: true, - ...config, - }) - }) -}) +} diff --git a/packages/server/src/integrations/tests/utils/dynamodb.ts b/packages/server/src/integrations/tests/utils/dynamodb.ts new file mode 100644 index 0000000000..aae6358ece --- /dev/null +++ b/packages/server/src/integrations/tests/utils/dynamodb.ts @@ -0,0 +1,41 @@ +import { Datasource, SourceName } from "@budibase/types" +import { GenericContainer, Wait } from "testcontainers" +import { testContainerUtils } from "@budibase/backend-core/tests" +import { startContainer } from "." +import { DYNAMODB_IMAGE } from "./images" +import { DynamoDBConfig } from "../../dynamodb" + +let ports: Promise + +export async function getDatasource(): Promise { + if (!ports) { + ports = startContainer( + new GenericContainer(DYNAMODB_IMAGE) + .withExposedPorts(8000) + .withWaitStrategy( + Wait.forSuccessfulCommand( + // https://stackoverflow.com/a/77373799 + `if [ "$(curl -s -o /dev/null -I -w ''%{http_code}'' http://localhost:8000)" == "400" ]; then exit 0; else exit 1; fi` + ).withStartupTimeout(60000) + ) + ) + } + + const port = (await ports).find(x => x.container === 8000)?.host + if (!port) { + throw new Error("DynamoDB port not found") + } + + const config: DynamoDBConfig = { + accessKeyId: "test", + secretAccessKey: "test", + region: "us-east-1", + endpoint: `http://127.0.0.1:${port}`, + } + + return { + type: "datasource", + source: SourceName.DYNAMODB, + config, + } +} diff --git a/packages/server/src/integrations/tests/utils/images.ts b/packages/server/src/integrations/tests/utils/images.ts index c09b130ea5..68d219ac7d 100644 --- a/packages/server/src/integrations/tests/utils/images.ts +++ b/packages/server/src/integrations/tests/utils/images.ts @@ -13,3 +13,4 @@ export const POSTGRES_LEGACY_IMAGE = `postgres:9.5.25` export const MONGODB_IMAGE = `mongo@${process.env.MONGODB_SHA}` export const MARIADB_IMAGE = `mariadb@${process.env.MARIADB_SHA}` export const ELASTICSEARCH_IMAGE = `elasticsearch@${process.env.ELASTICSEARCH_SHA}` +export const DYNAMODB_IMAGE = `amazon/dynamodb-local@${process.env.DYNAMODB_SHA}` diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts index 08777cab89..138861a9e6 100644 --- a/packages/server/src/integrations/tests/utils/index.ts +++ b/packages/server/src/integrations/tests/utils/index.ts @@ -7,6 +7,7 @@ import * as mssql from "./mssql" import * as mariadb from "./mariadb" import * as oracle from "./oracle" import * as elasticsearch from "./elasticsearch" +import * as dynamodb from "./dynamodb" import { testContainerUtils } from "@budibase/backend-core/tests" import { Knex } from "knex" import TestConfiguration from "../../../tests/utilities/TestConfiguration" @@ -25,6 +26,7 @@ export enum DatabaseName { ORACLE = "oracle", SQS = "sqs", ELASTICSEARCH = "elasticsearch", + DYNAMODB = "dynamodb", } const DATASOURCE_PLUS = [ @@ -50,6 +52,7 @@ const providers: Record = { // rest [DatabaseName.ELASTICSEARCH]: elasticsearch.getDatasource, [DatabaseName.MONGODB]: mongodb.getDatasource, + [DatabaseName.DYNAMODB]: dynamodb.getDatasource, } export interface DatasourceDescribeReturnPromise { diff --git a/packages/server/src/tests/jestSetup.ts b/packages/server/src/tests/jestSetup.ts index 60cf96cb51..6fedbf1f5b 100644 --- a/packages/server/src/tests/jestSetup.ts +++ b/packages/server/src/tests/jestSetup.ts @@ -23,6 +23,6 @@ nock.enableNetConnect(host => { testContainerUtils.setupEnv(env, coreEnv) -afterAll(() => { +afterAll(async () => { timers.cleanup() }) diff --git a/packages/server/src/tests/utilities/api/base.ts b/packages/server/src/tests/utilities/api/base.ts index 9b47cfb820..18a9fbc195 100644 --- a/packages/server/src/tests/utilities/api/base.ts +++ b/packages/server/src/tests/utilities/api/base.ts @@ -146,8 +146,9 @@ export abstract class TestAPI { } } + let resp: Response | undefined = undefined try { - return await req + resp = await req } catch (e: any) { // We've found that occasionally the connection between supertest and the // server supertest starts gets reset. Not sure why, but retrying it @@ -161,6 +162,7 @@ export abstract class TestAPI { } throw e } + return resp } protected async getHeaders( diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index def2ab4201..ffc5bc5ab0 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -143,7 +143,6 @@ async function branchMatches( branch: Readonly ): Promise { const toFilter: Record = {} - const preparedCtx = prepareContext(ctx) // Because we allow bindings on both the left and right of each condition in // automation branches, we can't pass the BranchSearchFilters directly to @@ -160,9 +159,9 @@ async function branchMatches( filter.conditions = filter.conditions.map(evaluateBindings) } else { for (const [field, value] of Object.entries(filter)) { - toFilter[field] = processStringSync(field, preparedCtx) + toFilter[field] = processStringSync(field, ctx) if (typeof value === "string" && findHBSBlocks(value).length > 0) { - filter[field] = processStringSync(value, preparedCtx) + filter[field] = processStringSync(value, ctx) } } } @@ -178,17 +177,6 @@ async function branchMatches( return result.length > 0 } -function prepareContext(context: AutomationContext) { - return { - ...context, - steps: { - ...context.steps, - ...context.stepsById, - ...context.stepsByName, - }, - } -} - async function enrichBaseContext(context: AutomationContext) { context.env = await sdkUtils.getEnvironmentVariables() @@ -304,41 +292,37 @@ class Orchestrator { } hasErrored(context: AutomationContext): boolean { - const [_trigger, ...steps] = context.steps - for (const step of steps) { - if (step.success === false) { - return true - } - } - return false + return context._error === true } async execute(): Promise { return await tracer.trace("execute", async span => { span.addTags({ appId: this.appId, automationId: this.automation._id }) - const job = cloneDeep(this.job) - delete job.data.event.appId - delete job.data.event.metadata + const data = cloneDeep(this.job.data) + delete data.event.appId + delete data.event.metadata - if (this.isCron() && !job.data.event.timestamp) { - job.data.event.timestamp = Date.now() + if (this.isCron() && !data.event.timestamp) { + data.event.timestamp = Date.now() } const trigger: AutomationTriggerResult = { - id: job.data.automation.definition.trigger.id, - stepId: job.data.automation.definition.trigger.stepId, + id: data.automation.definition.trigger.id, + stepId: data.automation.definition.trigger.stepId, inputs: null, - outputs: job.data.event, + outputs: data.event, } const result: AutomationResults = { trigger, steps: [trigger] } const ctx: AutomationContext = { trigger: trigger.outputs, - steps: [trigger.outputs], - stepsById: {}, + steps: { "0": trigger.outputs }, stepsByName: {}, + stepsById: {}, user: trigger.outputs.user, + _error: false, + _stepIndex: 1, } await enrichBaseContext(ctx) @@ -348,7 +332,7 @@ class Orchestrator { try { await helpers.withTimeout(timeout, async () => { const [stepOutputs, executionTime] = await utils.time(() => - this.executeSteps(ctx, job.data.automation.definition.steps) + this.executeSteps(ctx, data.automation.definition.steps) ) result.steps.push(...stepOutputs) @@ -400,9 +384,20 @@ class Orchestrator { step: AutomationStep, result: AutomationStepResult ) { - ctx.steps.push(result.outputs) + ctx.steps[step.id] = result.outputs + ctx.steps[step.name || step.id] = result.outputs + ctx.stepsById[step.id] = result.outputs ctx.stepsByName[step.name || step.id] = result.outputs + + ctx._stepIndex ||= 0 + ctx.steps[ctx._stepIndex] = result.outputs + ctx._stepIndex++ + + if (result.outputs.success === false) { + ctx._error = true + } + results.push(result) } @@ -449,7 +444,7 @@ class Orchestrator { stepToLoop: AutomationStep ): Promise { return await tracer.trace("executeLoopStep", async span => { - await processObject(step.inputs, prepareContext(ctx)) + await processObject(step.inputs, ctx) const maxIterations = getLoopMaxIterations(step) const items: Record[] = [] @@ -478,6 +473,7 @@ class Orchestrator { return stepFailure(stepToLoop, { status: AutomationStepStatus.MAX_ITERATIONS, iterations, + items, }) } @@ -488,6 +484,8 @@ class Orchestrator { }) return stepFailure(stepToLoop, { status: AutomationStepStatus.FAILURE_CONDITION, + iterations, + items, }) } @@ -558,7 +556,7 @@ class Orchestrator { } const inputs = automationUtils.cleanInputValues( - await processObject(cloneDeep(step.inputs), prepareContext(ctx)), + await processObject(cloneDeep(step.inputs), ctx), step.schema.inputs.properties ) @@ -566,7 +564,7 @@ class Orchestrator { inputs, appId: this.appId, emitter: this.emitter, - context: prepareContext(ctx), + context: ctx, }) if ( diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index fac8fa61ee..6172895af0 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -117,7 +117,8 @@ export function isSupportedUserSearch( { op: BasicOperator.EQUAL, key: "_id" }, { op: ArrayOperator.ONE_OF, key: "_id" }, ] - for (const [key, operation] of Object.entries(query)) { + const { allOr, onEmptyFilter, ...filters } = query + for (const [key, operation] of Object.entries(filters)) { if (typeof operation !== "object") { return false } diff --git a/packages/string-templates/src/helpers/constants.ts b/packages/string-templates/src/helpers/constants.ts index ee84a1dc47..fb6bf4e4f5 100644 --- a/packages/string-templates/src/helpers/constants.ts +++ b/packages/string-templates/src/helpers/constants.ts @@ -36,6 +36,7 @@ export const HelperFunctionNames = { ALL: "all", LITERAL: "literal", JS: "js", + DECODE_ID: "decodeId", } export const LITERAL_MARKER = "%LITERAL%" diff --git a/packages/string-templates/src/helpers/index.ts b/packages/string-templates/src/helpers/index.ts index fe74a6d711..ea09c2c545 100644 --- a/packages/string-templates/src/helpers/index.ts +++ b/packages/string-templates/src/helpers/index.ts @@ -25,13 +25,29 @@ function isObject(value: string | any[]) { ) } -const HELPERS = [ +export const HELPERS = [ // external helpers new Helper(HelperFunctionNames.OBJECT, (value: any) => { return new Handlebars.SafeString(JSON.stringify(value)) }), // javascript helper new Helper(HelperFunctionNames.JS, processJS, false), + new Helper(HelperFunctionNames.DECODE_ID, (_id: string | { _id: string }) => { + if (!_id) { + return [] + } + // have to replace on the way back as we swapped out the double quotes + // when encoding, but JSON can't handle the single quotes + const id = typeof _id === "string" ? _id : _id._id + const decoded: string = decodeURIComponent(id).replace(/'/g, '"') + try { + const parsed = JSON.parse(decoded) + return Array.isArray(parsed) ? parsed : [parsed] + } catch (err) { + // wasn't json - likely was handlebars for a many to many + return [_id] + } + }), // this help is applied to all statements new Helper( HelperFunctionNames.ALL, diff --git a/packages/string-templates/test/helpers.spec.ts b/packages/string-templates/test/helpers.spec.ts index 12de4f1c29..7ef09cb2a4 100644 --- a/packages/string-templates/test/helpers.spec.ts +++ b/packages/string-templates/test/helpers.spec.ts @@ -517,3 +517,44 @@ describe("helper overlap", () => { expect(output).toEqual("a") }) }) + +describe("Test the decodeId helper", () => { + it("should decode a valid encoded ID", async () => { + const encodedId = encodeURIComponent("[42]") // "%5B42%5D" + const output = await processString("{{ decodeId id }}", { id: encodedId }) + expect(output).toBe("42") + }) + + it("Should return an unchanged string if the string isn't encoded", async () => { + const unencodedId = "forty-two" + const output = await processString("{{ decodeId id }}", { id: unencodedId }) + expect(output).toBe("forty-two") + }) + + it("Should return a string of comma-separated IDs when passed multiple IDs in a URI encoded array", async () => { + const encodedIds = encodeURIComponent("[1,2,3]") // "%5B1%2C2%2C3%5D" + const output = await processString("{{ decodeId id }}", { id: encodedIds }) + expect(output).toBe("1,2,3") + }) + + it("Handles empty array gracefully", async () => { + const output = await processString("{{ decodeId value }}", { + value: [], + }) + expect(output).toBe("[[]]") + }) + + it("Handles undefined gracefully", async () => { + const output = await processString("{{ decodeId value }}", { + value: undefined, + }) + expect(output).toBe("") + }) + + it("Handles null gracefully", async () => { + const output = await processString("{{ decodeId value }}", { + value: undefined, + }) + expect(output).toBe("") + }) +}) diff --git a/packages/worker/src/tests/jestSetup.ts b/packages/worker/src/tests/jestSetup.ts index 6a98031d34..77501b2f06 100644 --- a/packages/worker/src/tests/jestSetup.ts +++ b/packages/worker/src/tests/jestSetup.ts @@ -12,8 +12,7 @@ nock.enableNetConnect(host => { return ( host.includes("localhost") || host.includes("127.0.0.1") || - host.includes("::1") || - host.includes("ethereal.email") // used in realEmail.spec.ts + host.includes("::1") ) })