From b622df26837d4afac8825d203999a0ab0353359c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 30 Aug 2024 15:10:55 +0100 Subject: [PATCH 1/7] Defaulting SQS to on in development environments, this removes the need for setting it in the management scripts. --- packages/backend-core/src/features/index.ts | 2 +- packages/server/scripts/dev/manage.js | 1 - packages/worker/scripts/dev/manage.js | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts index a8d6f0d0ef..aa80ad9b73 100644 --- a/packages/backend-core/src/features/index.ts +++ b/packages/backend-core/src/features/index.ts @@ -267,5 +267,5 @@ export class FlagSet, T extends { [key: string]: V }> { // default values set correctly and their types flow through the system. export const flags = new FlagSet({ DEFAULT_VALUES: Flag.boolean(false), - SQS: Flag.boolean(false), + SQS: Flag.boolean(env.isDev()), }) diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js index edcc4a5edb..0794c53506 100644 --- a/packages/server/scripts/dev/manage.js +++ b/packages/server/scripts/dev/manage.js @@ -47,7 +47,6 @@ async function init() { HTTP_LOGGING: "0", VERSION: "0.0.0+local", PASSWORD_MIN_LENGTH: "1", - TENANT_FEATURE_FLAGS: "*:SQS", } config = { ...config, ...existingConfig } diff --git a/packages/worker/scripts/dev/manage.js b/packages/worker/scripts/dev/manage.js index cec8c68d26..4f4b6eb060 100644 --- a/packages/worker/scripts/dev/manage.js +++ b/packages/worker/scripts/dev/manage.js @@ -30,7 +30,6 @@ async function init() { HTTP_LOGGING: "0", VERSION: "0.0.0+local", PASSWORD_MIN_LENGTH: "1", - TENANT_FEATURE_FLAGS: "*:SQS", } config = { ...config, ...existingConfig } From 0311e8433b0b5b273d42bbd69c70a8c5c3c9e573 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 30 Aug 2024 17:06:30 +0100 Subject: [PATCH 2/7] Fixing tests so they can work with SQS. --- .../server/src/api/routes/tests/table.spec.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 52d6c3d379..d6c1651ef0 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -35,7 +35,7 @@ const { basicTable } = setup.structures const ISO_REGEX_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/ describe.each([ - ["internal", undefined], + ["sqs", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], @@ -290,7 +290,7 @@ describe.each([ expected._rev = expect.stringMatching(/^2-.+/) } - expect(updatedTable).toEqual(expected) + expect(updatedTable).toEqual(expect.objectContaining(expected)) const persistedTable = await config.api.table.get(updatedTable._id!) expected = { @@ -304,7 +304,7 @@ describe.each([ if (isInternal) { expected._rev = expect.stringMatching(/^2-.+/) } - expect(persistedTable).toEqual(expected) + expect(persistedTable).toEqual(expect.objectContaining(expected)) }) }) @@ -687,7 +687,7 @@ describe.each([ basicTable(datasource, { name: generator.guid() }) ) const res = await config.api.table.get(table._id!) - expect(res).toEqual(table) + expect(res).toEqual(expect.objectContaining(table)) }) }) @@ -727,10 +727,12 @@ describe.each([ body: { message: `Table ${testTable._id} deleted.` }, }) expect(events.table.deleted).toHaveBeenCalledTimes(1) - expect(events.table.deleted).toHaveBeenCalledWith({ - ...testTable, - tableId: testTable._id, - }) + expect(events.table.deleted).toHaveBeenCalledWith( + expect.objectContaining({ + ...testTable, + tableId: testTable._id, + }) + ) }) it("deletes linked references to the table after deletion", async () => { From bdfd64c98cd9bf02ca4cfdbf87776c937f78ef4b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 30 Aug 2024 17:19:04 +0100 Subject: [PATCH 3/7] Fixing view test. --- packages/server/src/api/routes/tests/view.spec.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/view.spec.ts b/packages/server/src/api/routes/tests/view.spec.ts index 4f38603691..57b589e79d 100644 --- a/packages/server/src/api/routes/tests/view.spec.ts +++ b/packages/server/src/api/routes/tests/view.spec.ts @@ -444,7 +444,10 @@ describe("/views", () => { assertJsonExport(res) expect(events.table.exported).toHaveBeenCalledTimes(1) - expect(events.table.exported).toHaveBeenCalledWith(table, "json") + expect(events.table.exported).toHaveBeenCalledWith( + expect.objectContaining(table), + "json" + ) }) it("should be able to export a table as CSV", async () => { @@ -454,7 +457,10 @@ describe("/views", () => { assertCSVExport(res) expect(events.table.exported).toHaveBeenCalledTimes(1) - expect(events.table.exported).toHaveBeenCalledWith(table, "csv") + expect(events.table.exported).toHaveBeenCalledWith( + expect.objectContaining(table), + "csv" + ) }) it("should be able to export a view as JSON", async () => { From 6249f41bbb38989f4cd75d394f43fcf2405bacea Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 30 Aug 2024 17:34:10 +0100 Subject: [PATCH 4/7] Make sure Lucene correctly sets environment variables in development tests. --- packages/server/src/api/routes/tests/search.spec.ts | 6 +++++- packages/server/src/api/routes/tests/viewV2.spec.ts | 6 +++++- .../server/src/sdk/app/rows/search/tests/search.spec.ts | 6 +++++- .../worker/src/api/routes/global/tests/auditLogs.spec.ts | 4 +++- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 7ac6840be1..4b8c3eb5a1 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -68,7 +68,11 @@ describe.each([ beforeAll(async () => { await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init()) - if (isSqs) { + if (isLucene) { + envCleanup = setCoreEnv({ + TENANT_FEATURE_FLAGS: "*:!SQS", + }) + } else if (isSqs) { envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS", }) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 83ab09cf6d..356f01dee0 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -97,7 +97,11 @@ describe.each([ await withCoreEnv({ TENANT_FEATURE_FLAGS: isSqs ? "*:SQS" : "" }, () => config.init() ) - if (isSqs) { + if (isLucene) { + envCleanup = setCoreEnv({ + TENANT_FEATURE_FLAGS: "*:!SQS", + }) + } else if (isSqs) { envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS", }) diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index 558725c1ed..e7fd095865 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -45,7 +45,11 @@ describe.each([ config.init() ) - if (isSqs) { + if (isLucene) { + envCleanup = setCoreEnv({ + TENANT_FEATURE_FLAGS: "*:!SQS", + }) + } else if (isSqs) { envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS", }) diff --git a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts index 350c9a7edc..b09c27762d 100644 --- a/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts +++ b/packages/worker/src/api/routes/global/tests/auditLogs.spec.ts @@ -17,7 +17,9 @@ describe.each(["lucene", "sql"])("/api/global/auditlogs (%s)", method => { let envCleanup: (() => void) | undefined beforeAll(async () => { - if (method === "sql") { + if (method === "lucene") { + envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" }) + } else if (method === "sql") { envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }) } await config.beforeAll() From 7c1f2522e2412b93a404b79bbf0ba7aa4e2dba29 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 30 Aug 2024 17:42:54 +0100 Subject: [PATCH 5/7] Another test. --- .../server/src/api/routes/tests/row.spec.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index e941c0b267..f9d094190d 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -76,11 +76,11 @@ async function waitForEvent( describe.each([ ["internal", undefined], ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/rows (%s)", (providerType, dsProvider) => { const isInternal = dsProvider === undefined const isLucene = providerType === "lucene" @@ -95,9 +95,11 @@ describe.each([ let envCleanup: (() => void) | undefined beforeAll(async () => { - await withCoreEnv({ TENANT_FEATURE_FLAGS: "*SQS" }, () => config.init()) - if (isSqs) { - envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*SQS" }) + await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init()) + if (isLucene) { + envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:!SQS" }) + } else if (isSqs) { + envCleanup = setCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }) } if (dsProvider) { From 70708dc36e418b224f06e53cc0d35dc0aadf3c3b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 30 Aug 2024 17:47:08 +0100 Subject: [PATCH 6/7] Adding back missing tests. --- packages/server/src/api/routes/tests/row.spec.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index f9d094190d..765d757680 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -76,11 +76,11 @@ async function waitForEvent( describe.each([ ["internal", undefined], ["sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/rows (%s)", (providerType, dsProvider) => { const isInternal = dsProvider === undefined const isLucene = providerType === "lucene" From d5bc5debacb5745d84fc33b65724df49fa7a0e12 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 30 Aug 2024 17:57:02 +0100 Subject: [PATCH 7/7] Another fix. --- packages/server/src/api/routes/tests/misc.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/misc.spec.js b/packages/server/src/api/routes/tests/misc.spec.js index 2560198d00..fe7da1a119 100644 --- a/packages/server/src/api/routes/tests/misc.spec.js +++ b/packages/server/src/api/routes/tests/misc.spec.js @@ -159,7 +159,7 @@ describe("run misc tests", () => { const rowThree = rows.find(row => row.e === 3) expect(rowThree.a).toEqual("9") expect(rowThree.f).toEqual(["Two", "Four"]) - expect(rowThree.g).toEqual(null) + expect(rowThree.g).toEqual(undefined) const rowFour = rows.find(row => row.e === 4) expect(rowFour.a).toEqual("13")