From 056d65cb1e6d08c4d138845da795a27f610db6da Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 3 Jul 2024 15:07:35 +0100 Subject: [PATCH 1/4] Splitting app migrations for SQS into their own environment variable so that they can be enabled separately. --- packages/server/src/appMigrations/migrations.ts | 2 +- .../appMigrations/migrations/20240604153647_initial_sqs.ts | 2 +- .../migrations/tests/20240604153647_initial_sqs.spec.ts | 4 ++-- packages/server/src/environment.ts | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/server/src/appMigrations/migrations.ts b/packages/server/src/appMigrations/migrations.ts index 14eb9d0923..aeb4844751 100644 --- a/packages/server/src/appMigrations/migrations.ts +++ b/packages/server/src/appMigrations/migrations.ts @@ -10,6 +10,6 @@ export const MIGRATIONS: AppMigration[] = [ { id: "20240604153647_initial_sqs", func: m20240604153647_initial_sqs, - disabled: !env.SQS_SEARCH_ENABLE, + disabled: !env.SQS_MIGRATION_ENABLE, }, ] diff --git a/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts index 800de1418a..6ac0fc2523 100644 --- a/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts +++ b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts @@ -40,7 +40,7 @@ const migration = async () => { // only do initial search if environment is using SQS already // initial search makes sure that all the indexes have been created // and are ready to use, avoiding any initial waits for large tables - if (env.SQS_SEARCH_ENABLE) { + if (env.SQS_MIGRATION_ENABLE) { const tables = await sdk.tables.getAllInternalTables() // do these one by one - running in parallel could cause problems for (let table of tables) { diff --git a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts index 572e694855..08393378fb 100644 --- a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts +++ b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts @@ -67,11 +67,11 @@ function oldLinkDocument(): Omit { } async function sqsDisabled(cb: () => Promise) { - await config.withEnv({ SQS_SEARCH_ENABLE: "" }, cb) + await config.withEnv({ SQS_MIGRATION_ENABLE: "" }, cb) } async function sqsEnabled(cb: () => Promise) { - await config.withEnv({ SQS_SEARCH_ENABLE: "1" }, cb) + await config.withEnv({ SQS_MIGRATION_ENABLE: "1" }, cb) } beforeAll(async () => { diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 341483d861..c6123852fd 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -92,6 +92,7 @@ const environment = { SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE, SQL_ALIASING_DISABLE: process.env.SQL_ALIASING_DISABLE, SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE, + SQS_MIGRATION_ENABLE: process.env.SQS_MIGRATION_ENABLE, // flags ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS, DISABLE_THREADING: process.env.DISABLE_THREADING, From de9462403ca415be0c02819b3708029e93f13294 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 3 Jul 2024 15:08:56 +0100 Subject: [PATCH 2/4] Adding both environment variables. --- packages/server/src/appMigrations/migrations.ts | 2 +- .../src/appMigrations/migrations/20240604153647_initial_sqs.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/appMigrations/migrations.ts b/packages/server/src/appMigrations/migrations.ts index aeb4844751..7437753ada 100644 --- a/packages/server/src/appMigrations/migrations.ts +++ b/packages/server/src/appMigrations/migrations.ts @@ -10,6 +10,6 @@ export const MIGRATIONS: AppMigration[] = [ { id: "20240604153647_initial_sqs", func: m20240604153647_initial_sqs, - disabled: !env.SQS_MIGRATION_ENABLE, + disabled: !(env.SQS_MIGRATION_ENABLE || env.SQS_SEARCH_ENABLE), }, ] diff --git a/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts index 6ac0fc2523..da435705c1 100644 --- a/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts +++ b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts @@ -40,7 +40,7 @@ const migration = async () => { // only do initial search if environment is using SQS already // initial search makes sure that all the indexes have been created // and are ready to use, avoiding any initial waits for large tables - if (env.SQS_MIGRATION_ENABLE) { + if (env.SQS_MIGRATION_ENABLE || env.SQS_SEARCH_ENABLE) { const tables = await sdk.tables.getAllInternalTables() // do these one by one - running in parallel could cause problems for (let table of tables) { From c159ebba1f61079c146010f5c027d6a34b35b9e4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 3 Jul 2024 15:16:13 +0100 Subject: [PATCH 3/4] Updating migration test case to check both env vars. --- .../tests/20240604153647_initial_sqs.spec.ts | 115 +++++++++--------- 1 file changed, 60 insertions(+), 55 deletions(-) diff --git a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts index 08393378fb..0a34fb2bb4 100644 --- a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts +++ b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts @@ -66,64 +66,69 @@ function oldLinkDocument(): Omit { } } -async function sqsDisabled(cb: () => Promise) { - await config.withEnv({ SQS_MIGRATION_ENABLE: "" }, cb) +type SQSEnvVar = "SQS_MIGRATION_ENABLE" | "SQS_SEARCH_ENABLE" + +async function sqsDisabled(envVar: SQSEnvVar, cb: () => Promise) { + await config.withEnv({ [envVar]: "" }, cb) } -async function sqsEnabled(cb: () => Promise) { - await config.withEnv({ SQS_MIGRATION_ENABLE: "1" }, cb) +async function sqsEnabled(envVar: SQSEnvVar, cb: () => Promise) { + await config.withEnv({ [envVar]: "1" }, cb) } -beforeAll(async () => { - await sqsDisabled(async () => { - await config.init() - const table = await config.api.table.save(basicTable()) - tableId = table._id! - const db = dbCore.getDB(config.appId!) - // old link document - await db.put(oldLinkDocument()) - }) -}) - -describe("SQS migration", () => { - it("test migration runs as expected against an older DB", async () => { - const db = dbCore.getDB(config.appId!) - // confirm nothing exists initially - await sqsDisabled(async () => { - let error: any | undefined - try { - await db.get(SQLITE_DESIGN_DOC_ID) - } catch (err: any) { - error = err - } - expect(error).toBeDefined() - expect(error.status).toBe(404) - }) - await sqsEnabled(async () => { - await processMigrations(config.appId!, MIGRATIONS) - const designDoc = await db.get(SQLITE_DESIGN_DOC_ID) - expect(designDoc.sql.tables).toBeDefined() - const mainTableDef = designDoc.sql.tables[tableId] - expect(mainTableDef).toBeDefined() - expect(mainTableDef.fields[prefix("name")]).toEqual({ - field: "name", - type: SQLiteType.TEXT, +describe.each(["SQS_MIGRATION_ENABLE", "SQS_SEARCH_ENABLE"] as SQSEnvVar[])( + "SQS migration with (%s)", + envVar => { + beforeAll(async () => { + await sqsDisabled(envVar, async () => { + await config.init() + const table = await config.api.table.save(basicTable()) + tableId = table._id! + const db = dbCore.getDB(config.appId!) + // old link document + await db.put(oldLinkDocument()) }) - expect(mainTableDef.fields[prefix("description")]).toEqual({ - field: "description", - type: SQLiteType.TEXT, - }) - - const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo() - const linkDoc = await db.get(oldLinkDocID()) - expect(linkDoc.tableId).toEqual( - generateJunctionTableID(tableId1, tableId2) - ) - // should have swapped the documents - expect(linkDoc.doc1.tableId).toEqual(tableId2) - expect(linkDoc.doc1.rowId).toEqual(rowId2) - expect(linkDoc.doc2.tableId).toEqual(tableId1) - expect(linkDoc.doc2.rowId).toEqual(rowId1) }) - }) -}) + + it("test migration runs as expected against an older DB", async () => { + const db = dbCore.getDB(config.appId!) + // confirm nothing exists initially + await sqsDisabled(envVar, async () => { + let error: any | undefined + try { + await db.get(SQLITE_DESIGN_DOC_ID) + } catch (err: any) { + error = err + } + expect(error).toBeDefined() + expect(error.status).toBe(404) + }) + await sqsEnabled(envVar, async () => { + await processMigrations(config.appId!, MIGRATIONS) + const designDoc = await db.get(SQLITE_DESIGN_DOC_ID) + expect(designDoc.sql.tables).toBeDefined() + const mainTableDef = designDoc.sql.tables[tableId] + expect(mainTableDef).toBeDefined() + expect(mainTableDef.fields[prefix("name")]).toEqual({ + field: "name", + type: SQLiteType.TEXT, + }) + expect(mainTableDef.fields[prefix("description")]).toEqual({ + field: "description", + type: SQLiteType.TEXT, + }) + + const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo() + const linkDoc = await db.get(oldLinkDocID()) + expect(linkDoc.tableId).toEqual( + generateJunctionTableID(tableId1, tableId2) + ) + // should have swapped the documents + expect(linkDoc.doc1.tableId).toEqual(tableId2) + expect(linkDoc.doc1.rowId).toEqual(rowId2) + expect(linkDoc.doc2.tableId).toEqual(tableId1) + expect(linkDoc.doc2.rowId).toEqual(rowId1) + }) + }) + } +) From 24e840c9454930ddb4f1d305f827cfb555005e87 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 3 Jul 2024 14:55:42 +0000 Subject: [PATCH 4/4] Bump version to 2.29.11 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 541ff34409..d762e6c26c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.29.10", + "version": "2.29.11", "npmClient": "yarn", "packages": [ "packages/*",