MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14089 from Budibase/fix/separate-sqs-migration-env-var 

Separate SQS migration into another environment variable
This commit is contained in:
Michael Drury 2024-07-03 15:42:52 +01:00 committed by GitHub
parent d15ab0b871 c159ebba1f
commit f504ae9cdd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 63 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/server/src/appMigrations/migrations.ts
View File

@ -10,6 +10,6 @@ export const MIGRATIONS: AppMigration[] = [
{
id: "20240604153647_initial_sqs",
func: m20240604153647_initial_sqs,
disabled: !env.SQS_SEARCH_ENABLE,
disabled: !(env.SQS_MIGRATION_ENABLE || env.SQS_SEARCH_ENABLE),
},
]

2
packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts
View File

@ -40,7 +40,7 @@ const migration = async () => {
// only do initial search if environment is using SQS already
// initial search makes sure that all the indexes have been created
// and are ready to use, avoiding any initial waits for large tables
if (env.SQS_SEARCH_ENABLE) {
if (env.SQS_MIGRATION_ENABLE || env.SQS_SEARCH_ENABLE) {
const tables = await sdk.tables.getAllInternalTables()
// do these one by one - running in parallel could cause problems
for (let table of tables) {

115
packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts
View File

@ -66,64 +66,69 @@ function oldLinkDocument(): Omit<LinkDocument, "tableId"> {
}
}
async function sqsDisabled(cb: () => Promise<void>) {
await config.withEnv({ SQS_SEARCH_ENABLE: "" }, cb)
type SQSEnvVar = "SQS_MIGRATION_ENABLE" | "SQS_SEARCH_ENABLE"
async function sqsDisabled(envVar: SQSEnvVar, cb: () => Promise<void>) {
await config.withEnv({ [envVar]: "" }, cb)
}
async function sqsEnabled(cb: () => Promise<void>) {
await config.withEnv({ SQS_SEARCH_ENABLE: "1" }, cb)
async function sqsEnabled(envVar: SQSEnvVar, cb: () => Promise<void>) {
await config.withEnv({ [envVar]: "1" }, cb)
}
beforeAll(async () => {
await sqsDisabled(async () => {
await config.init()
const table = await config.api.table.save(basicTable())
tableId = table._id!
const db = dbCore.getDB(config.appId!)
// old link document
await db.put(oldLinkDocument())
})
})
describe("SQS migration", () => {
it("test migration runs as expected against an older DB", async () => {
const db = dbCore.getDB(config.appId!)
// confirm nothing exists initially
await sqsDisabled(async () => {
let error: any | undefined
try {
await db.get(SQLITE_DESIGN_DOC_ID)
} catch (err: any) {
error = err
}
expect(error).toBeDefined()
expect(error.status).toBe(404)
})
await sqsEnabled(async () => {
await processMigrations(config.appId!, MIGRATIONS)
const designDoc = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
expect(designDoc.sql.tables).toBeDefined()
const mainTableDef = designDoc.sql.tables[tableId]
expect(mainTableDef).toBeDefined()
expect(mainTableDef.fields[prefix("name")]).toEqual({
field: "name",
type: SQLiteType.TEXT,
describe.each(["SQS_MIGRATION_ENABLE", "SQS_SEARCH_ENABLE"] as SQSEnvVar[])(
"SQS migration with (%s)",
envVar => {
beforeAll(async () => {
await sqsDisabled(envVar, async () => {
await config.init()
const table = await config.api.table.save(basicTable())
tableId = table._id!
const db = dbCore.getDB(config.appId!)
// old link document
await db.put(oldLinkDocument())
})
expect(mainTableDef.fields[prefix("description")]).toEqual({
field: "description",
type: SQLiteType.TEXT,
})
const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo()
const linkDoc = await db.get<LinkDocument>(oldLinkDocID())
expect(linkDoc.tableId).toEqual(
generateJunctionTableID(tableId1, tableId2)
)
// should have swapped the documents
expect(linkDoc.doc1.tableId).toEqual(tableId2)
expect(linkDoc.doc1.rowId).toEqual(rowId2)
expect(linkDoc.doc2.tableId).toEqual(tableId1)
expect(linkDoc.doc2.rowId).toEqual(rowId1)
})
})
})
it("test migration runs as expected against an older DB", async () => {
const db = dbCore.getDB(config.appId!)
// confirm nothing exists initially
await sqsDisabled(envVar, async () => {
let error: any | undefined
try {
await db.get(SQLITE_DESIGN_DOC_ID)
} catch (err: any) {
error = err
}
expect(error).toBeDefined()
expect(error.status).toBe(404)
})
await sqsEnabled(envVar, async () => {
await processMigrations(config.appId!, MIGRATIONS)
const designDoc = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
expect(designDoc.sql.tables).toBeDefined()
const mainTableDef = designDoc.sql.tables[tableId]
expect(mainTableDef).toBeDefined()
expect(mainTableDef.fields[prefix("name")]).toEqual({
field: "name",
type: SQLiteType.TEXT,
})
expect(mainTableDef.fields[prefix("description")]).toEqual({
field: "description",
type: SQLiteType.TEXT,
})
const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo()
const linkDoc = await db.get<LinkDocument>(oldLinkDocID())
expect(linkDoc.tableId).toEqual(
generateJunctionTableID(tableId1, tableId2)
)
// should have swapped the documents
expect(linkDoc.doc1.tableId).toEqual(tableId2)
expect(linkDoc.doc1.rowId).toEqual(rowId2)
expect(linkDoc.doc2.tableId).toEqual(tableId1)
expect(linkDoc.doc2.rowId).toEqual(rowId1)
})
})
}
)

1
packages/server/src/environment.ts
View File

@ -92,6 +92,7 @@ const environment = {
SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE,
SQL_ALIASING_DISABLE: process.env.SQL_ALIASING_DISABLE,
SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE,
SQS_MIGRATION_ENABLE: process.env.SQS_MIGRATION_ENABLE,
// flags
ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
DISABLE_THREADING: process.env.DISABLE_THREADING,