Merge pull request #13854 from Budibase/BUDI-7656/add-migration

SQS migrations/app migrations

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -8,6 +8,7 @@ import {
   DatabaseOpts,
   DatabasePutOpts,
   DatabaseQueryOpts,
+  DBError,
   Document,
   isDocument,
   RowResponse,
@@ -41,7 +42,7 @@ function buildNano(couchInfo: { url: string; cookie: string }) {
 
 type DBCall<T> = () => Promise<T>
 
-class CouchDBError extends Error {
+class CouchDBError extends Error implements DBError {
   status: number
   statusCode: number
   reason: string

packages/server/src/api/controllers/application.ts

@@ -358,11 +358,14 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
       await createApp(appId)
     }
 
+    const latestMigrationId = appMigrations.getLatestEnabledMigrationId()
+    if (latestMigrationId) {
       // Initialise the app migration version as the latest one
       await appMigrations.updateAppMigrationMetadata({
         appId,
-      version: appMigrations.getLatestMigrationId(),
+        version: latestMigrationId,
       })
+    }
 
     await cache.app.invalidateAppMetadata(appId, newApplication)
     return newApplication

packages/server/src/api/controllers/migrations.ts

@@ -3,7 +3,7 @@ import { migrate as migrationImpl, MIGRATIONS } from "../../migrations"
 import { Ctx } from "@budibase/types"
 import {
   getAppMigrationVersion,
-  getLatestMigrationId,
+  getLatestEnabledMigrationId,
 } from "../../appMigrations"
 
 export async function migrate(ctx: Ctx) {
@@ -27,7 +27,9 @@ export async function getMigrationStatus(ctx: Ctx) {
 
   const latestAppliedMigration = await getAppMigrationVersion(appId)
 
-  const migrated = latestAppliedMigration === getLatestMigrationId()
+  const latestMigrationId = getLatestEnabledMigrationId()
+  const migrated =
+    !latestMigrationId || latestAppliedMigration >= latestMigrationId
 
   ctx.body = { migrated }
   ctx.status = 200

packages/server/src/api/controllers/static/index.ts

@@ -31,7 +31,7 @@ import {
 } from "@budibase/types"
 import {
   getAppMigrationVersion,
-  getLatestMigrationId,
+  getLatestEnabledMigrationId,
 } from "../../../appMigrations"
 
 import send from "koa-send"
@@ -133,7 +133,7 @@ const requiresMigration = async (ctx: Ctx) => {
     ctx.throw("AppId could not be found")
   }
 
-  const latestMigration = getLatestMigrationId()
+  const latestMigration = getLatestEnabledMigrationId()
   if (!latestMigration) {
     return false
   }

packages/server/src/appMigrations/appMigrationMetadata.ts

@@ -33,7 +33,7 @@ export async function getAppMigrationVersion(appId: string): Promise<string> {
   let version
   try {
     metadata = await getFromDB(appId)
-    version = metadata.version
+    version = metadata.version || ""
   } catch (err: any) {
     if (err.status !== 404) {
       throw err

packages/server/src/appMigrations/index.ts

@@ -10,14 +10,25 @@ export * from "./appMigrationMetadata"
 export type AppMigration = {
   id: string
   func: () => Promise<void>
+  // disabled so that by default all migrations listed are enabled
+  disabled?: boolean
 }
 
-export const getLatestMigrationId = () =>
-  MIGRATIONS.map(m => m.id)
-    .sort()
-    .reverse()[0]
+export function getLatestEnabledMigrationId(migrations?: AppMigration[]) {
+  let latestMigrationId: string | undefined
+  for (let migration of migrations || MIGRATIONS) {
+    // if a migration is disabled, all migrations after it are disabled
+    if (migration.disabled) {
+      break
+    }
+    latestMigrationId = migration.id
+  }
+  return latestMigrationId
+}
 
-const getTimestamp = (versionId: string) => versionId?.split("_")[0] || ""
+function getTimestamp(versionId: string) {
+  return versionId?.split("_")[0] || ""
+}
 
 export async function checkMissingMigrations(
   ctx: UserCtx,
@@ -25,9 +36,12 @@ export async function checkMissingMigrations(
   appId: string
 ) {
   const currentVersion = await getAppMigrationVersion(appId)
-  const latestMigration = getLatestMigrationId()
+  const latestMigration = getLatestEnabledMigrationId()
 
-  if (getTimestamp(currentVersion) < getTimestamp(latestMigration)) {
+  if (
+    latestMigration &&
+    getTimestamp(currentVersion) < getTimestamp(latestMigration)
+  ) {
     await queue.add(
       {
         appId,

packages/server/src/appMigrations/migrations.ts

@@ -1,7 +1,15 @@
 // This file should never be manually modified, use `yarn add-app-migration` in order to add a new one
 
+import env from "../environment"
 import { AppMigration } from "."
 
+import m20240604153647_initial_sqs from "./migrations/20240604153647_initial_sqs"
+
+// Migrations will be executed sorted by ID
 export const MIGRATIONS: AppMigration[] = [
-  // Migrations will be executed sorted by id
+  {
+    id: "20240604153647_initial_sqs",
+    func: m20240604153647_initial_sqs,
+    disabled: !env.SQS_SEARCH_ENABLE,
+  },
 ]

packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts

@@ -0,0 +1,52 @@
+import { context } from "@budibase/backend-core"
+import { allLinkDocs } from "../../db/utils"
+import LinkDocumentImpl from "../../db/linkedRows/LinkDocument"
+import sdk from "../../sdk"
+import env from "../../environment"
+
+const migration = async () => {
+  const linkDocs = await allLinkDocs()
+
+  const docsToUpdate = []
+  for (const linkDoc of linkDocs) {
+    if (linkDoc.tableId) {
+      // It already had the required data
+      continue
+    }
+
+    // it already has the junction table ID - no need to migrate
+    if (!linkDoc.tableId) {
+      const newLink = new LinkDocumentImpl(
+        linkDoc.doc1.tableId,
+        linkDoc.doc1.fieldName,
+        linkDoc.doc1.rowId,
+        linkDoc.doc2.tableId,
+        linkDoc.doc2.fieldName,
+        linkDoc.doc2.rowId
+      )
+      newLink._id = linkDoc._id!
+      newLink._rev = linkDoc._rev
+      docsToUpdate.push(newLink)
+    }
+  }
+
+  const db = context.getAppDB()
+  if (docsToUpdate.length) {
+    await db.bulkDocs(docsToUpdate)
+  }
+
+  // at the end make sure design doc is ready
+  await sdk.tables.sqs.syncDefinition()
+  // only do initial search if environment is using SQS already
+  // initial search makes sure that all the indexes have been created
+  // and are ready to use, avoiding any initial waits for large tables
+  if (env.SQS_SEARCH_ENABLE) {
+    const tables = await sdk.tables.getAllInternalTables()
+    // do these one by one - running in parallel could cause problems
+    for (let table of tables) {
+      await db.sql(`select * from ${table._id} limit 1`)
+    }
+  }
+}
+
+export default migration

packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts

@@ -0,0 +1,116 @@
+import * as setup from "../../../api/routes/tests/utilities"
+import { basicTable } from "../../../tests/utilities/structures"
+import {
+  db as dbCore,
+  SQLITE_DESIGN_DOC_ID,
+  context,
+} from "@budibase/backend-core"
+import {
+  LinkDocument,
+  DocumentType,
+  SQLiteDefinition,
+  SQLiteType,
+} from "@budibase/types"
+import {
+  generateJunctionTableID,
+  generateLinkID,
+  generateRowID,
+} from "../../../db/utils"
+import migration from "../20240604153647_initial_sqs"
+
+const config = setup.getConfig()
+let tableId: string
+
+function oldLinkDocInfo() {
+  const tableId1 = `${DocumentType.TABLE}_a`,
+    tableId2 = `${DocumentType.TABLE}_b`
+  return {
+    tableId1,
+    tableId2,
+    rowId1: generateRowID(tableId1, "b"),
+    rowId2: generateRowID(tableId2, "a"),
+    col1: "columnB",
+    col2: "columnA",
+  }
+}
+
+function oldLinkDocID() {
+  const { tableId1, tableId2, rowId1, rowId2, col1, col2 } = oldLinkDocInfo()
+  return generateLinkID(tableId1, tableId2, rowId1, rowId2, col1, col2)
+}
+
+function oldLinkDocument(): Omit<LinkDocument, "tableId"> {
+  const { tableId1, tableId2, rowId1, rowId2, col1, col2 } = oldLinkDocInfo()
+  return {
+    type: "link",
+    _id: oldLinkDocID(),
+    doc1: {
+      tableId: tableId1,
+      fieldName: col1,
+      rowId: rowId1,
+    },
+    doc2: {
+      tableId: tableId2,
+      fieldName: col2,
+      rowId: rowId2,
+    },
+  }
+}
+
+async function sqsDisabled(cb: () => Promise<void>) {
+  await config.withEnv({ SQS_SEARCH_ENABLE: "" }, cb)
+}
+
+async function sqsEnabled(cb: () => Promise<void>) {
+  await config.withEnv({ SQS_SEARCH_ENABLE: "1" }, cb)
+}
+
+beforeAll(async () => {
+  await sqsDisabled(async () => {
+    await config.init()
+    const table = await config.api.table.save(basicTable())
+    tableId = table._id!
+    const db = dbCore.getDB(config.appId!)
+    // old link document
+    await db.put(oldLinkDocument())
+  })
+})
+
+describe("SQS migration", () => {
+  it("test migration runs as expected against an older DB", async () => {
+    const db = dbCore.getDB(config.appId!)
+    // confirm nothing exists initially
+    await sqsDisabled(async () => {
+      let error: any | undefined
+      try {
+        await db.get(SQLITE_DESIGN_DOC_ID)
+      } catch (err: any) {
+        error = err
+      }
+      expect(error).toBeDefined()
+      expect(error.status).toBe(404)
+    })
+    await sqsEnabled(async () => {
+      await context.doInAppContext(config.appId!, async () => {
+        await migration()
+      })
+      const designDoc = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
+      expect(designDoc.sql.tables).toBeDefined()
+      const mainTableDef = designDoc.sql.tables[tableId]
+      expect(mainTableDef).toBeDefined()
+      expect(mainTableDef.fields.name).toEqual(SQLiteType.TEXT)
+      expect(mainTableDef.fields.description).toEqual(SQLiteType.TEXT)
+
+      const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo()
+      const linkDoc = await db.get<LinkDocument>(oldLinkDocID())
+      expect(linkDoc.tableId).toEqual(
+        generateJunctionTableID(tableId1, tableId2)
+      )
+      // should have swapped the documents
+      expect(linkDoc.doc1.tableId).toEqual(tableId2)
+      expect(linkDoc.doc1.rowId).toEqual(rowId2)
+      expect(linkDoc.doc2.tableId).toEqual(tableId1)
+      expect(linkDoc.doc2.rowId).toEqual(rowId1)
+    })
+  })
+})

packages/server/src/appMigrations/tests/migrations.spec.ts

@@ -1,6 +1,7 @@
 import { Header } from "@budibase/backend-core"
 import * as setup from "../../api/routes/tests/utilities"
 import * as migrations from "../migrations"
+import { AppMigration, getLatestEnabledMigrationId } from "../index"
 import { getAppMigrationVersion } from "../appMigrationMetadata"
 
 jest.mock<typeof migrations>("../migrations", () => ({
@@ -52,4 +53,29 @@ describe("migrations", () => {
       },
     })
   })
+
+  it("should disable all migrations after one that is disabled", () => {
+    const MIGRATION_ID1 = "20231211105810_new-test",
+      MIGRATION_ID2 = "20231211105812_new-test",
+      MIGRATION_ID3 = "20231211105814_new-test"
+    // create some migrations to test with
+    const migrations: AppMigration[] = [
+      {
+        id: MIGRATION_ID1,
+        func: async () => {},
+      },
+      {
+        id: MIGRATION_ID2,
+        func: async () => {},
+      },
+      {
+        id: MIGRATION_ID3,
+        func: async () => {},
+      },
+    ]
+
+    expect(getLatestEnabledMigrationId(migrations)).toBe(MIGRATION_ID3)
+    migrations[1].disabled = true
+    expect(getLatestEnabledMigrationId(migrations)).toBe(MIGRATION_ID1)
+  })
 })

packages/server/src/db/linkedRows/LinkDocument.ts

@@ -59,6 +59,9 @@ class LinkDocumentImpl implements LinkDocument {
     this.doc1 = docA.tableId > docB.tableId ? docA : docB
     this.doc2 = docA.tableId > docB.tableId ? docB : docA
   }
+  _rev?: string | undefined
+  createdAt?: string | number | undefined
+  updatedAt?: string | undefined
 }
 
 export default LinkDocumentImpl

packages/server/src/db/utils.ts

@@ -1,5 +1,5 @@
 import newid from "./newid"
-import { db as dbCore } from "@budibase/backend-core"
+import { context, db as dbCore } from "@budibase/backend-core"
 import {
   DatabaseQueryOpts,
   Datasource,
@@ -10,6 +10,7 @@ import {
   RelationshipFieldMetadata,
   SourceName,
   VirtualDocumentType,
+  LinkDocument,
 } from "@budibase/types"
 
 export { DocumentType, VirtualDocumentType } from "@budibase/types"
@@ -137,10 +138,24 @@ export function generateLinkID(
 /**
  * Gets parameters for retrieving link docs, this is a utility function for the getDocParams function.
  */
-export function getLinkParams(otherProps: any = {}) {
+function getLinkParams(otherProps: Partial<DatabaseQueryOpts> = {}) {
   return getDocParams(DocumentType.LINK, null, otherProps)
 }
 
+/**
+ * Gets all the link docs document from the current app db.
+ */
+export async function allLinkDocs() {
+  const db = context.getAppDB()
+
+  const response = await db.allDocs<LinkDocument>(
+    getLinkParams({
+      include_docs: true,
+    })
+  )
+  return response.rows.map(row => row.doc!)
+}
+
 /**
  * Generates a new layout ID.
  * @returns The new layout ID which the layout doc can be stored under.

packages/server/src/environment.ts

@@ -96,6 +96,7 @@ const environment = {
   DISABLE_THREADING: process.env.DISABLE_THREADING,
   DISABLE_AUTOMATION_LOGS: process.env.DISABLE_AUTOMATION_LOGS,
   DISABLE_RATE_LIMITING: process.env.DISABLE_RATE_LIMITING,
+  DISABLE_APP_MIGRATIONS: process.env.SKIP_APP_MIGRATIONS || false,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   SELF_HOSTED: process.env.SELF_HOSTED,

packages/server/src/middleware/appMigrations.ts

@@ -1,9 +1,16 @@
 import { UserCtx } from "@budibase/types"
 import { checkMissingMigrations } from "../appMigrations"
+import env from "../environment"
 
 export default async (ctx: UserCtx, next: any) => {
   const { appId } = ctx
 
+  // migrations can be disabled via environment variable if you
+  // need to completely disable migrations, e.g. for testing
+  if (env.DISABLE_APP_MIGRATIONS) {
+    return next()
+  }
+
   if (!appId) {
     return next()
   }

packages/server/src/sdk/app/tables/internal/sqs.ts

@@ -14,6 +14,7 @@ import {
   CONSTANT_INTERNAL_ROW_COLS,
   generateJunctionTableID,
 } from "../../../../db/utils"
+import { isEqual } from "lodash"
 
 const FieldTypeMap: Record<FieldType, SQLiteType> = {
   [FieldType.BOOLEAN]: SQLiteType.NUMERIC,
@@ -107,9 +108,23 @@ async function buildBaseDefinition(): Promise<PreSaveSQLiteDefinition> {
 
 export async function syncDefinition(): Promise<void> {
   const db = context.getAppDB()
+  let existing: SQLiteDefinition | undefined
+  try {
+    existing = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
+  } catch (err: any) {
+    if (err.status !== 404) {
+      throw err
+    }
+  }
   const definition = await buildBaseDefinition()
+  if (existing) {
+    definition._rev = existing._rev
+  }
+  // only write if something has changed
+  if (!existing || !isEqual(existing.sql, definition.sql)) {
     await db.put(definition)
   }
+}
 
 export async function addTable(table: Table) {
   const db = context.getAppDB()

packages/types/src/documents/app/sqlite.ts

@@ -30,4 +30,7 @@ export interface SQLiteDefinition {
   }
 }
 
-export type PreSaveSQLiteDefinition = Omit<SQLiteDefinition, "_rev">
+export interface PreSaveSQLiteDefinition
+  extends Omit<SQLiteDefinition, "_rev"> {
+  _rev?: string
+}

packages/types/src/sdk/db.ts

@@ -165,3 +165,13 @@ export interface Database {
   deleteIndex(...args: any[]): Promise<any>
   getIndexes(...args: any[]): Promise<any>
 }
+
+export interface DBError extends Error {
+  status: number
+  statusCode: number
+  reason: string
+  name: string
+  errid: string
+  error: string
+  description: string
+}

scripts/add-app-migration.js

@@ -21,7 +21,9 @@ const generateTimestamp = () => {
 }
 
 const createMigrationFile = () => {
-  const migrationFilename = `${generateTimestamp()}_${title}`
+  const migrationFilename = `${generateTimestamp()}_${title
+    .replace(/-/g, "_")
+    .replace(/ /g, "_")}`
   const migrationsDir = "../packages/server/src/appMigrations"
 
   const template = `const migration = async () => {