From 863587ef300cf138faffa863696434038ebfc273 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 29 Dec 2023 12:25:25 +0100
Subject: [PATCH 01/23] Fix scripts

---
 scripts/add-app-migration.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/add-app-migration.js b/scripts/add-app-migration.js
index a58d3a4fbe..a4e01be635 100644
--- a/scripts/add-app-migration.js
+++ b/scripts/add-app-migration.js
@@ -21,7 +21,9 @@ const generateTimestamp = () => {
 }
 
 const createMigrationFile = () => {
-  const migrationFilename = `${generateTimestamp()}_${title}`
+  const migrationFilename = `${generateTimestamp()}_${title
+    .replace(/-/g, "_")
+    .replace(/ /g, "_")}`
   const migrationsDir = "../packages/server/src/appMigrations"
 
   const template = `const migration = async () => {

From 90db9efb7045d0dfbf7db6e294442d64e150e816 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 29 Dec 2023 12:43:39 +0100
Subject: [PATCH 02/23] Allow skipping migrations

---
 packages/backend-core/src/environment.ts        | 1 +
 packages/server/src/middleware/appMigrations.ts | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts
index 138dbbd9e0..66c91b19fb 100644
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@@ -183,6 +183,7 @@ const environment = {
     environment[key] = value
   },
   ROLLING_LOG_MAX_SIZE: process.env.ROLLING_LOG_MAX_SIZE || "10M",
+  SKIP_APP_MIGRATIONS: process.env.SKIP_APP_MIGRATIONS || false,
 }
 
 // clean up any environment variable edge cases
diff --git a/packages/server/src/middleware/appMigrations.ts b/packages/server/src/middleware/appMigrations.ts
index 36e021c7ed..1fb13094c6 100644
--- a/packages/server/src/middleware/appMigrations.ts
+++ b/packages/server/src/middleware/appMigrations.ts
@@ -1,9 +1,14 @@
 import { UserCtx } from "@budibase/types"
 import { checkMissingMigrations } from "../appMigrations"
+import { env } from "@budibase/backend-core"
 
 export default async (ctx: UserCtx, next: any) => {
   const { appId } = ctx
 
+  if (env.SKIP_APP_MIGRATIONS) {
+    return next()
+  }
+
   if (!appId) {
     return next()
   }

From f484c6dabd9b135128115707d1627d625c92f95d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 29 Dec 2023 12:43:53 +0100
Subject: [PATCH 03/23] Fix initial run

---
 packages/server/src/appMigrations/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts
index b382d8b533..0758b9f324 100644
--- a/packages/server/src/appMigrations/index.ts
+++ b/packages/server/src/appMigrations/index.ts
@@ -17,7 +17,7 @@ export const getLatestMigrationId = () =>
     .sort()
     .reverse()[0]
 
-const getTimestamp = (versionId: string) => versionId?.split("_")[0]
+const getTimestamp = (versionId: string) => versionId?.split("_")[0] || ""
 
 export async function checkMissingMigrations(
   ctx: UserCtx,

From 61e16b63a510ec19dcc47a03f9bd5ceb0a95a743 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 29 Dec 2023 12:55:14 +0100
Subject: [PATCH 04/23] Fix initial run

---
 packages/server/src/appMigrations/appMigrationMetadata.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/appMigrations/appMigrationMetadata.ts b/packages/server/src/appMigrations/appMigrationMetadata.ts
index 202e78d964..d87ddff3ef 100644
--- a/packages/server/src/appMigrations/appMigrationMetadata.ts
+++ b/packages/server/src/appMigrations/appMigrationMetadata.ts
@@ -33,7 +33,7 @@ export async function getAppMigrationVersion(appId: string): Promise<string> {
   let version
   try {
     metadata = await getFromDB(appId)
-    version = metadata.version
+    version = metadata.version || ""
   } catch (err: any) {
     if (err.status !== 404) {
       throw err

From 4a481877def0483bf3864af5e0f11467835356d9 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 29 Dec 2023 12:55:52 +0100
Subject: [PATCH 05/23] Add empty migration

---
 packages/server/src/appMigrations/migrations.ts             | 6 ++++++
 .../migrations/20231229122514_update_link_documents.ts      | 5 +++++
 2 files changed, 11 insertions(+)
 create mode 100644 packages/server/src/appMigrations/migrations/20231229122514_update_link_documents.ts

diff --git a/packages/server/src/appMigrations/migrations.ts b/packages/server/src/appMigrations/migrations.ts
index d66e2e8895..544e702867 100644
--- a/packages/server/src/appMigrations/migrations.ts
+++ b/packages/server/src/appMigrations/migrations.ts
@@ -2,6 +2,12 @@
 
 import { AppMigration } from "."
 
+import m20231229122514_update_link_documents from "./migrations/20231229122514_update_link_documents"
+
 export const MIGRATIONS: AppMigration[] = [
   // Migrations will be executed sorted by id
+  {
+    id: "20231229122514_update_link_documents",
+    func: m20231229122514_update_link_documents
+  },
 ]
diff --git a/packages/server/src/appMigrations/migrations/20231229122514_update_link_documents.ts b/packages/server/src/appMigrations/migrations/20231229122514_update_link_documents.ts
new file mode 100644
index 0000000000..f23f72e070
--- /dev/null
+++ b/packages/server/src/appMigrations/migrations/20231229122514_update_link_documents.ts
@@ -0,0 +1,5 @@
+const migration = async () => {
+  // Add your migration logic here
+}
+
+export default migration

From e1c37e75a41a6613080c638e088d4a5dfe7c1753 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 29 Dec 2023 13:03:59 +0100
Subject: [PATCH 06/23] Add allLinkDocs utils

---
 packages/server/src/db/utils.ts | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts
index 1712563085..a487c562b6 100644
--- a/packages/server/src/db/utils.ts
+++ b/packages/server/src/db/utils.ts
@@ -1,5 +1,5 @@
 import newid from "./newid"
-import { db as dbCore } from "@budibase/backend-core"
+import { context, db as dbCore } from "@budibase/backend-core"
 import {
   DocumentType,
   FieldSchema,
@@ -7,6 +7,7 @@ import {
   VirtualDocumentType,
   INTERNAL_TABLE_SOURCE_ID,
   DatabaseQueryOpts,
+  LinkDocument,
 } from "@budibase/types"
 import { FieldTypes } from "../constants"
 
@@ -127,10 +128,24 @@ export function generateLinkID(
 /**
  * Gets parameters for retrieving link docs, this is a utility function for the getDocParams function.
  */
-export function getLinkParams(otherProps: any = {}) {
+function getLinkParams(otherProps: Partial<DatabaseQueryOpts> = {}) {
   return getDocParams(DocumentType.LINK, null, otherProps)
 }
 
+/**
+ * Gets all the link docs document from the current app db.
+ */
+export async function allLinkDocs() {
+  const db = context.getAppDB()
+
+  const response = await db.allDocs<LinkDocument>(
+    getLinkParams({
+      include_docs: true,
+    })
+  )
+  return response.rows.map(row => row.doc!)
+}
+
 /**
  * Generates a new layout ID.
  * @returns The new layout ID which the layout doc can be stored under.

From 4a1b99a9dc5f1b650a4ffe3cfa7602b51f6eedb3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 29 Dec 2023 13:32:45 +0100
Subject: [PATCH 07/23] Add migration

---
 .../20231229122514_update_link_documents.ts   | 23 ++++++++++++++++++-
 packages/types/src/documents/app/links.ts     |  1 +
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/appMigrations/migrations/20231229122514_update_link_documents.ts b/packages/server/src/appMigrations/migrations/20231229122514_update_link_documents.ts
index f23f72e070..e1e1d4efc6 100644
--- a/packages/server/src/appMigrations/migrations/20231229122514_update_link_documents.ts
+++ b/packages/server/src/appMigrations/migrations/20231229122514_update_link_documents.ts
@@ -1,5 +1,26 @@
+import { SEPARATOR, context } from "@budibase/backend-core"
+import { allLinkDocs } from "../../db/utils"
+
 const migration = async () => {
-  // Add your migration logic here
+  const linkDocs = await allLinkDocs()
+
+  const docsToUpdate = []
+  for (const linkDoc of linkDocs) {
+    if (linkDoc.tableId) {
+      // It already had the required data
+      continue
+    }
+
+    linkDoc.tableId = [linkDoc.doc1.tableId, linkDoc.doc2.tableId]
+      .sort()
+      .join(SEPARATOR)
+    docsToUpdate.push(linkDoc)
+  }
+
+  if (docsToUpdate.length) {
+    const db = context.getAppDB()
+    await db.bulkDocs(docsToUpdate)
+  }
 }
 
 export default migration
diff --git a/packages/types/src/documents/app/links.ts b/packages/types/src/documents/app/links.ts
index ae7e4de78e..2a9595d99f 100644
--- a/packages/types/src/documents/app/links.ts
+++ b/packages/types/src/documents/app/links.ts
@@ -8,6 +8,7 @@ export interface LinkInfo {
 
 export interface LinkDocument extends Document {
   type: string
+  tableId: string
   doc1: LinkInfo
   doc2: LinkInfo
 }

From e265cc635c3b1f0ad66773078b1b657cbd4c0589 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 29 Dec 2023 13:41:46 +0100
Subject: [PATCH 08/23] Create link docs to new docs

---
 packages/backend-core/src/users/db.ts              |  2 +-
 .../20231229122514_update_link_documents.ts        | 14 ++++++++++----
 packages/server/src/db/linkedRows/LinkDocument.ts  |  7 ++++++-
 packages/server/src/db/linkedRows/index.ts         |  1 -
 4 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts
index 326bed3cc5..01fa4899d1 100644
--- a/packages/backend-core/src/users/db.ts
+++ b/packages/backend-core/src/users/db.ts
@@ -2,7 +2,7 @@ import env from "../environment"
 import * as eventHelpers from "./events"
 import * as accountSdk from "../accounts"
 import * as cache from "../cache"
-import { doInTenant, getGlobalDB, getIdentity, getTenantId } from "../context"
+import { getGlobalDB, getIdentity, getTenantId } from "../context"
 import * as dbUtils from "../db"
 import { EmailUnavailableError, HTTPError } from "../errors"
 import * as platform from "../platform"
diff --git a/packages/server/src/appMigrations/migrations/20231229122514_update_link_documents.ts b/packages/server/src/appMigrations/migrations/20231229122514_update_link_documents.ts
index e1e1d4efc6..adaa0653c4 100644
--- a/packages/server/src/appMigrations/migrations/20231229122514_update_link_documents.ts
+++ b/packages/server/src/appMigrations/migrations/20231229122514_update_link_documents.ts
@@ -1,5 +1,6 @@
-import { SEPARATOR, context } from "@budibase/backend-core"
+import { context } from "@budibase/backend-core"
 import { allLinkDocs } from "../../db/utils"
+import LinkDocumentImpl from "../../db/linkedRows/LinkDocument"
 
 const migration = async () => {
   const linkDocs = await allLinkDocs()
@@ -11,9 +12,14 @@ const migration = async () => {
       continue
     }
 
-    linkDoc.tableId = [linkDoc.doc1.tableId, linkDoc.doc2.tableId]
-      .sort()
-      .join(SEPARATOR)
+    linkDoc.tableId = new LinkDocumentImpl(
+      linkDoc.doc1.tableId,
+      linkDoc.doc1.fieldName,
+      linkDoc.doc1.rowId,
+      linkDoc.doc2.tableId,
+      linkDoc.doc2.fieldName,
+      linkDoc.doc2.rowId
+    ).tableId
     docsToUpdate.push(linkDoc)
   }
 
diff --git a/packages/server/src/db/linkedRows/LinkDocument.ts b/packages/server/src/db/linkedRows/LinkDocument.ts
index 234f43cb48..05097ed84f 100644
--- a/packages/server/src/db/linkedRows/LinkDocument.ts
+++ b/packages/server/src/db/linkedRows/LinkDocument.ts
@@ -1,6 +1,6 @@
 import { generateLinkID } from "../utils"
 import { FieldTypes } from "../../constants"
-import { LinkDocument } from "@budibase/types"
+import { LinkDocument, SEPARATOR } from "@budibase/types"
 
 /**
  * Creates a new link document structure which can be put to the database. It is important to
@@ -17,6 +17,7 @@ import { LinkDocument } from "@budibase/types"
 class LinkDocumentImpl implements LinkDocument {
   _id: string
   type: string
+  tableId: string
   doc1: {
     rowId: string
     fieldName: string
@@ -54,7 +55,11 @@ class LinkDocumentImpl implements LinkDocument {
       fieldName: fieldName2,
       rowId: rowId2,
     }
+    this.tableId = [this.doc1.tableId, this.doc2.tableId].sort().join(SEPARATOR)
   }
+  _rev?: string | undefined
+  createdAt?: string | number | undefined
+  updatedAt?: string | undefined
 }
 
 export default LinkDocumentImpl
diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts
index 7af3f9392f..934b9d4a75 100644
--- a/packages/server/src/db/linkedRows/index.ts
+++ b/packages/server/src/db/linkedRows/index.ts
@@ -18,7 +18,6 @@ import {
   Row,
   LinkDocumentValue,
   FieldType,
-  LinkDocument,
   ContextUser,
 } from "@budibase/types"
 import sdk from "../../sdk"

From cf6f86cb2f4cbf346e5919c63c69cf37c1df256a Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 4 Jun 2024 17:24:11 +0100
Subject: [PATCH 09/23] Updating migrations to correctly cover all the required
 elements.

---
 .../backend-core/src/db/couch/DatabaseImpl.ts |  3 +-
 .../20231229122514_update_link_documents.ts   | 32 -----------
 .../20240604153647_update_link_documents.ts   | 57 +++++++++++++++++++
 .../server/src/sdk/app/tables/internal/sqs.ts | 13 ++++-
 packages/types/src/documents/app/sqlite.ts    |  5 +-
 packages/types/src/sdk/db.ts                  | 10 ++++
 6 files changed, 84 insertions(+), 36 deletions(-)
 delete mode 100644 packages/server/src/appMigrations/migrations/20231229122514_update_link_documents.ts
 create mode 100644 packages/server/src/appMigrations/migrations/20240604153647_update_link_documents.ts

diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts
index 8194d1aabf..1fa1e50e7e 100644
--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@@ -8,6 +8,7 @@ import {
   DatabaseOpts,
   DatabasePutOpts,
   DatabaseQueryOpts,
+  DBError,
   Document,
   isDocument,
   RowResponse,
@@ -41,7 +42,7 @@ function buildNano(couchInfo: { url: string; cookie: string }) {
 
 type DBCall<T> = () => Promise<T>
 
-class CouchDBError extends Error {
+class CouchDBError extends Error implements DBError {
   status: number
   statusCode: number
   reason: string
diff --git a/packages/server/src/appMigrations/migrations/20231229122514_update_link_documents.ts b/packages/server/src/appMigrations/migrations/20231229122514_update_link_documents.ts
deleted file mode 100644
index adaa0653c4..0000000000
--- a/packages/server/src/appMigrations/migrations/20231229122514_update_link_documents.ts
+++ /dev/null
@@ -1,32 +0,0 @@
-import { context } from "@budibase/backend-core"
-import { allLinkDocs } from "../../db/utils"
-import LinkDocumentImpl from "../../db/linkedRows/LinkDocument"
-
-const migration = async () => {
-  const linkDocs = await allLinkDocs()
-
-  const docsToUpdate = []
-  for (const linkDoc of linkDocs) {
-    if (linkDoc.tableId) {
-      // It already had the required data
-      continue
-    }
-
-    linkDoc.tableId = new LinkDocumentImpl(
-      linkDoc.doc1.tableId,
-      linkDoc.doc1.fieldName,
-      linkDoc.doc1.rowId,
-      linkDoc.doc2.tableId,
-      linkDoc.doc2.fieldName,
-      linkDoc.doc2.rowId
-    ).tableId
-    docsToUpdate.push(linkDoc)
-  }
-
-  if (docsToUpdate.length) {
-    const db = context.getAppDB()
-    await db.bulkDocs(docsToUpdate)
-  }
-}
-
-export default migration
diff --git a/packages/server/src/appMigrations/migrations/20240604153647_update_link_documents.ts b/packages/server/src/appMigrations/migrations/20240604153647_update_link_documents.ts
new file mode 100644
index 0000000000..475e7c721c
--- /dev/null
+++ b/packages/server/src/appMigrations/migrations/20240604153647_update_link_documents.ts
@@ -0,0 +1,57 @@
+import { context } from "@budibase/backend-core"
+import { allLinkDocs } from "../../db/utils"
+import LinkDocumentImpl from "../../db/linkedRows/LinkDocument"
+import sdk from "../../sdk"
+import env from "../../environment"
+import { DBError } from "@budibase/types"
+
+const migration = async () => {
+  const linkDocs = await allLinkDocs()
+
+  const docsToUpdate = []
+  for (const linkDoc of linkDocs) {
+    if (linkDoc.tableId) {
+      // It already had the required data
+      continue
+    }
+
+    // it already has the junction table ID - no need to migrate
+    if (!linkDoc.tableId) {
+      linkDoc.tableId = new LinkDocumentImpl(
+        linkDoc.doc1.tableId,
+        linkDoc.doc1.fieldName,
+        linkDoc.doc1.rowId,
+        linkDoc.doc2.tableId,
+        linkDoc.doc2.fieldName,
+        linkDoc.doc2.rowId
+      ).tableId
+      docsToUpdate.push(linkDoc)
+    }
+  }
+
+  const db = context.getAppDB()
+  if (docsToUpdate.length) {
+    await db.bulkDocs(docsToUpdate)
+  }
+
+  // at the end make sure design doc is ready
+  await sdk.tables.sqs.syncDefinition()
+  // only do initial search if environment is using SQS already
+  if (env.SQS_SEARCH_ENABLE) {
+    const tables = await sdk.tables.getAllInternalTables()
+    // do these one by one - running in parallel could cause problems
+    for (let table of tables) {
+      try {
+        await db.sql(`select * from ${table._id} limit 1`)
+      } catch (err) {
+        if (typeof err === "object") {
+          const dbErr = err as DBError
+          throw new Error(`Failed to run initial SQS search - ${dbErr.message}`)
+        }
+        throw err
+      }
+    }
+  }
+}
+
+export default migration
diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts
index eb57d1f3b8..b0c6f28be7 100644
--- a/packages/server/src/sdk/app/tables/internal/sqs.ts
+++ b/packages/server/src/sdk/app/tables/internal/sqs.ts
@@ -107,8 +107,17 @@ async function buildBaseDefinition(): Promise<PreSaveSQLiteDefinition> {
 
 export async function syncDefinition(): Promise<void> {
   const db = context.getAppDB()
-  const definition = await buildBaseDefinition()
-  await db.put(definition)
+  let rev: string | undefined
+  try {
+    const existing = await db.get(SQLITE_DESIGN_DOC_ID)
+    rev = existing._rev
+  } finally {
+    const definition = await buildBaseDefinition()
+    if (rev) {
+      definition._rev = rev
+    }
+    await db.put(definition)
+  }
 }
 
 export async function addTable(table: Table) {
diff --git a/packages/types/src/documents/app/sqlite.ts b/packages/types/src/documents/app/sqlite.ts
index 5636fef15b..516669bd59 100644
--- a/packages/types/src/documents/app/sqlite.ts
+++ b/packages/types/src/documents/app/sqlite.ts
@@ -30,4 +30,7 @@ export interface SQLiteDefinition {
   }
 }
 
-export type PreSaveSQLiteDefinition = Omit<SQLiteDefinition, "_rev">
+export interface PreSaveSQLiteDefinition
+  extends Omit<SQLiteDefinition, "_rev"> {
+  _rev?: string
+}
diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts
index 7ad740ad05..63c37195b7 100644
--- a/packages/types/src/sdk/db.ts
+++ b/packages/types/src/sdk/db.ts
@@ -165,3 +165,13 @@ export interface Database {
   deleteIndex(...args: any[]): Promise<any>
   getIndexes(...args: any[]): Promise<any>
 }
+
+export interface DBError extends Error {
+  status: number
+  statusCode: number
+  reason: string
+  name: string
+  errid: string
+  error: string
+  description: string
+}

From f735f8c6f50c9da63710e3b635df5288244e4bed Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 4 Jun 2024 17:24:56 +0100
Subject: [PATCH 10/23] Comment update.

---
 .../migrations/20240604153647_update_link_documents.ts          | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/server/src/appMigrations/migrations/20240604153647_update_link_documents.ts b/packages/server/src/appMigrations/migrations/20240604153647_update_link_documents.ts
index 475e7c721c..d092f416ff 100644
--- a/packages/server/src/appMigrations/migrations/20240604153647_update_link_documents.ts
+++ b/packages/server/src/appMigrations/migrations/20240604153647_update_link_documents.ts
@@ -37,6 +37,8 @@ const migration = async () => {
   // at the end make sure design doc is ready
   await sdk.tables.sqs.syncDefinition()
   // only do initial search if environment is using SQS already
+  // initial search makes sure that all the indexes have been created
+  // and are ready to use, avoiding any initial waits for large tables
   if (env.SQS_SEARCH_ENABLE) {
     const tables = await sdk.tables.getAllInternalTables()
     // do these one by one - running in parallel could cause problems

From dbda7b5ee4ee2117adc88540c677a4f595ba0e8f Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 4 Jun 2024 17:25:44 +0100
Subject: [PATCH 11/23] Updating migration name.

---
 packages/server/src/appMigrations/migrations.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/appMigrations/migrations.ts b/packages/server/src/appMigrations/migrations.ts
index 544e702867..3d14bd5fa6 100644
--- a/packages/server/src/appMigrations/migrations.ts
+++ b/packages/server/src/appMigrations/migrations.ts
@@ -2,12 +2,12 @@
 
 import { AppMigration } from "."
 
-import m20231229122514_update_link_documents from "./migrations/20231229122514_update_link_documents"
+import m20240604153647_update_link_documents from "./migrations/20240604153647_update_link_documents"
 
 export const MIGRATIONS: AppMigration[] = [
   // Migrations will be executed sorted by id
   {
-    id: "20231229122514_update_link_documents",
-    func: m20231229122514_update_link_documents
+    id: "20240604153647_update_link_documents",
+    func: m20240604153647_update_link_documents,
   },
 ]

From f062b73852590fbe5392d71f00ecb62a86262c7e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 4 Jun 2024 17:39:53 +0100
Subject: [PATCH 12/23] Only run migration if SQS is enabled.

---
 .../server/src/appMigrations/migrations.ts    | 20 +++++++++++--------
 ...ments.ts => 20240604153647_initial_sqs.ts} |  0
 2 files changed, 12 insertions(+), 8 deletions(-)
 rename packages/server/src/appMigrations/migrations/{20240604153647_update_link_documents.ts => 20240604153647_initial_sqs.ts} (100%)

diff --git a/packages/server/src/appMigrations/migrations.ts b/packages/server/src/appMigrations/migrations.ts
index 3d14bd5fa6..48b7f8f0b2 100644
--- a/packages/server/src/appMigrations/migrations.ts
+++ b/packages/server/src/appMigrations/migrations.ts
@@ -1,13 +1,17 @@
 // This file should never be manually modified, use `yarn add-app-migration` in order to add a new one
 
+import env from "../environment"
 import { AppMigration } from "."
 
-import m20240604153647_update_link_documents from "./migrations/20240604153647_update_link_documents"
+import m20240604153647_initial_sqs from "./migrations/20240604153647_initial_sqs"
 
-export const MIGRATIONS: AppMigration[] = [
-  // Migrations will be executed sorted by id
-  {
-    id: "20240604153647_update_link_documents",
-    func: m20240604153647_update_link_documents,
-  },
-]
+// Migrations will be executed sorted by ID
+export const MIGRATIONS: AppMigration[] = []
+
+// only run the SQS migration if SQS is enabled
+if (env.SQS_SEARCH_ENABLE) {
+  MIGRATIONS.push({
+    id: "20240604153647_initial_sqs",
+    func: m20240604153647_initial_sqs,
+  })
+}
diff --git a/packages/server/src/appMigrations/migrations/20240604153647_update_link_documents.ts b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts
similarity index 100%
rename from packages/server/src/appMigrations/migrations/20240604153647_update_link_documents.ts
rename to packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts

From 80ddadbe06d7521428b6e63d0229d5d148f36c4e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 5 Jun 2024 16:57:16 +0100
Subject: [PATCH 13/23] Updating app migration to use a slightly different
 mechanism to allow disabling the migration pipeline.

---
 packages/server/src/appMigrations/index.ts      |  6 ++++++
 packages/server/src/appMigrations/migrations.ts | 12 +++++-------
 packages/server/src/middleware/appMigrations.ts |  6 ++++--
 3 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts
index 0758b9f324..74e0efe422 100644
--- a/packages/server/src/appMigrations/index.ts
+++ b/packages/server/src/appMigrations/index.ts
@@ -10,8 +10,14 @@ export * from "./appMigrationMetadata"
 export type AppMigration = {
   id: string
   func: () => Promise<void>
+  // disabled so that by default all migrations listed are enabled
+  disabled?: boolean
 }
 
+// all migrations must be enabled for migrations to run
+export const migrationsEnabled = (): boolean =>
+  MIGRATIONS.find(m => m.disabled) == null
+
 export const getLatestMigrationId = () =>
   MIGRATIONS.map(m => m.id)
     .sort()
diff --git a/packages/server/src/appMigrations/migrations.ts b/packages/server/src/appMigrations/migrations.ts
index 48b7f8f0b2..14eb9d0923 100644
--- a/packages/server/src/appMigrations/migrations.ts
+++ b/packages/server/src/appMigrations/migrations.ts
@@ -6,12 +6,10 @@ import { AppMigration } from "."
 import m20240604153647_initial_sqs from "./migrations/20240604153647_initial_sqs"
 
 // Migrations will be executed sorted by ID
-export const MIGRATIONS: AppMigration[] = []
-
-// only run the SQS migration if SQS is enabled
-if (env.SQS_SEARCH_ENABLE) {
-  MIGRATIONS.push({
+export const MIGRATIONS: AppMigration[] = [
+  {
     id: "20240604153647_initial_sqs",
     func: m20240604153647_initial_sqs,
-  })
-}
+    disabled: !env.SQS_SEARCH_ENABLE,
+  },
+]
diff --git a/packages/server/src/middleware/appMigrations.ts b/packages/server/src/middleware/appMigrations.ts
index 1fb13094c6..5ec3126fc7 100644
--- a/packages/server/src/middleware/appMigrations.ts
+++ b/packages/server/src/middleware/appMigrations.ts
@@ -1,11 +1,13 @@
 import { UserCtx } from "@budibase/types"
-import { checkMissingMigrations } from "../appMigrations"
+import { checkMissingMigrations, migrationsEnabled } from "../appMigrations"
 import { env } from "@budibase/backend-core"
 
 export default async (ctx: UserCtx, next: any) => {
   const { appId } = ctx
 
-  if (env.SKIP_APP_MIGRATIONS) {
+  // migrations can be disabled via environment variable, or can be disabled
+  // due to some of the migrations not being ready to run - disables all migrations
+  if (env.SKIP_APP_MIGRATIONS || !migrationsEnabled()) {
     return next()
   }
 

From 3cf35881928eea3258e62dfc73d572dde0c2f30e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 5 Jun 2024 17:09:46 +0100
Subject: [PATCH 14/23] Add test for migration disabling.

---
 .../server/src/appMigrations/tests/migrations.spec.ts | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/packages/server/src/appMigrations/tests/migrations.spec.ts b/packages/server/src/appMigrations/tests/migrations.spec.ts
index 7af2346934..9f924aa2c5 100644
--- a/packages/server/src/appMigrations/tests/migrations.spec.ts
+++ b/packages/server/src/appMigrations/tests/migrations.spec.ts
@@ -1,6 +1,7 @@
 import { Header } from "@budibase/backend-core"
 import * as setup from "../../api/routes/tests/utilities"
 import * as migrations from "../migrations"
+import { migrationsEnabled } from "../index"
 import { getAppMigrationVersion } from "../appMigrationMetadata"
 
 jest.mock<typeof migrations>("../migrations", () => ({
@@ -52,4 +53,14 @@ describe("migrations", () => {
       },
     })
   })
+
+  it("should disable migrations if any migration is disabled", () => {
+    migrations.MIGRATIONS.push({
+      id: "20231211105814_new-test",
+      func: async () => {},
+      disabled: true,
+    })
+
+    expect(migrationsEnabled()).toBe(false)
+  })
 })

From 0a5a7884403bafe545f839ae82906973dde55084 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 5 Jun 2024 18:19:44 +0100
Subject: [PATCH 15/23] Updating migrations to disable all migrations after the
 first disabled migration.

---
 .../server/src/api/controllers/application.ts |  2 +-
 .../server/src/api/controllers/migrations.ts  |  4 ++--
 .../src/api/controllers/static/index.ts       |  4 ++--
 packages/server/src/appMigrations/index.ts    | 24 ++++++++++++-------
 .../appMigrations/tests/migrations.spec.ts    | 18 +++++++++++---
 .../server/src/middleware/appMigrations.ts    |  4 ++--
 6 files changed, 38 insertions(+), 18 deletions(-)

diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index e73058239b..1555dc8afd 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -361,7 +361,7 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
     // Initialise the app migration version as the latest one
     await appMigrations.updateAppMigrationMetadata({
       appId,
-      version: appMigrations.getLatestMigrationId(),
+      version: appMigrations.getLatestEnabledMigrationId(),
     })
 
     await cache.app.invalidateAppMetadata(appId, newApplication)
diff --git a/packages/server/src/api/controllers/migrations.ts b/packages/server/src/api/controllers/migrations.ts
index c8f786578d..9a23226837 100644
--- a/packages/server/src/api/controllers/migrations.ts
+++ b/packages/server/src/api/controllers/migrations.ts
@@ -3,7 +3,7 @@ import { migrate as migrationImpl, MIGRATIONS } from "../../migrations"
 import { Ctx } from "@budibase/types"
 import {
   getAppMigrationVersion,
-  getLatestMigrationId,
+  getLatestEnabledMigrationId,
 } from "../../appMigrations"
 
 export async function migrate(ctx: Ctx) {
@@ -27,7 +27,7 @@ export async function getMigrationStatus(ctx: Ctx) {
 
   const latestAppliedMigration = await getAppMigrationVersion(appId)
 
-  const migrated = latestAppliedMigration === getLatestMigrationId()
+  const migrated = latestAppliedMigration === getLatestEnabledMigrationId()
 
   ctx.body = { migrated }
   ctx.status = 200
diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index 63240c8ecb..8db7816a9c 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -31,7 +31,7 @@ import {
 } from "@budibase/types"
 import {
   getAppMigrationVersion,
-  getLatestMigrationId,
+  getLatestEnabledMigrationId,
 } from "../../../appMigrations"
 
 import send from "koa-send"
@@ -133,7 +133,7 @@ const requiresMigration = async (ctx: Ctx) => {
     ctx.throw("AppId could not be found")
   }
 
-  const latestMigration = getLatestMigrationId()
+  const latestMigration = getLatestEnabledMigrationId()
   if (!latestMigration) {
     return false
   }
diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts
index 74e0efe422..a3e3858559 100644
--- a/packages/server/src/appMigrations/index.ts
+++ b/packages/server/src/appMigrations/index.ts
@@ -14,16 +14,24 @@ export type AppMigration = {
   disabled?: boolean
 }
 
-// all migrations must be enabled for migrations to run
-export const migrationsEnabled = (): boolean =>
-  MIGRATIONS.find(m => m.disabled) == null
-
-export const getLatestMigrationId = () =>
-  MIGRATIONS.map(m => m.id)
+export function getLatestEnabledMigrationId() {
+  const enabledMigrations: AppMigration[] = []
+  for (let migration of MIGRATIONS) {
+    // if a migration is disabled, all migrations after it are disabled
+    if (migration.disabled) {
+      break
+    }
+    enabledMigrations.push(migration)
+  }
+  return enabledMigrations
+    .map(m => m.id)
     .sort()
     .reverse()[0]
+}
 
-const getTimestamp = (versionId: string) => versionId?.split("_")[0] || ""
+function getTimestamp(versionId: string) {
+  return versionId?.split("_")[0] || ""
+}
 
 export async function checkMissingMigrations(
   ctx: UserCtx,
@@ -31,7 +39,7 @@ export async function checkMissingMigrations(
   appId: string
 ) {
   const currentVersion = await getAppMigrationVersion(appId)
-  const latestMigration = getLatestMigrationId()
+  const latestMigration = getLatestEnabledMigrationId()
 
   if (getTimestamp(currentVersion) < getTimestamp(latestMigration)) {
     await queue.add(
diff --git a/packages/server/src/appMigrations/tests/migrations.spec.ts b/packages/server/src/appMigrations/tests/migrations.spec.ts
index 9f924aa2c5..1ea54396f7 100644
--- a/packages/server/src/appMigrations/tests/migrations.spec.ts
+++ b/packages/server/src/appMigrations/tests/migrations.spec.ts
@@ -1,7 +1,7 @@
 import { Header } from "@budibase/backend-core"
 import * as setup from "../../api/routes/tests/utilities"
 import * as migrations from "../migrations"
-import { migrationsEnabled } from "../index"
+import { getLatestEnabledMigrationId } from "../index"
 import { getAppMigrationVersion } from "../appMigrationMetadata"
 
 jest.mock<typeof migrations>("../migrations", () => ({
@@ -55,12 +55,24 @@ describe("migrations", () => {
   })
 
   it("should disable migrations if any migration is disabled", () => {
+    // remove all migrations
+    migrations.MIGRATIONS.splice(0, migrations.MIGRATIONS.length)
+    migrations.MIGRATIONS.push({
+      id: "20231211105810_new-test",
+      func: async () => {},
+    })
+    migrations.MIGRATIONS.push({
+      id: "20231211105812_new-test",
+      func: async () => {},
+    })
     migrations.MIGRATIONS.push({
       id: "20231211105814_new-test",
       func: async () => {},
-      disabled: true,
     })
 
-    expect(migrationsEnabled()).toBe(false)
+    expect(getLatestEnabledMigrationId()).toBe("20231211105814_new-test")
+
+    migrations.MIGRATIONS[1].disabled = true
+    expect(getLatestEnabledMigrationId()).toBe("20231211105810_new-test")
   })
 })
diff --git a/packages/server/src/middleware/appMigrations.ts b/packages/server/src/middleware/appMigrations.ts
index 5ec3126fc7..a8ea886ab2 100644
--- a/packages/server/src/middleware/appMigrations.ts
+++ b/packages/server/src/middleware/appMigrations.ts
@@ -1,5 +1,5 @@
 import { UserCtx } from "@budibase/types"
-import { checkMissingMigrations, migrationsEnabled } from "../appMigrations"
+import { checkMissingMigrations } from "../appMigrations"
 import { env } from "@budibase/backend-core"
 
 export default async (ctx: UserCtx, next: any) => {
@@ -7,7 +7,7 @@ export default async (ctx: UserCtx, next: any) => {
 
   // migrations can be disabled via environment variable, or can be disabled
   // due to some of the migrations not being ready to run - disables all migrations
-  if (env.SKIP_APP_MIGRATIONS || !migrationsEnabled()) {
+  if (env.SKIP_APP_MIGRATIONS) {
     return next()
   }
 

From f0f8c14216156fcadec02b43e29cafeda01a7f78 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 6 Jun 2024 14:58:03 +0100
Subject: [PATCH 16/23] Fixing an issue introduced here which caused template
 test to fail.

---
 packages/server/src/sdk/app/tables/internal/sqs.ts | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts
index b0c6f28be7..52a0cfbda0 100644
--- a/packages/server/src/sdk/app/tables/internal/sqs.ts
+++ b/packages/server/src/sdk/app/tables/internal/sqs.ts
@@ -108,16 +108,15 @@ async function buildBaseDefinition(): Promise<PreSaveSQLiteDefinition> {
 export async function syncDefinition(): Promise<void> {
   const db = context.getAppDB()
   let rev: string | undefined
-  try {
+  if (await db.exists(SQLITE_DESIGN_DOC_ID)) {
     const existing = await db.get(SQLITE_DESIGN_DOC_ID)
     rev = existing._rev
-  } finally {
-    const definition = await buildBaseDefinition()
-    if (rev) {
-      definition._rev = rev
-    }
-    await db.put(definition)
   }
+  const definition = await buildBaseDefinition()
+  if (rev) {
+    definition._rev = rev
+  }
+  await db.put(definition)
 }
 
 export async function addTable(table: Table) {

From 3c4cf69463b2460660b7ff6f63523c1b5ae0e39d Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 6 Jun 2024 15:14:59 +0100
Subject: [PATCH 17/23] Updating migration integrity test to allow certain
 properties to change.

---
 .../tests/migrations.integrity.spec.ts        | 29 +++++++++++++++++--
 1 file changed, 26 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/appMigrations/tests/migrations.integrity.spec.ts b/packages/server/src/appMigrations/tests/migrations.integrity.spec.ts
index 145a06d7f5..2de27ef9d4 100644
--- a/packages/server/src/appMigrations/tests/migrations.integrity.spec.ts
+++ b/packages/server/src/appMigrations/tests/migrations.integrity.spec.ts
@@ -2,6 +2,23 @@ import { context } from "@budibase/backend-core"
 import * as setup from "../../api/routes/tests/utilities"
 import * as migrations from "../migrations"
 
+function removeChangeableKeys(documents: Document[]) {
+  const changeableKeys = ["createdAt", "updatedAt", "_rev", "rev"]
+  function iterate(obj: Record<string, any>) {
+    for (let key of Object.keys(obj)) {
+      if (typeof obj[key] === "object") {
+        iterate(obj[key])
+      } else if (changeableKeys.indexOf(key) !== -1) {
+        delete obj[key]
+      }
+    }
+  }
+  for (let doc of documents) {
+    iterate(doc)
+  }
+  return documents
+}
+
 describe("migration integrity", () => {
   // These test is checking that each migration is "idempotent".
   // We should be able to rerun any migration, with any rerun not modifiying anything. The code should be aware that the migration already ran
@@ -13,12 +30,18 @@ describe("migration integrity", () => {
       const db = context.getAppDB()
       for (const migration of migrations.MIGRATIONS) {
         await migration.func()
-        const docs = await db.allDocs({ include_docs: true })
+        const preResp = await db.allDocs({ include_docs: true })
 
         await migration.func()
-        const latestDocs = await db.allDocs({ include_docs: true })
+        const postResp = await db.allDocs({ include_docs: true })
 
-        expect(docs).toEqual(latestDocs)
+        const preDocs = removeChangeableKeys(
+          preResp.rows.map(row => row.doc as Document)
+        )
+        const postDocs = removeChangeableKeys(
+          postResp.rows.map(row => row.doc as Document)
+        )
+        expect(preDocs).toEqual(postDocs)
       }
     })
   })

From 244fbe42b145666897e3a7db59b91b61ba3bc4b0 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 6 Jun 2024 16:04:23 +0100
Subject: [PATCH 18/23] Updating migration to check if the SQL definitions have
 changed, if they haven't don't write them again.

---
 .../tests/migrations.integrity.spec.ts        | 29 ++-----------------
 .../server/src/sdk/app/tables/internal/sqs.ts | 15 ++++++----
 2 files changed, 12 insertions(+), 32 deletions(-)

diff --git a/packages/server/src/appMigrations/tests/migrations.integrity.spec.ts b/packages/server/src/appMigrations/tests/migrations.integrity.spec.ts
index 2de27ef9d4..145a06d7f5 100644
--- a/packages/server/src/appMigrations/tests/migrations.integrity.spec.ts
+++ b/packages/server/src/appMigrations/tests/migrations.integrity.spec.ts
@@ -2,23 +2,6 @@ import { context } from "@budibase/backend-core"
 import * as setup from "../../api/routes/tests/utilities"
 import * as migrations from "../migrations"
 
-function removeChangeableKeys(documents: Document[]) {
-  const changeableKeys = ["createdAt", "updatedAt", "_rev", "rev"]
-  function iterate(obj: Record<string, any>) {
-    for (let key of Object.keys(obj)) {
-      if (typeof obj[key] === "object") {
-        iterate(obj[key])
-      } else if (changeableKeys.indexOf(key) !== -1) {
-        delete obj[key]
-      }
-    }
-  }
-  for (let doc of documents) {
-    iterate(doc)
-  }
-  return documents
-}
-
 describe("migration integrity", () => {
   // These test is checking that each migration is "idempotent".
   // We should be able to rerun any migration, with any rerun not modifiying anything. The code should be aware that the migration already ran
@@ -30,18 +13,12 @@ describe("migration integrity", () => {
       const db = context.getAppDB()
       for (const migration of migrations.MIGRATIONS) {
         await migration.func()
-        const preResp = await db.allDocs({ include_docs: true })
+        const docs = await db.allDocs({ include_docs: true })
 
         await migration.func()
-        const postResp = await db.allDocs({ include_docs: true })
+        const latestDocs = await db.allDocs({ include_docs: true })
 
-        const preDocs = removeChangeableKeys(
-          preResp.rows.map(row => row.doc as Document)
-        )
-        const postDocs = removeChangeableKeys(
-          postResp.rows.map(row => row.doc as Document)
-        )
-        expect(preDocs).toEqual(postDocs)
+        expect(docs).toEqual(latestDocs)
       }
     })
   })
diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts
index 52a0cfbda0..6e0e6e7496 100644
--- a/packages/server/src/sdk/app/tables/internal/sqs.ts
+++ b/packages/server/src/sdk/app/tables/internal/sqs.ts
@@ -14,6 +14,7 @@ import {
   CONSTANT_INTERNAL_ROW_COLS,
   generateJunctionTableID,
 } from "../../../../db/utils"
+import { isEqual } from "lodash"
 
 const FieldTypeMap: Record<FieldType, SQLiteType> = {
   [FieldType.BOOLEAN]: SQLiteType.NUMERIC,
@@ -107,16 +108,18 @@ async function buildBaseDefinition(): Promise<PreSaveSQLiteDefinition> {
 
 export async function syncDefinition(): Promise<void> {
   const db = context.getAppDB()
-  let rev: string | undefined
+  let existing: SQLiteDefinition | undefined
   if (await db.exists(SQLITE_DESIGN_DOC_ID)) {
-    const existing = await db.get(SQLITE_DESIGN_DOC_ID)
-    rev = existing._rev
+    existing = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
   }
   const definition = await buildBaseDefinition()
-  if (rev) {
-    definition._rev = rev
+  if (existing) {
+    definition._rev = existing._rev
+  }
+  // only write if something has changed
+  if (!existing || !isEqual(existing.sql, definition.sql)) {
+    await db.put(definition)
   }
-  await db.put(definition)
 }
 
 export async function addTable(table: Table) {

From 70aa43680d3115ddbc1b7d0cc2511146b6074953 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 6 Jun 2024 16:49:03 +0100
Subject: [PATCH 19/23] Addressing PR comments.

---
 packages/backend-core/src/environment.ts      |  1 -
 .../server/src/api/controllers/migrations.ts  |  2 +-
 packages/server/src/appMigrations/index.ts    |  4 +-
 .../migrations/20240604153647_initial_sqs.ts  | 10 +----
 .../appMigrations/tests/migrations.spec.ts    | 43 ++++++++++---------
 .../server/src/middleware/appMigrations.ts    |  4 +-
 .../server/src/sdk/app/tables/internal/sqs.ts |  6 ++-
 7 files changed, 34 insertions(+), 36 deletions(-)

diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts
index 14b04ae9f8..1e7da2f9a2 100644
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@@ -193,7 +193,6 @@ const environment = {
   },
   ROLLING_LOG_MAX_SIZE: process.env.ROLLING_LOG_MAX_SIZE || "10M",
   DISABLE_SCIM_CALLS: process.env.DISABLE_SCIM_CALLS,
-  SKIP_APP_MIGRATIONS: process.env.SKIP_APP_MIGRATIONS || false,
 }
 
 // clean up any environment variable edge cases
diff --git a/packages/server/src/api/controllers/migrations.ts b/packages/server/src/api/controllers/migrations.ts
index 9a23226837..97c0ae8712 100644
--- a/packages/server/src/api/controllers/migrations.ts
+++ b/packages/server/src/api/controllers/migrations.ts
@@ -27,7 +27,7 @@ export async function getMigrationStatus(ctx: Ctx) {
 
   const latestAppliedMigration = await getAppMigrationVersion(appId)
 
-  const migrated = latestAppliedMigration === getLatestEnabledMigrationId()
+  const migrated = latestAppliedMigration >= getLatestEnabledMigrationId()
 
   ctx.body = { migrated }
   ctx.status = 200
diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts
index a3e3858559..cac1e82716 100644
--- a/packages/server/src/appMigrations/index.ts
+++ b/packages/server/src/appMigrations/index.ts
@@ -14,9 +14,9 @@ export type AppMigration = {
   disabled?: boolean
 }
 
-export function getLatestEnabledMigrationId() {
+export function getLatestEnabledMigrationId(migrations?: AppMigration[]) {
   const enabledMigrations: AppMigration[] = []
-  for (let migration of MIGRATIONS) {
+  for (let migration of migrations || MIGRATIONS) {
     // if a migration is disabled, all migrations after it are disabled
     if (migration.disabled) {
       break
diff --git a/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts
index d092f416ff..05174b8cf1 100644
--- a/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts
+++ b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts
@@ -43,15 +43,7 @@ const migration = async () => {
     const tables = await sdk.tables.getAllInternalTables()
     // do these one by one - running in parallel could cause problems
     for (let table of tables) {
-      try {
-        await db.sql(`select * from ${table._id} limit 1`)
-      } catch (err) {
-        if (typeof err === "object") {
-          const dbErr = err as DBError
-          throw new Error(`Failed to run initial SQS search - ${dbErr.message}`)
-        }
-        throw err
-      }
+      await db.sql(`select * from ${table._id} limit 1`)
     }
   }
 }
diff --git a/packages/server/src/appMigrations/tests/migrations.spec.ts b/packages/server/src/appMigrations/tests/migrations.spec.ts
index 1ea54396f7..1da94f503f 100644
--- a/packages/server/src/appMigrations/tests/migrations.spec.ts
+++ b/packages/server/src/appMigrations/tests/migrations.spec.ts
@@ -1,7 +1,7 @@
 import { Header } from "@budibase/backend-core"
 import * as setup from "../../api/routes/tests/utilities"
 import * as migrations from "../migrations"
-import { getLatestEnabledMigrationId } from "../index"
+import { AppMigration, getLatestEnabledMigrationId } from "../index"
 import { getAppMigrationVersion } from "../appMigrationMetadata"
 
 jest.mock<typeof migrations>("../migrations", () => ({
@@ -54,25 +54,28 @@ describe("migrations", () => {
     })
   })
 
-  it("should disable migrations if any migration is disabled", () => {
-    // remove all migrations
-    migrations.MIGRATIONS.splice(0, migrations.MIGRATIONS.length)
-    migrations.MIGRATIONS.push({
-      id: "20231211105810_new-test",
-      func: async () => {},
-    })
-    migrations.MIGRATIONS.push({
-      id: "20231211105812_new-test",
-      func: async () => {},
-    })
-    migrations.MIGRATIONS.push({
-      id: "20231211105814_new-test",
-      func: async () => {},
-    })
+  it("should disable all migrations after one that is disabled", () => {
+    const MIGRATION_ID1 = "20231211105810_new-test",
+      MIGRATION_ID2 = "20231211105812_new-test",
+      MIGRATION_ID3 = "20231211105814_new-test"
+    // create some migrations to test with
+    const migrations: AppMigration[] = [
+      {
+        id: MIGRATION_ID1,
+        func: async () => {},
+      },
+      {
+        id: MIGRATION_ID2,
+        func: async () => {},
+      },
+      {
+        id: MIGRATION_ID3,
+        func: async () => {},
+      },
+    ]
 
-    expect(getLatestEnabledMigrationId()).toBe("20231211105814_new-test")
-
-    migrations.MIGRATIONS[1].disabled = true
-    expect(getLatestEnabledMigrationId()).toBe("20231211105810_new-test")
+    expect(getLatestEnabledMigrationId(migrations)).toBe(MIGRATION_ID3)
+    migrations[1].disabled = true
+    expect(getLatestEnabledMigrationId(migrations)).toBe(MIGRATION_ID1)
   })
 })
diff --git a/packages/server/src/middleware/appMigrations.ts b/packages/server/src/middleware/appMigrations.ts
index a8ea886ab2..fdaeb3a03d 100644
--- a/packages/server/src/middleware/appMigrations.ts
+++ b/packages/server/src/middleware/appMigrations.ts
@@ -5,8 +5,8 @@ import { env } from "@budibase/backend-core"
 export default async (ctx: UserCtx, next: any) => {
   const { appId } = ctx
 
-  // migrations can be disabled via environment variable, or can be disabled
-  // due to some of the migrations not being ready to run - disables all migrations
+  // migrations can be disabled via environment variable if you
+  // need to completely disable migrations, e.g. for testing
   if (env.SKIP_APP_MIGRATIONS) {
     return next()
   }
diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts
index 6e0e6e7496..4819b9d8d5 100644
--- a/packages/server/src/sdk/app/tables/internal/sqs.ts
+++ b/packages/server/src/sdk/app/tables/internal/sqs.ts
@@ -109,8 +109,12 @@ async function buildBaseDefinition(): Promise<PreSaveSQLiteDefinition> {
 export async function syncDefinition(): Promise<void> {
   const db = context.getAppDB()
   let existing: SQLiteDefinition | undefined
-  if (await db.exists(SQLITE_DESIGN_DOC_ID)) {
+  try {
     existing = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
+  } catch (err: any) {
+    if (err.status !== 404) {
+      throw err
+    }
   }
   const definition = await buildBaseDefinition()
   if (existing) {

From 1124ee1cc8adbc6111ccbbe625101b794ca14f3c Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 6 Jun 2024 17:20:25 +0100
Subject: [PATCH 20/23] Adding a test case for the new migration.

---
 .../migrations/20240604153647_initial_sqs.ts  |   8 +-
 .../tests/20240604153647_initial_sqs.spec.ts  | 116 ++++++++++++++++++
 packages/server/src/environment.ts            |   1 +
 .../server/src/middleware/appMigrations.ts    |   4 +-
 4 files changed, 124 insertions(+), 5 deletions(-)
 create mode 100644 packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts

diff --git a/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts
index 05174b8cf1..6ad3c09c3c 100644
--- a/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts
+++ b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts
@@ -17,15 +17,17 @@ const migration = async () => {
 
     // it already has the junction table ID - no need to migrate
     if (!linkDoc.tableId) {
-      linkDoc.tableId = new LinkDocumentImpl(
+      const newLink = new LinkDocumentImpl(
         linkDoc.doc1.tableId,
         linkDoc.doc1.fieldName,
         linkDoc.doc1.rowId,
         linkDoc.doc2.tableId,
         linkDoc.doc2.fieldName,
         linkDoc.doc2.rowId
-      ).tableId
-      docsToUpdate.push(linkDoc)
+      )
+      newLink._id = linkDoc._id!
+      newLink._rev = linkDoc._rev
+      docsToUpdate.push(newLink)
     }
   }
 
diff --git a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts
new file mode 100644
index 0000000000..64420d239f
--- /dev/null
+++ b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts
@@ -0,0 +1,116 @@
+import * as setup from "../../../api/routes/tests/utilities"
+import { basicTable } from "../../../tests/utilities/structures"
+import {
+  db as dbCore,
+  SQLITE_DESIGN_DOC_ID,
+  context,
+} from "@budibase/backend-core"
+import {
+  LinkDocument,
+  DocumentType,
+  SQLiteDefinition,
+  SQLiteType,
+} from "@budibase/types"
+import {
+  generateJunctionTableID,
+  generateLinkID,
+  generateRowID,
+} from "../../../db/utils"
+import migration from "../20240604153647_initial_sqs"
+
+const config = setup.getConfig()
+let tableId: string
+
+function oldLinkDocInfo() {
+  const tableId1 = `${DocumentType.TABLE}_a`,
+    tableId2 = `${DocumentType.TABLE}_b`
+  return {
+    tableId1,
+    tableId2,
+    rowId1: generateRowID(tableId1, "b"),
+    rowId2: generateRowID(tableId2, "a"),
+    col1: "columnB",
+    col2: "columnA",
+  }
+}
+
+function oldLinkDocID() {
+  const { tableId1, tableId2, rowId1, rowId2, col1, col2 } = oldLinkDocInfo()
+  return generateLinkID(tableId1, tableId2, rowId1, rowId2, col1, col2)
+}
+
+function oldLinkDocument(): Omit<LinkDocument, "tableId"> {
+  const { tableId1, tableId2, rowId1, rowId2, col1, col2 } = oldLinkDocInfo()
+  return {
+    type: "link",
+    _id: oldLinkDocID(),
+    doc1: {
+      tableId: tableId1,
+      fieldName: col1,
+      rowId: rowId1,
+    },
+    doc2: {
+      tableId: tableId2,
+      fieldName: col2,
+      rowId: rowId2,
+    },
+  }
+}
+
+async function sqsDisabled(cb: () => Promise<void>) {
+  await config.withEnv({ SQS_SEARCH_ENABLE: "" }, cb)
+}
+
+async function sqsEnabled(cb: () => Promise<void>) {
+  await config.withEnv({ SQS_SEARCH_ENABLE: "1" }, cb)
+}
+
+beforeAll(async () => {
+  await sqsDisabled(async () => {
+    await config.init()
+    const table = await config.api.table.save(basicTable())
+    tableId = table._id!
+    const db = dbCore.getDB(config.appId!)
+    // old link document
+    await db.put(oldLinkDocument())
+  })
+})
+
+describe("SQS migration", () => {
+  it("test migration runs as expected against an older DB", async () => {
+    const db = dbCore.getDB(config.appId!)
+    // confirm nothing exists initially
+    await sqsDisabled(async () => {
+      let error: any | undefined
+      try {
+        await db.get(SQLITE_DESIGN_DOC_ID)
+      } catch (err: any) {
+        error = err
+      }
+      expect(error).toBeDefined()
+      expect(error.status).toBe(404)
+    })
+    await sqsEnabled(async () => {
+      await context.doInAppContext(config.appId!, async () => {
+        await migration()
+      })
+      const designDoc = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
+      expect(designDoc.sql.tables).toBeDefined()
+      const mainTableDef = designDoc.sql.tables[tableId]
+      expect(mainTableDef).toBeDefined()
+      expect(mainTableDef.fields.name).toEqual(SQLiteType.TEXT)
+      expect(mainTableDef.fields.description).toEqual(SQLiteType.TEXT)
+
+      const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo()
+      const linkDoc = await db.get<LinkDocument>(oldLinkDocID())
+      expect(linkDoc.tableId).toEqual(
+        generateJunctionTableID(tableId1, tableId2)
+      )
+      // should have swapped the documents
+      expect(linkDoc.doc1.tableId).toEqual(tableId2)
+      expect(linkDoc.doc1.rowId).toEqual(rowId2)
+      expect(linkDoc.doc2.tableId).toEqual(tableId1)
+      expect(linkDoc.doc2.rowId).toEqual(rowId1)
+    })
+  })
+})
diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts
index 2c91f1cb48..b44d7547a2 100644
--- a/packages/server/src/environment.ts
+++ b/packages/server/src/environment.ts
@@ -96,6 +96,7 @@ const environment = {
   DISABLE_THREADING: process.env.DISABLE_THREADING,
   DISABLE_AUTOMATION_LOGS: process.env.DISABLE_AUTOMATION_LOGS,
   DISABLE_RATE_LIMITING: process.env.DISABLE_RATE_LIMITING,
+  DISABLE_APP_MIGRATIONS: process.env.SKIP_APP_MIGRATIONS || false,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   SELF_HOSTED: process.env.SELF_HOSTED,
diff --git a/packages/server/src/middleware/appMigrations.ts b/packages/server/src/middleware/appMigrations.ts
index fdaeb3a03d..6ad356427b 100644
--- a/packages/server/src/middleware/appMigrations.ts
+++ b/packages/server/src/middleware/appMigrations.ts
@@ -1,13 +1,13 @@
 import { UserCtx } from "@budibase/types"
 import { checkMissingMigrations } from "../appMigrations"
-import { env } from "@budibase/backend-core"
+import env from "../environment"
 
 export default async (ctx: UserCtx, next: any) => {
   const { appId } = ctx
 
   // migrations can be disabled via environment variable if you
   // need to completely disable migrations, e.g. for testing
-  if (env.SKIP_APP_MIGRATIONS) {
+  if (env.DISABLE_APP_MIGRATIONS) {
     return next()
   }
 

From a4e646fc64eae71ddff60c5043236a400b67ddbb Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 6 Jun 2024 17:26:37 +0100
Subject: [PATCH 21/23] Changing how latest migration is determined.

---
 packages/server/src/appMigrations/index.ts | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts
index cac1e82716..8bd3ae7425 100644
--- a/packages/server/src/appMigrations/index.ts
+++ b/packages/server/src/appMigrations/index.ts
@@ -15,18 +15,15 @@ export type AppMigration = {
 }
 
 export function getLatestEnabledMigrationId(migrations?: AppMigration[]) {
-  const enabledMigrations: AppMigration[] = []
+  let latestMigrationId: string | undefined
   for (let migration of migrations || MIGRATIONS) {
     // if a migration is disabled, all migrations after it are disabled
     if (migration.disabled) {
       break
     }
-    enabledMigrations.push(migration)
+    latestMigrationId = migration.id
   }
-  return enabledMigrations
-    .map(m => m.id)
-    .sort()
-    .reverse()[0]
+  return latestMigrationId
 }
 
 function getTimestamp(versionId: string) {
@@ -41,7 +38,10 @@ export async function checkMissingMigrations(
   const currentVersion = await getAppMigrationVersion(appId)
   const latestMigration = getLatestEnabledMigrationId()
 
-  if (getTimestamp(currentVersion) < getTimestamp(latestMigration)) {
+  if (
+    latestMigration &&
+    getTimestamp(currentVersion) < getTimestamp(latestMigration)
+  ) {
     await queue.add(
       {
         appId,

From cd300b0453de15b30c5d08bd0ca00bbd35eb9a3d Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 6 Jun 2024 17:27:40 +0100
Subject: [PATCH 22/23] Linting.

---
 .../src/appMigrations/migrations/20240604153647_initial_sqs.ts   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts
index 6ad3c09c3c..800de1418a 100644
--- a/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts
+++ b/packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts
@@ -3,7 +3,6 @@ import { allLinkDocs } from "../../db/utils"
 import LinkDocumentImpl from "../../db/linkedRows/LinkDocument"
 import sdk from "../../sdk"
 import env from "../../environment"
-import { DBError } from "@budibase/types"
 
 const migration = async () => {
   const linkDocs = await allLinkDocs()

From 5537f7b7820306713a520a23e619c9d1910c6a67 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 6 Jun 2024 17:35:35 +0100
Subject: [PATCH 23/23] build fixes.

---
 packages/server/src/api/controllers/application.ts | 13 ++++++++-----
 packages/server/src/api/controllers/migrations.ts  |  4 +++-
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 1555dc8afd..830acc55bf 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -358,11 +358,14 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
       await createApp(appId)
     }
 
-    // Initialise the app migration version as the latest one
-    await appMigrations.updateAppMigrationMetadata({
-      appId,
-      version: appMigrations.getLatestEnabledMigrationId(),
-    })
+    const latestMigrationId = appMigrations.getLatestEnabledMigrationId()
+    if (latestMigrationId) {
+      // Initialise the app migration version as the latest one
+      await appMigrations.updateAppMigrationMetadata({
+        appId,
+        version: latestMigrationId,
+      })
+    }
 
     await cache.app.invalidateAppMetadata(appId, newApplication)
     return newApplication
diff --git a/packages/server/src/api/controllers/migrations.ts b/packages/server/src/api/controllers/migrations.ts
index 97c0ae8712..89255e89b1 100644
--- a/packages/server/src/api/controllers/migrations.ts
+++ b/packages/server/src/api/controllers/migrations.ts
@@ -27,7 +27,9 @@ export async function getMigrationStatus(ctx: Ctx) {
 
   const latestAppliedMigration = await getAppMigrationVersion(appId)
 
-  const migrated = latestAppliedMigration >= getLatestEnabledMigrationId()
+  const latestMigrationId = getLatestEnabledMigrationId()
+  const migrated =
+    !latestMigrationId || latestAppliedMigration >= latestMigrationId
 
   ctx.body = { migrated }
   ctx.status = 200