From c6840960a4d3633d84f7f5566cffe7864964db94 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 27 Nov 2023 11:06:49 +0100
Subject: [PATCH 01/30] Add appMigrationMetadata utils

---
 .../src/appMigrations/appMigrationMetadata.ts | 39 +++++++++++++++++++
 packages/types/src/documents/document.ts      |  1 +
 2 files changed, 40 insertions(+)
 create mode 100644 packages/server/src/appMigrations/appMigrationMetadata.ts

diff --git a/packages/server/src/appMigrations/appMigrationMetadata.ts b/packages/server/src/appMigrations/appMigrationMetadata.ts
new file mode 100644
index 0000000000..e871018245
--- /dev/null
+++ b/packages/server/src/appMigrations/appMigrationMetadata.ts
@@ -0,0 +1,39 @@
+import { Duration, cache, db, env } from "@budibase/backend-core"
+import { Database, App, DocumentType, Document } from "@budibase/types"
+
+export interface AppMigrationDoc extends Document {
+  version: string
+}
+
+const EXPIRY_SECONDS = Duration.fromDays(1).toSeconds()
+
+async function populateFromDB(appId: string) {
+  return db.doWithDB(
+    appId,
+    (db: Database) => {
+      return db.get<App>(DocumentType.APP_MIGRATION_METADATA)
+    },
+    { skip_setup: true }
+  )
+}
+
+export async function getAppMigrationMetadata(appId: string): Promise<string> {
+  const cacheKey = `appmigrations_${env.VERSION}_${appId}`
+
+  let metadata: AppMigrationDoc | undefined = await cache.get(cacheKey)
+  if (!metadata) {
+    try {
+      metadata = await populateFromDB(appId)
+    } catch (err: any) {
+      if (err.status !== 404) {
+        throw err
+      }
+
+      metadata = { version: "" }
+    }
+
+    await cache.store(cacheKey, metadata, EXPIRY_SECONDS)
+  }
+
+  return metadata.version
+}
diff --git a/packages/types/src/documents/document.ts b/packages/types/src/documents/document.ts
index fb9589b24d..18feb9b518 100644
--- a/packages/types/src/documents/document.ts
+++ b/packages/types/src/documents/document.ts
@@ -37,6 +37,7 @@ export enum DocumentType {
   USER_FLAG = "flag",
   AUTOMATION_METADATA = "meta_au",
   AUDIT_LOG = "al",
+  APP_MIGRATION_METADATA = "_design/migrations",
 }
 
 // these are the core documents that make up the data, design

From 75554d1bd29a27f72c2466fde76647e82a0b1519 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 27 Nov 2023 12:25:48 +0100
Subject: [PATCH 02/30] Add migration script

---
 package.json                                  |  3 +-
 .../server/src/appMigrations/migrations.ts    |  3 +
 scripts/add-app-migration.js                  | 69 +++++++++++++++++++
 3 files changed, 74 insertions(+), 1 deletion(-)
 create mode 100644 packages/server/src/appMigrations/migrations.ts
 create mode 100644 scripts/add-app-migration.js

diff --git a/package.json b/package.json
index 2978483448..08f77016fb 100644
--- a/package.json
+++ b/package.json
@@ -79,7 +79,8 @@
     "security:audit": "node scripts/audit.js",
     "postinstall": "husky install",
     "submodules:load": "git submodule init && git submodule update && yarn",
-    "submodules:unload": "git submodule deinit --all && yarn"
+    "submodules:unload": "git submodule deinit --all && yarn",
+    "add-app-migration": "node scripts/add-app-migration.js"
   },
   "workspaces": {
     "packages": [
diff --git a/packages/server/src/appMigrations/migrations.ts b/packages/server/src/appMigrations/migrations.ts
new file mode 100644
index 0000000000..a39f679a17
--- /dev/null
+++ b/packages/server/src/appMigrations/migrations.ts
@@ -0,0 +1,3 @@
+// This file should never be manually modified, use `yarn add-app-migration` in order to add a new one
+
+export const MIGRATIONS: Record<string, { migration: () => Promise<void> }> = {}
diff --git a/scripts/add-app-migration.js b/scripts/add-app-migration.js
new file mode 100644
index 0000000000..8d56701033
--- /dev/null
+++ b/scripts/add-app-migration.js
@@ -0,0 +1,69 @@
+const fs = require("fs")
+const path = require("path")
+
+const generateTimestamp = () => {
+  const now = new Date()
+  const year = now.getFullYear()
+  const month = String(now.getMonth() + 1).padStart(2, "0")
+  const day = String(now.getDate()).padStart(2, "0")
+  const hours = String(now.getHours()).padStart(2, "0")
+  const minutes = String(now.getMinutes()).padStart(2, "0")
+  const seconds = String(now.getSeconds()).padStart(2, "0")
+
+  return `${year}${month}${day}${hours}${minutes}${seconds}`
+}
+
+const createMigrationFile = () => {
+  const timestamp = generateTimestamp()
+  const migrationsDir = "../packages/server/src/appMigrations"
+
+  const template = `const migration = async () => {
+  // Add your migration logic here
+}
+
+export default migration
+`
+
+  const newMigrationPath = path.join(
+    migrationsDir,
+    "migrations",
+    `${timestamp}.ts`
+  )
+  fs.writeFileSync(path.resolve(__dirname, newMigrationPath), template)
+
+  console.log(`New migration created: ${newMigrationPath}`)
+
+  // Append the new migration to the main migrations file
+  const migrationsFilePath = path.join(migrationsDir, "migrations.ts")
+
+  const migrationDir = fs.readdirSync(
+    path.join(__dirname, migrationsDir, "migrations")
+  )
+  const migrations = migrationDir
+    .filter(m => m.endsWith(".ts"))
+    .map(m => m.substring(0, m.length - 3))
+
+  let migrationFileContent =
+    "// This file should never be manually modified, use `yarn add-app-migration` in order to add a new one\n\n"
+
+  for (const migration of migrations) {
+    migrationFileContent += `import m${migration} from "./migrations/${migration}"\n`
+  }
+
+  migrationFileContent += `\nexport const MIGRATIONS: Record<string, { migration: () => Promise<void> }> = {\n`
+
+  for (const migration of migrations) {
+    migrationFileContent += `  [${migration}]: { migration: m${migration} },\n`
+  }
+
+  migrationFileContent += `}\n`
+
+  fs.writeFileSync(
+    path.resolve(__dirname, migrationsFilePath),
+    migrationFileContent
+  )
+
+  console.log(`Main migrations file updated: ${migrationsFilePath}`)
+}
+
+createMigrationFile()

From 8ac9420e5be93b69c8d7990bbe49c9b81a017ae7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 27 Nov 2023 15:40:37 +0100
Subject: [PATCH 03/30] Add middleware to queue

---
 packages/backend-core/src/queue/constants.ts   |  1 +
 packages/backend-core/src/queue/listeners.ts   |  2 ++
 packages/server/src/api/index.ts               |  3 +++
 packages/server/src/appMigrations/index.ts     | 18 ++++++++++++++++++
 packages/server/src/appMigrations/queue.ts     | 13 +++++++++++++
 .../server/src/middleware/appMigrations.ts     | 14 ++++++++++++++
 packages/types/src/sdk/locks.ts                |  1 +
 7 files changed, 52 insertions(+)
 create mode 100644 packages/server/src/appMigrations/index.ts
 create mode 100644 packages/server/src/appMigrations/queue.ts
 create mode 100644 packages/server/src/middleware/appMigrations.ts

diff --git a/packages/backend-core/src/queue/constants.ts b/packages/backend-core/src/queue/constants.ts
index e1ffcfee36..eb4f21aced 100644
--- a/packages/backend-core/src/queue/constants.ts
+++ b/packages/backend-core/src/queue/constants.ts
@@ -3,4 +3,5 @@ export enum JobQueue {
   APP_BACKUP = "appBackupQueue",
   AUDIT_LOG = "auditLogQueue",
   SYSTEM_EVENT_QUEUE = "systemEventQueue",
+  APP_MIGRATION = "appMigration",
 }
diff --git a/packages/backend-core/src/queue/listeners.ts b/packages/backend-core/src/queue/listeners.ts
index 42e3172364..063a01bd2f 100644
--- a/packages/backend-core/src/queue/listeners.ts
+++ b/packages/backend-core/src/queue/listeners.ts
@@ -87,6 +87,7 @@ enum QueueEventType {
   APP_BACKUP_EVENT = "app-backup-event",
   AUDIT_LOG_EVENT = "audit-log-event",
   SYSTEM_EVENT = "system-event",
+  APP_MIGRATION = "app-migration",
 }
 
 const EventTypeMap: { [key in JobQueue]: QueueEventType } = {
@@ -94,6 +95,7 @@ const EventTypeMap: { [key in JobQueue]: QueueEventType } = {
   [JobQueue.APP_BACKUP]: QueueEventType.APP_BACKUP_EVENT,
   [JobQueue.AUDIT_LOG]: QueueEventType.AUDIT_LOG_EVENT,
   [JobQueue.SYSTEM_EVENT_QUEUE]: QueueEventType.SYSTEM_EVENT,
+  [JobQueue.APP_MIGRATION]: QueueEventType.APP_MIGRATION,
 }
 
 function logging(queue: Queue, jobQueue: JobQueue) {
diff --git a/packages/server/src/api/index.ts b/packages/server/src/api/index.ts
index 6221909347..a01e3764f0 100644
--- a/packages/server/src/api/index.ts
+++ b/packages/server/src/api/index.ts
@@ -4,6 +4,7 @@ import currentApp from "../middleware/currentapp"
 import zlib from "zlib"
 import { mainRoutes, staticRoutes, publicRoutes } from "./routes"
 import { middleware as pro } from "@budibase/pro"
+import migrations from "../middleware/appMigrations"
 
 export { shutdown } from "./routes/public"
 const compress = require("koa-compress")
@@ -47,6 +48,8 @@ router
   // @ts-ignore
   .use(currentApp)
   .use(auth.auditLog)
+  // @ts-ignore
+  .use(migrations)
 
 // authenticated routes
 for (let route of mainRoutes) {
diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts
new file mode 100644
index 0000000000..079e3e8c44
--- /dev/null
+++ b/packages/server/src/appMigrations/index.ts
@@ -0,0 +1,18 @@
+import queue from "./queue"
+import { getAppMigrationMetadata } from "./appMigrationMetadata"
+import { MIGRATIONS } from "./migrations"
+
+const latestMigration = Object.keys(MIGRATIONS).sort().reverse()[0]
+
+export async function checkMissingMigrations(appId: string) {
+  const currentVersion = await getAppMigrationMetadata(appId)
+
+  if (currentVersion < latestMigration) {
+    await queue.add(
+      {
+        appId,
+      }
+      // TODO: idempotency
+    )
+  }
+}
diff --git a/packages/server/src/appMigrations/queue.ts b/packages/server/src/appMigrations/queue.ts
new file mode 100644
index 0000000000..d842b97e49
--- /dev/null
+++ b/packages/server/src/appMigrations/queue.ts
@@ -0,0 +1,13 @@
+import { queue } from "@budibase/backend-core"
+import { Job } from "bull"
+
+const appMigrationQueue = queue.createQueue(queue.JobQueue.APP_MIGRATION)
+appMigrationQueue.process(processMessage)
+
+async function processMessage(job: Job) {
+  const { appId } = job.data
+
+  console.log(appId)
+}
+
+export default appMigrationQueue
diff --git a/packages/server/src/middleware/appMigrations.ts b/packages/server/src/middleware/appMigrations.ts
new file mode 100644
index 0000000000..a94b8823e8
--- /dev/null
+++ b/packages/server/src/middleware/appMigrations.ts
@@ -0,0 +1,14 @@
+import { UserCtx } from "@budibase/types"
+import { checkMissingMigrations } from "../appMigrations"
+
+export default async (ctx: UserCtx, next: any) => {
+  const { appId } = ctx
+
+  if (!appId) {
+    return next()
+  }
+
+  await checkMissingMigrations(appId)
+
+  return next()
+}
diff --git a/packages/types/src/sdk/locks.ts b/packages/types/src/sdk/locks.ts
index 6ff91d4315..82a7089b3f 100644
--- a/packages/types/src/sdk/locks.ts
+++ b/packages/types/src/sdk/locks.ts
@@ -20,6 +20,7 @@ export enum LockName {
   UPDATE_TENANTS_DOC = "update_tenants_doc",
   PERSIST_WRITETHROUGH = "persist_writethrough",
   QUOTA_USAGE_EVENT = "quota_usage_event",
+  APP_MIGRATION = "app_migrations",
 }
 
 export type LockOptions = {

From bbcbb58658548f16e49df215d0359379f3aecf3d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 27 Nov 2023 15:57:15 +0100
Subject: [PATCH 04/30] Add gitkeep

---
 packages/server/src/appMigrations/migrations/.gitkeep | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 packages/server/src/appMigrations/migrations/.gitkeep

diff --git a/packages/server/src/appMigrations/migrations/.gitkeep b/packages/server/src/appMigrations/migrations/.gitkeep
new file mode 100644
index 0000000000..e69de29bb2

From 45fcf2c14315d994e27e66652f2b88d302e75aa5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 27 Nov 2023 15:59:23 +0100
Subject: [PATCH 05/30] Do not use cache for dev

---
 packages/server/src/appMigrations/appMigrationMetadata.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/appMigrations/appMigrationMetadata.ts b/packages/server/src/appMigrations/appMigrationMetadata.ts
index e871018245..6ffac8e41c 100644
--- a/packages/server/src/appMigrations/appMigrationMetadata.ts
+++ b/packages/server/src/appMigrations/appMigrationMetadata.ts
@@ -21,7 +21,7 @@ export async function getAppMigrationMetadata(appId: string): Promise<string> {
   const cacheKey = `appmigrations_${env.VERSION}_${appId}`
 
   let metadata: AppMigrationDoc | undefined = await cache.get(cacheKey)
-  if (!metadata) {
+  if (!metadata || env.isDev()) {
     try {
       metadata = await populateFromDB(appId)
     } catch (err: any) {

From f2fcf0f6c2f9ad55bfb031d1efff4df454fd5659 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 27 Nov 2023 16:23:57 +0100
Subject: [PATCH 06/30] Idempotency

---
 packages/server/src/appMigrations/index.ts | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts
index 079e3e8c44..ba91d20594 100644
--- a/packages/server/src/appMigrations/index.ts
+++ b/packages/server/src/appMigrations/index.ts
@@ -11,8 +11,12 @@ export async function checkMissingMigrations(appId: string) {
     await queue.add(
       {
         appId,
+      },
+      {
+        jobId: appId,
+        removeOnComplete: true,
+        removeOnFail: true,
       }
-      // TODO: idempotency
     )
   }
 }

From 1d124a59cbc9ba50ef2d9c35cf7f8faf249a57e6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 29 Nov 2023 11:03:33 +0100
Subject: [PATCH 07/30] Process migration

---
 .../src/appMigrations/appMigrationMetadata.ts | 30 +++++++++---
 packages/server/src/appMigrations/index.ts    |  5 +-
 packages/server/src/appMigrations/queue.ts    | 47 ++++++++++++++++++-
 3 files changed, 72 insertions(+), 10 deletions(-)

diff --git a/packages/server/src/appMigrations/appMigrationMetadata.ts b/packages/server/src/appMigrations/appMigrationMetadata.ts
index 6ffac8e41c..828bca31f4 100644
--- a/packages/server/src/appMigrations/appMigrationMetadata.ts
+++ b/packages/server/src/appMigrations/appMigrationMetadata.ts
@@ -1,5 +1,5 @@
-import { Duration, cache, db, env } from "@budibase/backend-core"
-import { Database, App, DocumentType, Document } from "@budibase/types"
+import { Duration, cache, context, db, env } from "@budibase/backend-core"
+import { Database, DocumentType, Document } from "@budibase/types"
 
 export interface AppMigrationDoc extends Document {
   version: string
@@ -7,23 +7,25 @@ export interface AppMigrationDoc extends Document {
 
 const EXPIRY_SECONDS = Duration.fromDays(1).toSeconds()
 
-async function populateFromDB(appId: string) {
+async function getFromDB(appId: string) {
   return db.doWithDB(
     appId,
     (db: Database) => {
-      return db.get<App>(DocumentType.APP_MIGRATION_METADATA)
+      return db.get<AppMigrationDoc>(DocumentType.APP_MIGRATION_METADATA)
     },
     { skip_setup: true }
   )
 }
 
+const getCacheKey = (appId: string) => `appmigrations_${env.VERSION}_${appId}`
+
 export async function getAppMigrationMetadata(appId: string): Promise<string> {
-  const cacheKey = `appmigrations_${env.VERSION}_${appId}`
+  const cacheKey = getCacheKey(appId)
 
   let metadata: AppMigrationDoc | undefined = await cache.get(cacheKey)
   if (!metadata || env.isDev()) {
     try {
-      metadata = await populateFromDB(appId)
+      metadata = await getFromDB(appId)
     } catch (err: any) {
       if (err.status !== 404) {
         throw err
@@ -37,3 +39,19 @@ export async function getAppMigrationMetadata(appId: string): Promise<string> {
 
   return metadata.version
 }
+
+export async function updateAppMigrationMetadata({
+  appId,
+  version,
+}: {
+  appId: string
+  version: string
+}): Promise<void> {
+  const db = context.getAppDB()
+  const appMigrationDoc = await getFromDB(appId)
+  await db.put({ ...appMigrationDoc, version })
+
+  const cacheKey = getCacheKey(appId)
+
+  await cache.destroy(cacheKey)
+}
diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts
index ba91d20594..419d4c457a 100644
--- a/packages/server/src/appMigrations/index.ts
+++ b/packages/server/src/appMigrations/index.ts
@@ -1,4 +1,4 @@
-import queue from "./queue"
+import queue, { PROCESS_MIGRATION_TIMEOUT } from "./queue"
 import { getAppMigrationMetadata } from "./appMigrationMetadata"
 import { MIGRATIONS } from "./migrations"
 
@@ -13,9 +13,10 @@ export async function checkMissingMigrations(appId: string) {
         appId,
       },
       {
-        jobId: appId,
+        jobId: `${appId}_${latestMigration}`,
         removeOnComplete: true,
         removeOnFail: true,
+        timeout: PROCESS_MIGRATION_TIMEOUT,
       }
     )
   }
diff --git a/packages/server/src/appMigrations/queue.ts b/packages/server/src/appMigrations/queue.ts
index d842b97e49..dcbd7e60db 100644
--- a/packages/server/src/appMigrations/queue.ts
+++ b/packages/server/src/appMigrations/queue.ts
@@ -1,13 +1,56 @@
-import { queue } from "@budibase/backend-core"
+import { context, locks, queue } from "@budibase/backend-core"
+import { LockName, LockType } from "@budibase/types"
 import { Job } from "bull"
+import { MIGRATIONS } from "./migrations"
+import {
+  getAppMigrationMetadata,
+  updateAppMigrationMetadata,
+} from "./appMigrationMetadata"
 
 const appMigrationQueue = queue.createQueue(queue.JobQueue.APP_MIGRATION)
 appMigrationQueue.process(processMessage)
 
+export async function runMigration(migrationId: string) {
+  await MIGRATIONS[migrationId].migration()
+}
+
+// TODO
+export const PROCESS_MIGRATION_TIMEOUT = 30000
+
 async function processMessage(job: Job) {
   const { appId } = job.data
+  console.log(`Processing app migration for "${appId}"`)
 
-  console.log(appId)
+  await locks.doWithLock(
+    {
+      name: LockName.APP_MIGRATION,
+      type: LockType.DEFAULT,
+      resource: appId,
+      ttl: PROCESS_MIGRATION_TIMEOUT,
+    },
+    async () => {
+      await context.doInAppContext(appId, async () => {
+        const currentVersion = await getAppMigrationMetadata(appId)
+
+        const pendingMigrations = Object.keys(MIGRATIONS).filter(
+          m => m > currentVersion
+        )
+
+        let index = 0
+        for (const migration of pendingMigrations) {
+          const counter = `(${++index}/${pendingMigrations.length})`
+          console.info(`Running migration ${migration}... ${counter}`, {
+            migration,
+            appId,
+          })
+          await runMigration(migration)
+          await updateAppMigrationMetadata({ appId, version: migration })
+        }
+      })
+    }
+  )
+
+  console.log(`App migration for "${appId}" processed`)
 }
 
 export default appMigrationQueue

From 25c16ae22923a4372c7a243ffd53867b500a83c8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 29 Nov 2023 11:16:10 +0100
Subject: [PATCH 08/30] Add migration integrity tests

---
 .../appMigrations/tests/migrations.spec.ts    | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 packages/server/src/appMigrations/tests/migrations.spec.ts

diff --git a/packages/server/src/appMigrations/tests/migrations.spec.ts b/packages/server/src/appMigrations/tests/migrations.spec.ts
new file mode 100644
index 0000000000..c2fddc2766
--- /dev/null
+++ b/packages/server/src/appMigrations/tests/migrations.spec.ts
@@ -0,0 +1,26 @@
+import { context } from "@budibase/backend-core"
+import * as setup from "../../api/routes/tests/utilities"
+import { MIGRATIONS } from "../migrations"
+import { runMigration } from "../queue"
+
+describe("migration", () => {
+  it("each migration can rerun safely", async () => {
+    const config = setup.getConfig()
+    await config.init()
+
+    const migrations = Object.keys(MIGRATIONS)
+
+    await config.doInContext(config.getAppId(), async () => {
+      const db = context.getAppDB()
+      for (const migration of migrations) {
+        await runMigration(migration)
+        const docs = await db.allDocs({ include_docs: true })
+
+        await runMigration(migration)
+        const latestDocs = await db.allDocs({ include_docs: true })
+
+        expect(docs).toEqual(latestDocs)
+      }
+    })
+  })
+})

From 98702798fb8badcdb3e615422c0143942cbb9c96 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 29 Nov 2023 11:26:41 +0100
Subject: [PATCH 09/30] Allow custom timeout

---
 packages/server/src/appMigrations/queue.ts | 4 +++-
 packages/server/src/environment.ts         | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/appMigrations/queue.ts b/packages/server/src/appMigrations/queue.ts
index dcbd7e60db..f783b88e2a 100644
--- a/packages/server/src/appMigrations/queue.ts
+++ b/packages/server/src/appMigrations/queue.ts
@@ -6,6 +6,7 @@ import {
   getAppMigrationMetadata,
   updateAppMigrationMetadata,
 } from "./appMigrationMetadata"
+import environment from "../environment"
 
 const appMigrationQueue = queue.createQueue(queue.JobQueue.APP_MIGRATION)
 appMigrationQueue.process(processMessage)
@@ -15,7 +16,8 @@ export async function runMigration(migrationId: string) {
 }
 
 // TODO
-export const PROCESS_MIGRATION_TIMEOUT = 30000
+export const PROCESS_MIGRATION_TIMEOUT =
+  environment.APP_MIGRATION_TIMEOUT || 60000
 
 async function processMessage(job: Job) {
   const { appId } = job.data
diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts
index e7eea5f0b6..d549cdf219 100644
--- a/packages/server/src/environment.ts
+++ b/packages/server/src/environment.ts
@@ -87,6 +87,7 @@ const environment = {
   },
   TOP_LEVEL_PATH:
     process.env.TOP_LEVEL_PATH || process.env.SERVER_TOP_LEVEL_PATH,
+  APP_MIGRATION_TIMEOUT: parseIntSafe(process.env.APP_MIGRATION_TIMEOUT),
 }
 
 // threading can cause memory issues with node-ts in development

From a4fd4ef6351d81df38ebae4bc64535259b89e3d9 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 29 Nov 2023 12:28:13 +0100
Subject: [PATCH 10/30] Save history

---
 .../src/appMigrations/appMigrationMetadata.ts       | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/appMigrations/appMigrationMetadata.ts b/packages/server/src/appMigrations/appMigrationMetadata.ts
index 828bca31f4..1df0b266ae 100644
--- a/packages/server/src/appMigrations/appMigrationMetadata.ts
+++ b/packages/server/src/appMigrations/appMigrationMetadata.ts
@@ -3,6 +3,7 @@ import { Database, DocumentType, Document } from "@budibase/types"
 
 export interface AppMigrationDoc extends Document {
   version: string
+  history: Record<string, { runAt: number }>
 }
 
 const EXPIRY_SECONDS = Duration.fromDays(1).toSeconds()
@@ -31,7 +32,7 @@ export async function getAppMigrationMetadata(appId: string): Promise<string> {
         throw err
       }
 
-      metadata = { version: "" }
+      metadata = { version: "", history: {} }
     }
 
     await cache.store(cacheKey, metadata, EXPIRY_SECONDS)
@@ -49,7 +50,15 @@ export async function updateAppMigrationMetadata({
 }): Promise<void> {
   const db = context.getAppDB()
   const appMigrationDoc = await getFromDB(appId)
-  await db.put({ ...appMigrationDoc, version })
+  const updatedMigrationDoc: AppMigrationDoc = {
+    ...appMigrationDoc,
+    version,
+    history: {
+      ...appMigrationDoc.history,
+      [version]: { runAt: Date.now() },
+    },
+  }
+  await db.put(updatedMigrationDoc)
 
   const cacheKey = getCacheKey(appId)
 

From 3ee59b0e965e45f869ba71ab5270080ccd73cb55 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 29 Nov 2023 12:34:50 +0100
Subject: [PATCH 11/30] Migrations in a queue

---
 packages/server/src/appMigrations/migrations.ts   |  7 ++++++-
 packages/server/src/appMigrations/queue.ts        | 15 +++++++--------
 .../src/appMigrations/tests/migrations.spec.ts    |  9 +++------
 3 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/packages/server/src/appMigrations/migrations.ts b/packages/server/src/appMigrations/migrations.ts
index a39f679a17..893881e716 100644
--- a/packages/server/src/appMigrations/migrations.ts
+++ b/packages/server/src/appMigrations/migrations.ts
@@ -1,3 +1,8 @@
 // This file should never be manually modified, use `yarn add-app-migration` in order to add a new one
 
-export const MIGRATIONS: Record<string, { migration: () => Promise<void> }> = {}
+export const MIGRATIONS: {
+  migrationId: string
+  migrationFunc: () => Promise<void>
+}[] = [
+  // Migrations will be executed sorted by migrationId
+]
diff --git a/packages/server/src/appMigrations/queue.ts b/packages/server/src/appMigrations/queue.ts
index f783b88e2a..8d05a8b89a 100644
--- a/packages/server/src/appMigrations/queue.ts
+++ b/packages/server/src/appMigrations/queue.ts
@@ -11,10 +11,6 @@ import environment from "../environment"
 const appMigrationQueue = queue.createQueue(queue.JobQueue.APP_MIGRATION)
 appMigrationQueue.process(processMessage)
 
-export async function runMigration(migrationId: string) {
-  await MIGRATIONS[migrationId].migration()
-}
-
 // TODO
 export const PROCESS_MIGRATION_TIMEOUT =
   environment.APP_MIGRATION_TIMEOUT || 60000
@@ -34,8 +30,8 @@ async function processMessage(job: Job) {
       await context.doInAppContext(appId, async () => {
         const currentVersion = await getAppMigrationMetadata(appId)
 
-        const pendingMigrations = Object.keys(MIGRATIONS).filter(
-          m => m > currentVersion
+        const pendingMigrations = MIGRATIONS.filter(
+          m => m.migrationId > currentVersion
         )
 
         let index = 0
@@ -45,8 +41,11 @@ async function processMessage(job: Job) {
             migration,
             appId,
           })
-          await runMigration(migration)
-          await updateAppMigrationMetadata({ appId, version: migration })
+          await migration.migrationFunc()
+          await updateAppMigrationMetadata({
+            appId,
+            version: migration.migrationId,
+          })
         }
       })
     }
diff --git a/packages/server/src/appMigrations/tests/migrations.spec.ts b/packages/server/src/appMigrations/tests/migrations.spec.ts
index c2fddc2766..a5ceb05d69 100644
--- a/packages/server/src/appMigrations/tests/migrations.spec.ts
+++ b/packages/server/src/appMigrations/tests/migrations.spec.ts
@@ -1,22 +1,19 @@
 import { context } from "@budibase/backend-core"
 import * as setup from "../../api/routes/tests/utilities"
 import { MIGRATIONS } from "../migrations"
-import { runMigration } from "../queue"
 
 describe("migration", () => {
   it("each migration can rerun safely", async () => {
     const config = setup.getConfig()
     await config.init()
 
-    const migrations = Object.keys(MIGRATIONS)
-
     await config.doInContext(config.getAppId(), async () => {
       const db = context.getAppDB()
-      for (const migration of migrations) {
-        await runMigration(migration)
+      for (const migration of MIGRATIONS) {
+        await migration.migrationFunc()
         const docs = await db.allDocs({ include_docs: true })
 
-        await runMigration(migration)
+        await migration.migrationFunc()
         const latestDocs = await db.allDocs({ include_docs: true })
 
         expect(docs).toEqual(latestDocs)

From 63339eb686deb46e6806c0462c2ed00bc56ee494 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 29 Nov 2023 13:28:52 +0100
Subject: [PATCH 12/30] Run as array

---
 .../src/appMigrations/appMigrationMetadata.ts | 51 ++++++++++++++-----
 packages/server/src/appMigrations/index.ts    | 11 ++--
 packages/server/src/appMigrations/queue.ts    | 26 +++++++---
 scripts/add-app-migration.js                  | 13 +++--
 4 files changed, 71 insertions(+), 30 deletions(-)

diff --git a/packages/server/src/appMigrations/appMigrationMetadata.ts b/packages/server/src/appMigrations/appMigrationMetadata.ts
index 1df0b266ae..97d3623d82 100644
--- a/packages/server/src/appMigrations/appMigrationMetadata.ts
+++ b/packages/server/src/appMigrations/appMigrationMetadata.ts
@@ -20,25 +20,30 @@ async function getFromDB(appId: string) {
 
 const getCacheKey = (appId: string) => `appmigrations_${env.VERSION}_${appId}`
 
-export async function getAppMigrationMetadata(appId: string): Promise<string> {
+export async function getAppMigrationVersion(appId: string): Promise<string> {
   const cacheKey = getCacheKey(appId)
 
   let metadata: AppMigrationDoc | undefined = await cache.get(cacheKey)
-  if (!metadata || env.isDev()) {
-    try {
-      metadata = await getFromDB(appId)
-    } catch (err: any) {
-      if (err.status !== 404) {
-        throw err
-      }
 
-      metadata = { version: "", history: {} }
-    }
-
-    await cache.store(cacheKey, metadata, EXPIRY_SECONDS)
+  if (metadata && !env.isDev()) {
+    return metadata.version
   }
 
-  return metadata.version
+  let version
+  try {
+    metadata = await getFromDB(appId)
+    version = metadata.version
+  } catch (err: any) {
+    if (err.status !== 404) {
+      throw err
+    }
+
+    version = ""
+  }
+
+  await cache.store(cacheKey, version, EXPIRY_SECONDS)
+
+  return version
 }
 
 export async function updateAppMigrationMetadata({
@@ -49,7 +54,25 @@ export async function updateAppMigrationMetadata({
   version: string
 }): Promise<void> {
   const db = context.getAppDB()
-  const appMigrationDoc = await getFromDB(appId)
+
+  let appMigrationDoc: AppMigrationDoc
+
+  try {
+    appMigrationDoc = await getFromDB(appId)
+  } catch (err: any) {
+    if (err.status !== 404) {
+      throw err
+    }
+
+    appMigrationDoc = {
+      _id: DocumentType.APP_MIGRATION_METADATA,
+      version: "",
+      history: {},
+    }
+    await db.put(appMigrationDoc)
+    appMigrationDoc = await getFromDB(appId)
+  }
+
   const updatedMigrationDoc: AppMigrationDoc = {
     ...appMigrationDoc,
     version,
diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts
index 419d4c457a..1226333d96 100644
--- a/packages/server/src/appMigrations/index.ts
+++ b/packages/server/src/appMigrations/index.ts
@@ -1,11 +1,13 @@
-import queue, { PROCESS_MIGRATION_TIMEOUT } from "./queue"
-import { getAppMigrationMetadata } from "./appMigrationMetadata"
+import queue from "./queue"
+import { getAppMigrationVersion } from "./appMigrationMetadata"
 import { MIGRATIONS } from "./migrations"
 
-const latestMigration = Object.keys(MIGRATIONS).sort().reverse()[0]
+const latestMigration = MIGRATIONS.map(m => m.migrationId)
+  .sort()
+  .reverse()[0]
 
 export async function checkMissingMigrations(appId: string) {
-  const currentVersion = await getAppMigrationMetadata(appId)
+  const currentVersion = await getAppMigrationVersion(appId)
 
   if (currentVersion < latestMigration) {
     await queue.add(
@@ -16,7 +18,6 @@ export async function checkMissingMigrations(appId: string) {
         jobId: `${appId}_${latestMigration}`,
         removeOnComplete: true,
         removeOnFail: true,
-        timeout: PROCESS_MIGRATION_TIMEOUT,
       }
     )
   }
diff --git a/packages/server/src/appMigrations/queue.ts b/packages/server/src/appMigrations/queue.ts
index 8d05a8b89a..eae354ff50 100644
--- a/packages/server/src/appMigrations/queue.ts
+++ b/packages/server/src/appMigrations/queue.ts
@@ -3,7 +3,7 @@ import { LockName, LockType } from "@budibase/types"
 import { Job } from "bull"
 import { MIGRATIONS } from "./migrations"
 import {
-  getAppMigrationMetadata,
+  getAppMigrationVersion,
   updateAppMigrationMetadata,
 } from "./appMigrationMetadata"
 import environment from "../environment"
@@ -28,24 +28,34 @@ async function processMessage(job: Job) {
     },
     async () => {
       await context.doInAppContext(appId, async () => {
-        const currentVersion = await getAppMigrationMetadata(appId)
+        let currentVersion = await getAppMigrationVersion(appId)
 
         const pendingMigrations = MIGRATIONS.filter(
           m => m.migrationId > currentVersion
-        )
+        ).sort((a, b) => a.migrationId.localeCompare(b.migrationId))
+
+        const migrationIds = MIGRATIONS.map(m => m.migrationId).sort()
 
         let index = 0
-        for (const migration of pendingMigrations) {
+        for (const { migrationId, migrationFunc } of pendingMigrations) {
+          const expectedMigration =
+            migrationIds[migrationIds.indexOf(currentVersion) + 1]
+
+          if (expectedMigration !== migrationId) {
+            throw `Migration ${migrationId} could not run, update for "${migrationId}" is running but ${expectedMigration} is expected`
+          }
+
           const counter = `(${++index}/${pendingMigrations.length})`
-          console.info(`Running migration ${migration}... ${counter}`, {
-            migration,
+          console.info(`Running migration ${migrationId}... ${counter}`, {
+            migrationId,
             appId,
           })
-          await migration.migrationFunc()
+          await migrationFunc()
           await updateAppMigrationMetadata({
             appId,
-            version: migration.migrationId,
+            version: migrationId,
           })
+          currentVersion = migrationId
         }
       })
     }
diff --git a/scripts/add-app-migration.js b/scripts/add-app-migration.js
index 8d56701033..d7af7a5d79 100644
--- a/scripts/add-app-migration.js
+++ b/scripts/add-app-migration.js
@@ -50,13 +50,20 @@ export default migration
     migrationFileContent += `import m${migration} from "./migrations/${migration}"\n`
   }
 
-  migrationFileContent += `\nexport const MIGRATIONS: Record<string, { migration: () => Promise<void> }> = {\n`
+  migrationFileContent += `\nexport const MIGRATIONS: {
+  migrationId: string
+  migrationFunc: () => Promise<void>
+}[] = [
+  // Migrations will be executed sorted by migrationId\n`
 
   for (const migration of migrations) {
-    migrationFileContent += `  [${migration}]: { migration: m${migration} },\n`
+    migrationFileContent += `  {
+    migrationId: "${migration}",
+    migrationFunc: m${migration}
+  },\n`
   }
 
-  migrationFileContent += `}\n`
+  migrationFileContent += `]\n`
 
   fs.writeFileSync(
     path.resolve(__dirname, migrationsFilePath),

From 91b293fd497a4ef14cefdd335211dda7d86a9d9a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 30 Nov 2023 12:22:05 +0100
Subject: [PATCH 13/30] Use new autoextend

---
 packages/server/src/appMigrations/queue.ts | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/appMigrations/queue.ts b/packages/server/src/appMigrations/queue.ts
index eae354ff50..3a364fe0f9 100644
--- a/packages/server/src/appMigrations/queue.ts
+++ b/packages/server/src/appMigrations/queue.ts
@@ -11,10 +11,6 @@ import environment from "../environment"
 const appMigrationQueue = queue.createQueue(queue.JobQueue.APP_MIGRATION)
 appMigrationQueue.process(processMessage)
 
-// TODO
-export const PROCESS_MIGRATION_TIMEOUT =
-  environment.APP_MIGRATION_TIMEOUT || 60000
-
 async function processMessage(job: Job) {
   const { appId } = job.data
   console.log(`Processing app migration for "${appId}"`)
@@ -22,9 +18,9 @@ async function processMessage(job: Job) {
   await locks.doWithLock(
     {
       name: LockName.APP_MIGRATION,
-      type: LockType.DEFAULT,
+      type: LockType.AUTO_EXTEND,
       resource: appId,
-      ttl: PROCESS_MIGRATION_TIMEOUT,
+      ttl: 60000,
     },
     async () => {
       await context.doInAppContext(appId, async () => {

From d634ff2edb81f75b69b73675ef3198b605968f69 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 4 Dec 2023 16:30:00 +0100
Subject: [PATCH 14/30] Remove unexpected ttl

---
 packages/server/src/appMigrations/queue.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/appMigrations/queue.ts b/packages/server/src/appMigrations/queue.ts
index 3a364fe0f9..4bc9e870cd 100644
--- a/packages/server/src/appMigrations/queue.ts
+++ b/packages/server/src/appMigrations/queue.ts
@@ -20,7 +20,6 @@ async function processMessage(job: Job) {
       name: LockName.APP_MIGRATION,
       type: LockType.AUTO_EXTEND,
       resource: appId,
-      ttl: 60000,
     },
     async () => {
       await context.doInAppContext(appId, async () => {

From e0d8e4c67102ba26a9e4bb9d90af9a07c0d6b067 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 4 Dec 2023 16:30:47 +0100
Subject: [PATCH 15/30] Clean

---
 packages/server/src/appMigrations/queue.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/appMigrations/queue.ts b/packages/server/src/appMigrations/queue.ts
index 4bc9e870cd..d1bb9f3829 100644
--- a/packages/server/src/appMigrations/queue.ts
+++ b/packages/server/src/appMigrations/queue.ts
@@ -6,7 +6,6 @@ import {
   getAppMigrationVersion,
   updateAppMigrationMetadata,
 } from "./appMigrationMetadata"
-import environment from "../environment"
 
 const appMigrationQueue = queue.createQueue(queue.JobQueue.APP_MIGRATION)
 appMigrationQueue.process(processMessage)

From 7b05c7eb182698b2fb0c1b25417e057566c62bbd Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 4 Dec 2023 16:39:01 +0100
Subject: [PATCH 16/30] Use migrations context

---
 .../src/appMigrations/migrationsProcessor.ts  | 60 +++++++++++++++++++
 packages/server/src/appMigrations/queue.ts    | 52 +---------------
 2 files changed, 63 insertions(+), 49 deletions(-)
 create mode 100644 packages/server/src/appMigrations/migrationsProcessor.ts

diff --git a/packages/server/src/appMigrations/migrationsProcessor.ts b/packages/server/src/appMigrations/migrationsProcessor.ts
new file mode 100644
index 0000000000..d639fcd2c1
--- /dev/null
+++ b/packages/server/src/appMigrations/migrationsProcessor.ts
@@ -0,0 +1,60 @@
+import { context, locks } from "@budibase/backend-core"
+import { LockName, LockType } from "@budibase/types"
+
+import {
+  getAppMigrationVersion,
+  updateAppMigrationMetadata,
+} from "./appMigrationMetadata"
+
+export async function processMigrations(
+  appId: string,
+  migrations: {
+    migrationId: string
+    migrationFunc: () => Promise<void>
+  }[]
+) {
+  console.log(`Processing app migration for "${appId}"`)
+
+  await locks.doWithLock(
+    {
+      name: LockName.APP_MIGRATION,
+      type: LockType.AUTO_EXTEND,
+      resource: appId,
+    },
+    async () => {
+      await context.doInAppMigrationContext(appId, async () => {
+        let currentVersion = await getAppMigrationVersion(appId)
+
+        const pendingMigrations = migrations
+          .filter(m => m.migrationId > currentVersion)
+          .sort((a, b) => a.migrationId.localeCompare(b.migrationId))
+
+        const migrationIds = migrations.map(m => m.migrationId).sort()
+
+        let index = 0
+        for (const { migrationId, migrationFunc } of pendingMigrations) {
+          const expectedMigration =
+            migrationIds[migrationIds.indexOf(currentVersion) + 1]
+
+          if (expectedMigration !== migrationId) {
+            throw `Migration ${migrationId} could not run, update for "${migrationId}" is running but ${expectedMigration} is expected`
+          }
+
+          const counter = `(${++index}/${pendingMigrations.length})`
+          console.info(`Running migration ${migrationId}... ${counter}`, {
+            migrationId,
+            appId,
+          })
+          await migrationFunc()
+          await updateAppMigrationMetadata({
+            appId,
+            version: migrationId,
+          })
+          currentVersion = migrationId
+        }
+      })
+    }
+  )
+
+  console.log(`App migration for "${appId}" processed`)
+}
diff --git a/packages/server/src/appMigrations/queue.ts b/packages/server/src/appMigrations/queue.ts
index d1bb9f3829..72bb2f9b12 100644
--- a/packages/server/src/appMigrations/queue.ts
+++ b/packages/server/src/appMigrations/queue.ts
@@ -1,61 +1,15 @@
-import { context, locks, queue } from "@budibase/backend-core"
-import { LockName, LockType } from "@budibase/types"
+import { queue } from "@budibase/backend-core"
 import { Job } from "bull"
 import { MIGRATIONS } from "./migrations"
-import {
-  getAppMigrationVersion,
-  updateAppMigrationMetadata,
-} from "./appMigrationMetadata"
+import { processMigrations } from "./migrationsProcessor"
 
 const appMigrationQueue = queue.createQueue(queue.JobQueue.APP_MIGRATION)
 appMigrationQueue.process(processMessage)
 
 async function processMessage(job: Job) {
   const { appId } = job.data
-  console.log(`Processing app migration for "${appId}"`)
 
-  await locks.doWithLock(
-    {
-      name: LockName.APP_MIGRATION,
-      type: LockType.AUTO_EXTEND,
-      resource: appId,
-    },
-    async () => {
-      await context.doInAppContext(appId, async () => {
-        let currentVersion = await getAppMigrationVersion(appId)
-
-        const pendingMigrations = MIGRATIONS.filter(
-          m => m.migrationId > currentVersion
-        ).sort((a, b) => a.migrationId.localeCompare(b.migrationId))
-
-        const migrationIds = MIGRATIONS.map(m => m.migrationId).sort()
-
-        let index = 0
-        for (const { migrationId, migrationFunc } of pendingMigrations) {
-          const expectedMigration =
-            migrationIds[migrationIds.indexOf(currentVersion) + 1]
-
-          if (expectedMigration !== migrationId) {
-            throw `Migration ${migrationId} could not run, update for "${migrationId}" is running but ${expectedMigration} is expected`
-          }
-
-          const counter = `(${++index}/${pendingMigrations.length})`
-          console.info(`Running migration ${migrationId}... ${counter}`, {
-            migrationId,
-            appId,
-          })
-          await migrationFunc()
-          await updateAppMigrationMetadata({
-            appId,
-            version: migrationId,
-          })
-          currentVersion = migrationId
-        }
-      })
-    }
-  )
-
-  console.log(`App migration for "${appId}" processed`)
+  await processMigrations(appId, MIGRATIONS)
 }
 
 export default appMigrationQueue

From 22bc8e1a376460a5b6c3d555a0e19366f0eb2092 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 4 Dec 2023 16:49:03 +0100
Subject: [PATCH 17/30] Add tests

---
 .../tests/migrationsProcessor.spec.ts         | 55 +++++++++++++++++++
 1 file changed, 55 insertions(+)
 create mode 100644 packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts

diff --git a/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts b/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts
new file mode 100644
index 0000000000..460524cb9c
--- /dev/null
+++ b/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts
@@ -0,0 +1,55 @@
+import * as setup from "../../api/routes/tests/utilities"
+import { processMigrations } from "../migrationsProcessor"
+import { getAppMigrationVersion } from "../appMigrationMetadata"
+import { context } from "@budibase/backend-core"
+
+describe("migrationsProcessor", () => {
+  it("running migrations will update the latest applied migration", async () => {
+    const testMigrations: {
+      migrationId: string
+      migrationFunc: () => Promise<void>
+    }[] = [
+      { migrationId: "123", migrationFunc: async () => {} },
+      { migrationId: "124", migrationFunc: async () => {} },
+      { migrationId: "125", migrationFunc: async () => {} },
+    ]
+
+    const config = setup.getConfig()
+    await config.init()
+
+    const appId = config.getAppId()
+
+    await config.doInContext(appId, () =>
+      processMigrations(appId, testMigrations)
+    )
+
+    expect(
+      await config.doInContext(appId, () => getAppMigrationVersion(appId))
+    ).toBe("125")
+  })
+
+  it("no context can be initialised within a migration", async () => {
+    const testMigrations: {
+      migrationId: string
+      migrationFunc: () => Promise<void>
+    }[] = [
+      {
+        migrationId: "123",
+        migrationFunc: async () => {
+          await context.doInAppMigrationContext("any", () => {})
+        },
+      },
+    ]
+
+    const config = setup.getConfig()
+    await config.init()
+
+    const appId = config.getAppId()
+
+    await expect(
+      config.doInContext(appId, () => processMigrations(appId, testMigrations))
+    ).rejects.toThrowError(
+      "The context cannot be changed, a migration is currently running"
+    )
+  })
+})

From 5355238713cc1765547c0e24fec0bb59ea13ce1a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Dec 2023 11:02:44 +0100
Subject: [PATCH 18/30] Add title to migration

---
 package.json                 |  6 +++---
 scripts/add-app-migration.js | 11 +++++++++--
 scripts/build.js             |  2 +-
 yarn.lock                    | 15 ++++++++++++++-
 4 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/package.json b/package.json
index 08f77016fb..cb27e65914 100644
--- a/package.json
+++ b/package.json
@@ -17,14 +17,14 @@
     "kill-port": "^1.6.1",
     "lerna": "7.1.1",
     "madge": "^6.0.0",
-    "minimist": "^1.2.8",
     "nx": "16.4.3",
     "nx-cloud": "16.0.5",
     "prettier": "2.8.8",
     "prettier-plugin-svelte": "^2.3.0",
     "svelte": "3.49.0",
     "svelte-eslint-parser": "^0.32.0",
-    "typescript": "5.2.2"
+    "typescript": "5.2.2",
+    "yargs": "^17.7.2"
   },
   "scripts": {
     "preinstall": "node scripts/syncProPackage.js",
@@ -80,7 +80,7 @@
     "postinstall": "husky install",
     "submodules:load": "git submodule init && git submodule update && yarn",
     "submodules:unload": "git submodule deinit --all && yarn",
-    "add-app-migration": "node scripts/add-app-migration.js"
+    "add-app-migration": "node scripts/add-app-migration.js --title"
   },
   "workspaces": {
     "packages": [
diff --git a/scripts/add-app-migration.js b/scripts/add-app-migration.js
index d7af7a5d79..78f71eca93 100644
--- a/scripts/add-app-migration.js
+++ b/scripts/add-app-migration.js
@@ -1,6 +1,13 @@
 const fs = require("fs")
 const path = require("path")
 
+const argv = require("yargs").demandOption(
+  ["title"],
+  "Please provide the required parameter: --title=[title]"
+).argv
+
+const { title } = argv
+
 const generateTimestamp = () => {
   const now = new Date()
   const year = now.getFullYear()
@@ -14,7 +21,7 @@ const generateTimestamp = () => {
 }
 
 const createMigrationFile = () => {
-  const timestamp = generateTimestamp()
+  const migrationFilename = `${generateTimestamp()}_${title}`
   const migrationsDir = "../packages/server/src/appMigrations"
 
   const template = `const migration = async () => {
@@ -27,7 +34,7 @@ export default migration
   const newMigrationPath = path.join(
     migrationsDir,
     "migrations",
-    `${timestamp}.ts`
+    `${migrationFilename}.ts`
   )
   fs.writeFileSync(path.resolve(__dirname, newMigrationPath), template)
 
diff --git a/scripts/build.js b/scripts/build.js
index 99ce945eb8..ec7861332f 100755
--- a/scripts/build.js
+++ b/scripts/build.js
@@ -13,7 +13,7 @@ const {
 } = require("@esbuild-plugins/tsconfig-paths")
 const { nodeExternalsPlugin } = require("esbuild-node-externals")
 
-var argv = require("minimist")(process.argv.slice(2))
+var { argv } = require("yargs")
 
 function runBuild(entry, outfile) {
   const isDev = process.env.NODE_ENV !== "production"
diff --git a/yarn.lock b/yarn.lock
index a09ae20de6..e69419cb7a 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -15618,7 +15618,7 @@ minimist-options@4.1.0:
     is-plain-obj "^1.1.0"
     kind-of "^6.0.3"
 
-minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.5, minimist@^1.2.6, minimist@^1.2.8:
+minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.5, minimist@^1.2.6:
   version "1.2.8"
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c"
   integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==
@@ -22845,6 +22845,19 @@ yargs@^17.3.1, yargs@^17.5.1, yargs@^17.6.2:
     y18n "^5.0.5"
     yargs-parser "^21.1.1"
 
+yargs@^17.7.2:
+  version "17.7.2"
+  resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269"
+  integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==
+  dependencies:
+    cliui "^8.0.1"
+    escalade "^3.1.1"
+    get-caller-file "^2.0.5"
+    require-directory "^2.1.1"
+    string-width "^4.2.3"
+    y18n "^5.0.5"
+    yargs-parser "^21.1.1"
+
 yauzl@^2.4.2:
   version "2.10.0"
   resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9"

From 9625935a9b6892e4e993e139fe0d382c74e114ed Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Dec 2023 11:03:52 +0100
Subject: [PATCH 19/30] Add readme

---
 packages/server/README.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 packages/server/README.md

diff --git a/packages/server/README.md b/packages/server/README.md
new file mode 100644
index 0000000000..939a377404
--- /dev/null
+++ b/packages/server/README.md
@@ -0,0 +1,14 @@
+# Budibase server project
+
+This project contains all the server specific logic required to run a Budibase app
+
+## App migrations
+
+A migration system has been created in order to modify existing apps when breaking changes are added. These migrations will run on the app startup (both from the client side or the builder side), blocking the access until they are correctly applied.
+
+### Create a new migration
+
+In order to add a new migration:
+
+1. Run `yarn add-app-migration [title]`
+2. Write your code on the newly created file

From bd8c52094b5c12462c59e312bee392e6435ffaef Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Dec 2023 12:37:21 +0100
Subject: [PATCH 20/30] Initalise migration version on creation

---
 packages/server/src/api/controllers/application.ts | 6 ++++++
 packages/server/src/appMigrations/index.ts         | 8 ++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index b2b7a3e5cb..a6352be44b 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -52,6 +52,7 @@ import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
 import { sdk as sharedCoreSDK } from "@budibase/shared-core"
+import * as appMigrations from "../../appMigrations"
 
 // utility function, need to do away with this
 async function getLayouts() {
@@ -334,6 +335,11 @@ async function performAppCreate(ctx: UserCtx) {
     /* istanbul ignore next */
     if (!env.isTest()) {
       await createApp(appId)
+      // Initialise app migration version
+      await appMigrations.updateAppMigrationMetadata({
+        appId,
+        version: appMigrations.latestMigration,
+      })
     }
 
     await cache.app.invalidateAppMetadata(appId, newApplication)
diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts
index 1226333d96..3efe38e55a 100644
--- a/packages/server/src/appMigrations/index.ts
+++ b/packages/server/src/appMigrations/index.ts
@@ -2,14 +2,18 @@ import queue from "./queue"
 import { getAppMigrationVersion } from "./appMigrationMetadata"
 import { MIGRATIONS } from "./migrations"
 
-const latestMigration = MIGRATIONS.map(m => m.migrationId)
+export * from "./appMigrationMetadata"
+
+export const latestMigration = MIGRATIONS.map(m => m.migrationId)
   .sort()
   .reverse()[0]
 
+const getTimestamp = (versionId: string) => versionId.split("_")[0]
+
 export async function checkMissingMigrations(appId: string) {
   const currentVersion = await getAppMigrationVersion(appId)
 
-  if (currentVersion < latestMigration) {
+  if (getTimestamp(currentVersion) < getTimestamp(latestMigration)) {
     await queue.add(
       {
         appId,

From 8c1d0d1e4ea5fc991748a1981077f79787369251 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Dec 2023 12:40:26 +0100
Subject: [PATCH 21/30] Save version as string

---
 packages/server/src/appMigrations/appMigrationMetadata.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/appMigrations/appMigrationMetadata.ts b/packages/server/src/appMigrations/appMigrationMetadata.ts
index 97d3623d82..b8db0e3c0e 100644
--- a/packages/server/src/appMigrations/appMigrationMetadata.ts
+++ b/packages/server/src/appMigrations/appMigrationMetadata.ts
@@ -3,7 +3,7 @@ import { Database, DocumentType, Document } from "@budibase/types"
 
 export interface AppMigrationDoc extends Document {
   version: string
-  history: Record<string, { runAt: number }>
+  history: Record<string, { runAt: string }>
 }
 
 const EXPIRY_SECONDS = Duration.fromDays(1).toSeconds()
@@ -78,7 +78,7 @@ export async function updateAppMigrationMetadata({
     version,
     history: {
       ...appMigrationDoc.history,
-      [version]: { runAt: Date.now() },
+      [version]: { runAt: new Date().toISOString() },
     },
   }
   await db.put(updatedMigrationDoc)

From 69864a174ad0151af77463388d3a1ac1f97f6e39 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Dec 2023 12:42:09 +0100
Subject: [PATCH 22/30] Fix

---
 packages/server/src/api/controllers/application.ts | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index a6352be44b..c468038f54 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -335,13 +335,14 @@ async function performAppCreate(ctx: UserCtx) {
     /* istanbul ignore next */
     if (!env.isTest()) {
       await createApp(appId)
-      // Initialise app migration version
-      await appMigrations.updateAppMigrationMetadata({
-        appId,
-        version: appMigrations.latestMigration,
-      })
     }
 
+    // Initialise app migration version
+    await appMigrations.updateAppMigrationMetadata({
+      appId,
+      version: appMigrations.latestMigration,
+    })
+
     await cache.app.invalidateAppMetadata(appId, newApplication)
     return newApplication
   })

From d289a8869a4960184671dfe2110cbf10bbd22f2a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Dec 2023 12:58:53 +0100
Subject: [PATCH 23/30] Comments

---
 packages/server/src/api/controllers/application.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index c468038f54..bb4f447f79 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -337,7 +337,7 @@ async function performAppCreate(ctx: UserCtx) {
       await createApp(appId)
     }
 
-    // Initialise app migration version
+    // Initialise the app migration version as the latest one
     await appMigrations.updateAppMigrationMetadata({
       appId,
       version: appMigrations.latestMigration,

From e12fc874c828150e2aa87574b7a240552a6aaf4e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Dec 2023 14:09:59 +0100
Subject: [PATCH 24/30] Fix test when no migrations exist

---
 packages/server/src/appMigrations/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts
index 3efe38e55a..b1a2b742bc 100644
--- a/packages/server/src/appMigrations/index.ts
+++ b/packages/server/src/appMigrations/index.ts
@@ -8,7 +8,7 @@ export const latestMigration = MIGRATIONS.map(m => m.migrationId)
   .sort()
   .reverse()[0]
 
-const getTimestamp = (versionId: string) => versionId.split("_")[0]
+const getTimestamp = (versionId: string) => versionId?.split("_")[0]
 
 export async function checkMissingMigrations(appId: string) {
   const currentVersion = await getAppMigrationVersion(appId)

From a8070829c9239e80aca45e86f96391915cb3721e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Dec 2023 15:29:11 +0100
Subject: [PATCH 25/30] Rename

---
 packages/server/src/appMigrations/index.ts    |  7 ++++-
 .../server/src/appMigrations/migrations.ts    |  7 ++---
 .../src/appMigrations/migrationsProcessor.ts  | 28 +++++++++----------
 .../appMigrations/tests/migrations.spec.ts    |  4 +--
 .../tests/migrationsProcessor.spec.ts         | 21 ++++++--------
 5 files changed, 32 insertions(+), 35 deletions(-)

diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts
index b1a2b742bc..a4ffe64604 100644
--- a/packages/server/src/appMigrations/index.ts
+++ b/packages/server/src/appMigrations/index.ts
@@ -4,7 +4,12 @@ import { MIGRATIONS } from "./migrations"
 
 export * from "./appMigrationMetadata"
 
-export const latestMigration = MIGRATIONS.map(m => m.migrationId)
+export type AppMigration = {
+  id: string
+  func: () => Promise<void>
+}
+
+export const latestMigration = MIGRATIONS.map(m => m.id)
   .sort()
   .reverse()[0]
 
diff --git a/packages/server/src/appMigrations/migrations.ts b/packages/server/src/appMigrations/migrations.ts
index 893881e716..02b93a781b 100644
--- a/packages/server/src/appMigrations/migrations.ts
+++ b/packages/server/src/appMigrations/migrations.ts
@@ -1,8 +1,7 @@
 // This file should never be manually modified, use `yarn add-app-migration` in order to add a new one
 
-export const MIGRATIONS: {
-  migrationId: string
-  migrationFunc: () => Promise<void>
-}[] = [
+import { AppMigration } from "."
+
+export const MIGRATIONS: AppMigration[] = [
   // Migrations will be executed sorted by migrationId
 ]
diff --git a/packages/server/src/appMigrations/migrationsProcessor.ts b/packages/server/src/appMigrations/migrationsProcessor.ts
index d639fcd2c1..8d119f3864 100644
--- a/packages/server/src/appMigrations/migrationsProcessor.ts
+++ b/packages/server/src/appMigrations/migrationsProcessor.ts
@@ -5,13 +5,11 @@ import {
   getAppMigrationVersion,
   updateAppMigrationMetadata,
 } from "./appMigrationMetadata"
+import { AppMigration } from "."
 
 export async function processMigrations(
   appId: string,
-  migrations: {
-    migrationId: string
-    migrationFunc: () => Promise<void>
-  }[]
+  migrations: AppMigration[]
 ) {
   console.log(`Processing app migration for "${appId}"`)
 
@@ -26,31 +24,31 @@ export async function processMigrations(
         let currentVersion = await getAppMigrationVersion(appId)
 
         const pendingMigrations = migrations
-          .filter(m => m.migrationId > currentVersion)
-          .sort((a, b) => a.migrationId.localeCompare(b.migrationId))
+          .filter(m => m.id > currentVersion)
+          .sort((a, b) => a.id.localeCompare(b.id))
 
-        const migrationIds = migrations.map(m => m.migrationId).sort()
+        const migrationIds = migrations.map(m => m.id).sort()
 
         let index = 0
-        for (const { migrationId, migrationFunc } of pendingMigrations) {
+        for (const { id, func } of pendingMigrations) {
           const expectedMigration =
             migrationIds[migrationIds.indexOf(currentVersion) + 1]
 
-          if (expectedMigration !== migrationId) {
-            throw `Migration ${migrationId} could not run, update for "${migrationId}" is running but ${expectedMigration} is expected`
+          if (expectedMigration !== id) {
+            throw `Migration ${id} could not run, update for "${id}" is running but ${expectedMigration} is expected`
           }
 
           const counter = `(${++index}/${pendingMigrations.length})`
-          console.info(`Running migration ${migrationId}... ${counter}`, {
-            migrationId,
+          console.info(`Running migration ${id}... ${counter}`, {
+            migrationId: id,
             appId,
           })
-          await migrationFunc()
+          await func()
           await updateAppMigrationMetadata({
             appId,
-            version: migrationId,
+            version: id,
           })
-          currentVersion = migrationId
+          currentVersion = id
         }
       })
     }
diff --git a/packages/server/src/appMigrations/tests/migrations.spec.ts b/packages/server/src/appMigrations/tests/migrations.spec.ts
index a5ceb05d69..b761837dd4 100644
--- a/packages/server/src/appMigrations/tests/migrations.spec.ts
+++ b/packages/server/src/appMigrations/tests/migrations.spec.ts
@@ -10,10 +10,10 @@ describe("migration", () => {
     await config.doInContext(config.getAppId(), async () => {
       const db = context.getAppDB()
       for (const migration of MIGRATIONS) {
-        await migration.migrationFunc()
+        await migration.func()
         const docs = await db.allDocs({ include_docs: true })
 
-        await migration.migrationFunc()
+        await migration.func()
         const latestDocs = await db.allDocs({ include_docs: true })
 
         expect(docs).toEqual(latestDocs)
diff --git a/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts b/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts
index 460524cb9c..189f6c068b 100644
--- a/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts
+++ b/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts
@@ -2,16 +2,14 @@ import * as setup from "../../api/routes/tests/utilities"
 import { processMigrations } from "../migrationsProcessor"
 import { getAppMigrationVersion } from "../appMigrationMetadata"
 import { context } from "@budibase/backend-core"
+import { AppMigration } from ".."
 
 describe("migrationsProcessor", () => {
   it("running migrations will update the latest applied migration", async () => {
-    const testMigrations: {
-      migrationId: string
-      migrationFunc: () => Promise<void>
-    }[] = [
-      { migrationId: "123", migrationFunc: async () => {} },
-      { migrationId: "124", migrationFunc: async () => {} },
-      { migrationId: "125", migrationFunc: async () => {} },
+    const testMigrations: AppMigration[] = [
+      { id: "123", func: async () => {} },
+      { id: "124", func: async () => {} },
+      { id: "125", func: async () => {} },
     ]
 
     const config = setup.getConfig()
@@ -29,13 +27,10 @@ describe("migrationsProcessor", () => {
   })
 
   it("no context can be initialised within a migration", async () => {
-    const testMigrations: {
-      migrationId: string
-      migrationFunc: () => Promise<void>
-    }[] = [
+    const testMigrations: AppMigration[] = [
       {
-        migrationId: "123",
-        migrationFunc: async () => {
+        id: "123",
+        func: async () => {
           await context.doInAppMigrationContext("any", () => {})
         },
       },

From c94bd63374a9454450799219ae84090927c0ef71 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Dec 2023 15:31:32 +0100
Subject: [PATCH 26/30] Fix scripts

---
 packages/server/src/appMigrations/migrations.ts |  2 +-
 scripts/add-app-migration.js                    | 13 +++++--------
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/appMigrations/migrations.ts b/packages/server/src/appMigrations/migrations.ts
index 02b93a781b..d66e2e8895 100644
--- a/packages/server/src/appMigrations/migrations.ts
+++ b/packages/server/src/appMigrations/migrations.ts
@@ -3,5 +3,5 @@
 import { AppMigration } from "."
 
 export const MIGRATIONS: AppMigration[] = [
-  // Migrations will be executed sorted by migrationId
+  // Migrations will be executed sorted by id
 ]
diff --git a/scripts/add-app-migration.js b/scripts/add-app-migration.js
index 78f71eca93..a58d3a4fbe 100644
--- a/scripts/add-app-migration.js
+++ b/scripts/add-app-migration.js
@@ -51,22 +51,19 @@ export default migration
     .map(m => m.substring(0, m.length - 3))
 
   let migrationFileContent =
-    "// This file should never be manually modified, use `yarn add-app-migration` in order to add a new one\n\n"
+    '// This file should never be manually modified, use `yarn add-app-migration` in order to add a new one\n\nimport { AppMigration } from "."\n\n'
 
   for (const migration of migrations) {
     migrationFileContent += `import m${migration} from "./migrations/${migration}"\n`
   }
 
-  migrationFileContent += `\nexport const MIGRATIONS: {
-  migrationId: string
-  migrationFunc: () => Promise<void>
-}[] = [
-  // Migrations will be executed sorted by migrationId\n`
+  migrationFileContent += `\nexport const MIGRATIONS: AppMigration[] = [
+  // Migrations will be executed sorted by id\n`
 
   for (const migration of migrations) {
     migrationFileContent += `  {
-    migrationId: "${migration}",
-    migrationFunc: m${migration}
+    id: "${migration}",
+    func: m${migration}
   },\n`
   }
 

From 7cf9a915f4112771cb054052e7e91aa774680afd Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Dec 2023 15:37:45 +0100
Subject: [PATCH 27/30] Fix tests

---
 packages/server/src/appMigrations/appMigrationMetadata.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/appMigrations/appMigrationMetadata.ts b/packages/server/src/appMigrations/appMigrationMetadata.ts
index b8db0e3c0e..202e78d964 100644
--- a/packages/server/src/appMigrations/appMigrationMetadata.ts
+++ b/packages/server/src/appMigrations/appMigrationMetadata.ts
@@ -25,6 +25,7 @@ export async function getAppMigrationVersion(appId: string): Promise<string> {
 
   let metadata: AppMigrationDoc | undefined = await cache.get(cacheKey)
 
+  // We don't want to cache in dev, in order to be able to tweak it
   if (metadata && !env.isDev()) {
     return metadata.version
   }
@@ -75,7 +76,7 @@ export async function updateAppMigrationMetadata({
 
   const updatedMigrationDoc: AppMigrationDoc = {
     ...appMigrationDoc,
-    version,
+    version: version || "",
     history: {
       ...appMigrationDoc.history,
       [version]: { runAt: new Date().toISOString() },

From ee1a198f1d6890a9e2d7c1da2c0cc48774783c81 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 11 Dec 2023 09:38:05 +0100
Subject: [PATCH 28/30] Add test comments

---
 packages/server/src/appMigrations/tests/migrations.spec.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/server/src/appMigrations/tests/migrations.spec.ts b/packages/server/src/appMigrations/tests/migrations.spec.ts
index b761837dd4..9d80cc5f99 100644
--- a/packages/server/src/appMigrations/tests/migrations.spec.ts
+++ b/packages/server/src/appMigrations/tests/migrations.spec.ts
@@ -3,6 +3,8 @@ import * as setup from "../../api/routes/tests/utilities"
 import { MIGRATIONS } from "../migrations"
 
 describe("migration", () => {
+  // These test is checking that each migration is "idempotent".
+  // We should be able to rerun any migration, with any rerun not modifiying anything. The code should be aware that the migration already ran
   it("each migration can rerun safely", async () => {
     const config = setup.getConfig()
     await config.init()

From 8fa09a1a2b2ea1164f07b8371f3b785a9b098b2b Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Mon, 11 Dec 2023 09:31:03 +0000
Subject: [PATCH 29/30] remove pricing banner

---
 .../src/pages/builder/portal/apps/index.svelte   | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/apps/index.svelte b/packages/builder/src/pages/builder/portal/apps/index.svelte
index cf2c61b11d..ad0d3658ea 100644
--- a/packages/builder/src/pages/builder/portal/apps/index.svelte
+++ b/packages/builder/src/pages/builder/portal/apps/index.svelte
@@ -1,6 +1,5 @@
 <script>
   import {
-    banner,
     Heading,
     Layout,
     Button,
@@ -11,7 +10,6 @@
     Notification,
     Body,
     Search,
-    BANNER_TYPES,
   } from "@budibase/bbui"
   import Spinner from "components/common/Spinner.svelte"
   import CreateAppModal from "components/start/CreateAppModal.svelte"
@@ -200,20 +198,6 @@
       if (usersLimitLockAction) {
         usersLimitLockAction()
       }
-      if (!$admin.isDev) {
-        await banner.show({
-          messages: [
-            {
-              message:
-                "We've updated our pricing - see our website to learn more.",
-              type: BANNER_TYPES.NEUTRAL,
-              extraButtonText: "Learn More",
-              extraButtonAction: () =>
-                window.open("https://budibase.com/pricing"),
-            },
-          ],
-        })
-      }
     } catch (error) {
       notifications.error("Error getting init info")
     }

From 1735321949f15a1565280bdebe66eeecb7bbcd5c Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Mon, 11 Dec 2023 10:20:28 +0000
Subject: [PATCH 30/30] Bump version to 2.13.36

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index d6a5f41281..449b09f49d 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.13.35",
+  "version": "2.13.36",
   "npmClient": "yarn",
   "packages": [
     "packages/*"