Fixing an issue with multi-tenant app migration failures - the app migration context wasn't getting the tenant ID as needed. Also added some better logging as well as cleaning up the queues properly to remove these dud events.

packages/backend-core/src/queue/queue.ts

@@ -21,6 +21,7 @@ let cleanupInterval: NodeJS.Timeout
 async function cleanup() {
   for (let queue of QUEUES) {
     await queue.clean(CLEANUP_PERIOD_MS, "completed")
+    await queue.clean(CLEANUP_PERIOD_MS, "failed")
   }
 }
 

packages/server/src/appMigrations/index.ts

@@ -48,8 +48,6 @@ export async function checkMissingMigrations(
       },
       {
         jobId: `${appId}_${latestMigration}`,
-        removeOnComplete: true,
-        removeOnFail: true,
       }
     )
 

packages/server/src/appMigrations/migrationsProcessor.ts

@@ -12,7 +12,8 @@ export async function processMigrations(
   migrations: AppMigration[]
 ) {
   console.log(`Processing app migration for "${appId}"`)
-
+  // have to wrap in context, this gets the tenant from the app ID
+  await context.doInAppContext(appId, async () => {
     await locks.doWithLock(
       {
         name: LockName.APP_MIGRATION,
@@ -20,8 +21,8 @@ export async function processMigrations(
         resource: appId,
       },
       async () => {
-      await context.doInAppMigrationContext(appId, async () => {
         try {
+          await context.doInAppMigrationContext(appId, async () => {
             let currentVersion = await getAppMigrationVersion(appId)
 
             const pendingMigrations = migrations
@@ -53,13 +54,14 @@ export async function processMigrations(
               })
               currentVersion = id
             }
+          })
         } catch (err) {
           logging.logAlert("Failed to run app migration", err)
           throw err
         }
-      })
       }
     )
 
     console.log(`App migration for "${appId}" processed`)
+  })
 }

packages/server/src/appMigrations/queue.ts

@@ -1,9 +1,23 @@
-import { queue } from "@budibase/backend-core"
+import { queue, logging } from "@budibase/backend-core"
 import { Job } from "bull"
 import { MIGRATIONS } from "./migrations"
 import { processMigrations } from "./migrationsProcessor"
 
-const appMigrationQueue = queue.createQueue(queue.JobQueue.APP_MIGRATION)
+const MAX_ATTEMPTS = 3
+
+const appMigrationQueue = queue.createQueue(queue.JobQueue.APP_MIGRATION, {
+  jobOptions: {
+    attempts: MAX_ATTEMPTS,
+    removeOnComplete: true,
+    removeOnFail: true,
+  },
+  maxStalledCount: MAX_ATTEMPTS,
+  removeStalledCb: async (job: Job) => {
+    logging.logAlert(
+      `App migration failed, queue job ID: ${job.id} - reason: ${job.failedReason}`
+    )
+  },
+})
 appMigrationQueue.process(processMessage)
 
 async function processMessage(job: Job) {