From c328dd5cd43f41872981d619960811bb67db46c2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Jun 2024 18:32:06 +0100 Subject: [PATCH 1/7] Making sure any error that occurs in the app migration system gets logged. --- .../src/appMigrations/migrationsProcessor.ts | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/server/src/appMigrations/migrationsProcessor.ts b/packages/server/src/appMigrations/migrationsProcessor.ts index badd50f53c..d4f474cd30 100644 --- a/packages/server/src/appMigrations/migrationsProcessor.ts +++ b/packages/server/src/appMigrations/migrationsProcessor.ts @@ -12,16 +12,16 @@ export async function processMigrations( migrations: AppMigration[] ) { console.log(`Processing app migration for "${appId}"`) - // have to wrap in context, this gets the tenant from the app ID - await context.doInAppContext(appId, async () => { - await locks.doWithLock( - { - name: LockName.APP_MIGRATION, - type: LockType.AUTO_EXTEND, - resource: appId, - }, - async () => { - try { + try { + // have to wrap in context, this gets the tenant from the app ID + await context.doInAppContext(appId, async () => { + await locks.doWithLock( + { + name: LockName.APP_MIGRATION, + type: LockType.AUTO_EXTEND, + resource: appId, + }, + async () => { await context.doInAppMigrationContext(appId, async () => { let currentVersion = await getAppMigrationVersion(appId) @@ -55,13 +55,13 @@ export async function processMigrations( currentVersion = id } }) - } catch (err) { - logging.logAlert("Failed to run app migration", err) - throw err } - } - ) + ) - console.log(`App migration for "${appId}" processed`) - }) + console.log(`App migration for "${appId}" processed`) + }) + } catch (err) { + logging.logAlert("Failed to run app migration", err) + throw err + } } From 3a95aa6aeb1b9b5e28401981206737ee49aeab4b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Jun 2024 18:56:24 +0100 Subject: [PATCH 2/7] Adding the version to the status to help understand what version the service is using. --- packages/types/src/api/web/system/index.ts | 1 + packages/types/src/api/web/system/status.ts | 11 ++++++++++ .../src/api/controllers/system/status.ts | 22 +++++++++++++------ 3 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 packages/types/src/api/web/system/status.ts diff --git a/packages/types/src/api/web/system/index.ts b/packages/types/src/api/web/system/index.ts index d19c9296c9..8820bf5cd1 100644 --- a/packages/types/src/api/web/system/index.ts +++ b/packages/types/src/api/web/system/index.ts @@ -1 +1,2 @@ export * from "./environment" +export * from "./status" diff --git a/packages/types/src/api/web/system/status.ts b/packages/types/src/api/web/system/status.ts new file mode 100644 index 0000000000..3d64cc4d97 --- /dev/null +++ b/packages/types/src/api/web/system/status.ts @@ -0,0 +1,11 @@ +export type SystemStatusResponse = { + passing?: boolean + checks?: { + login: boolean + search: boolean + } + health?: { + passing: boolean + } + version?: string +} diff --git a/packages/worker/src/api/controllers/system/status.ts b/packages/worker/src/api/controllers/system/status.ts index b763a67d4f..662ca05c48 100644 --- a/packages/worker/src/api/controllers/system/status.ts +++ b/packages/worker/src/api/controllers/system/status.ts @@ -1,16 +1,24 @@ -import { accounts } from "@budibase/backend-core" +import { accounts, env as coreEnv } from "@budibase/backend-core" +import { Ctx, SystemStatusResponse } from "@budibase/types" import env from "../../../environment" -import { BBContext } from "@budibase/types" -export const fetch = async (ctx: BBContext) => { +export const fetch = async (ctx: Ctx) => { + let status: SystemStatusResponse | undefined if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { - const status = await accounts.getStatus() - ctx.body = status - } else { - ctx.body = { + status = await accounts.getStatus() + } + + if (!status) { + status = { health: { passing: true, }, } } + + if (coreEnv.VERSION) { + status.version = coreEnv.VERSION + } + + ctx.body = status } From d4d3537f41e198069d2bba56579643ec2f297160 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Jun 2024 19:02:30 +0100 Subject: [PATCH 3/7] Updating test. --- packages/worker/src/api/routes/system/tests/status.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/worker/src/api/routes/system/tests/status.spec.ts b/packages/worker/src/api/routes/system/tests/status.spec.ts index 71e01a0e72..cfb9e2d68f 100644 --- a/packages/worker/src/api/routes/system/tests/status.spec.ts +++ b/packages/worker/src/api/routes/system/tests/status.spec.ts @@ -27,6 +27,7 @@ describe("/api/system/status", () => { health: { passing: true, }, + version: expect.any(String), }) expect(accounts.getStatus).toHaveBeenCalledTimes(0) config.cloudHosted() From 834de1f64a0a264ecbb27c449e383ba0dd519294 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Jun 2024 19:19:43 +0100 Subject: [PATCH 4/7] Log every step of the app migration process for easier visibility. --- packages/server/src/appMigrations/migrationsProcessor.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/server/src/appMigrations/migrationsProcessor.ts b/packages/server/src/appMigrations/migrationsProcessor.ts index d4f474cd30..0337fc09f4 100644 --- a/packages/server/src/appMigrations/migrationsProcessor.ts +++ b/packages/server/src/appMigrations/migrationsProcessor.ts @@ -15,6 +15,7 @@ export async function processMigrations( try { // have to wrap in context, this gets the tenant from the app ID await context.doInAppContext(appId, async () => { + console.log(`Acquiring app migration lock for "${appId}"`) await locks.doWithLock( { name: LockName.APP_MIGRATION, @@ -23,6 +24,7 @@ export async function processMigrations( }, async () => { await context.doInAppMigrationContext(appId, async () => { + console.log(`Lock acquired starting app migration for "${appId}"`) let currentVersion = await getAppMigrationVersion(appId) const pendingMigrations = migrations @@ -30,6 +32,9 @@ export async function processMigrations( .sort((a, b) => a.id.localeCompare(b.id)) const migrationIds = migrations.map(m => m.id).sort() + console.log( + `App migrations to run for "${appId}" - ${migrationIds.join(",")}` + ) let index = 0 for (const { id, func } of pendingMigrations) { From 1a0d6ef5b0d99daf33ff434a309e4036a4e6138c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Jun 2024 20:46:08 +0100 Subject: [PATCH 5/7] Only run app migrations in API service - testing this in QA. --- packages/server/src/appMigrations/queue.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/server/src/appMigrations/queue.ts b/packages/server/src/appMigrations/queue.ts index 9dbc732e82..309de7e1cf 100644 --- a/packages/server/src/appMigrations/queue.ts +++ b/packages/server/src/appMigrations/queue.ts @@ -2,6 +2,7 @@ import { queue, logging } from "@budibase/backend-core" import { Job } from "bull" import { MIGRATIONS } from "./migrations" import { processMigrations } from "./migrationsProcessor" +import { apiEnabled } from "../features" const MAX_ATTEMPTS = 3 @@ -18,7 +19,11 @@ const appMigrationQueue = queue.createQueue(queue.JobQueue.APP_MIGRATION, { ) }, }) -appMigrationQueue.process(processMessage) + +// only run app migrations in main API services +if (apiEnabled()) { + appMigrationQueue.process(processMessage) +} async function processMessage(job: Job) { const { appId } = job.data From 739ac5d03cec459150040561eea5734b855e8b5c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Jun 2024 21:37:01 +0100 Subject: [PATCH 6/7] Putting a better startup process in place for app migrations and adding them to bullboard as well. --- packages/server/src/appMigrations/index.ts | 4 +- packages/server/src/appMigrations/queue.ts | 50 +++++++++++++------- packages/server/src/automations/bullboard.ts | 5 ++ packages/server/src/startup/index.ts | 2 + 4 files changed, 42 insertions(+), 19 deletions(-) diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts index 89c71ae26f..0440580d18 100644 --- a/packages/server/src/appMigrations/index.ts +++ b/packages/server/src/appMigrations/index.ts @@ -1,4 +1,4 @@ -import queue from "./queue" +import { getAppMigrationQueue } from "./queue" import { Next } from "koa" import { getAppMigrationVersion } from "./appMigrationMetadata" import { MIGRATIONS } from "./migrations" @@ -37,8 +37,10 @@ export async function checkMissingMigrations( ) { const currentVersion = await getAppMigrationVersion(appId) const latestMigration = getLatestEnabledMigrationId() + const queue = getAppMigrationQueue() if ( + queue && latestMigration && getTimestamp(currentVersion) < getTimestamp(latestMigration) ) { diff --git a/packages/server/src/appMigrations/queue.ts b/packages/server/src/appMigrations/queue.ts index 309de7e1cf..37a87dc777 100644 --- a/packages/server/src/appMigrations/queue.ts +++ b/packages/server/src/appMigrations/queue.ts @@ -4,25 +4,37 @@ import { MIGRATIONS } from "./migrations" import { processMigrations } from "./migrationsProcessor" import { apiEnabled } from "../features" -const MAX_ATTEMPTS = 3 +const MAX_ATTEMPTS = 1 -const appMigrationQueue = queue.createQueue(queue.JobQueue.APP_MIGRATION, { - jobOptions: { - attempts: MAX_ATTEMPTS, - removeOnComplete: true, - removeOnFail: true, - }, - maxStalledCount: MAX_ATTEMPTS, - removeStalledCb: async (job: Job) => { - logging.logAlert( - `App migration failed, queue job ID: ${job.id} - reason: ${job.failedReason}` - ) - }, -}) +export type AppMigrationJob = { + appId: string +} -// only run app migrations in main API services -if (apiEnabled()) { - appMigrationQueue.process(processMessage) +let appMigrationQueue: queue.Queue | undefined + +export function init() { + // only run app migrations in main API services + if (!apiEnabled()) { + return + } + const appMigrationQueue = queue.createQueue( + queue.JobQueue.APP_MIGRATION, + { + jobOptions: { + attempts: MAX_ATTEMPTS, + removeOnComplete: true, + removeOnFail: true, + }, + maxStalledCount: MAX_ATTEMPTS, + removeStalledCb: async (job: Job) => { + logging.logAlert( + `App migration failed, queue job ID: ${job.id} - reason: ${job.failedReason}` + ) + }, + } + ) + + return appMigrationQueue.process(processMessage) } async function processMessage(job: Job) { @@ -31,4 +43,6 @@ async function processMessage(job: Job) { await processMigrations(appId, MIGRATIONS) } -export default appMigrationQueue +export function getAppMigrationQueue() { + return appMigrationQueue +} diff --git a/packages/server/src/automations/bullboard.ts b/packages/server/src/automations/bullboard.ts index 34f18754a2..aa4287b2d0 100644 --- a/packages/server/src/automations/bullboard.ts +++ b/packages/server/src/automations/bullboard.ts @@ -3,6 +3,7 @@ import { KoaAdapter } from "@bull-board/koa" import { queue } from "@budibase/backend-core" import * as automation from "../threads/automation" import { backups } from "@budibase/pro" +import { getAppMigrationQueue } from "../appMigrations/queue" import { createBullBoard } from "@bull-board/api" import BullQueue from "bull" @@ -16,10 +17,14 @@ const PATH_PREFIX = "/bulladmin" export async function init() { // Set up queues for bull board admin const backupQueue = backups.getBackupQueue() + const appMigrationQueue = getAppMigrationQueue() const queues = [automationQueue] if (backupQueue) { queues.push(backupQueue) } + if (appMigrationQueue) { + queues.push(appMigrationQueue) + } const adapters = [] const serverAdapter: any = new KoaAdapter() for (let queue of queues) { diff --git a/packages/server/src/startup/index.ts b/packages/server/src/startup/index.ts index 48d500a0cf..750acdb0aa 100644 --- a/packages/server/src/startup/index.ts +++ b/packages/server/src/startup/index.ts @@ -15,6 +15,7 @@ import * as fileSystem from "../utilities/fileSystem" import { default as eventEmitter, init as eventInit } from "../events" import * as migrations from "../migrations" import * as bullboard from "../automations/bullboard" +import * as appMigrations from "../appMigrations/queue" import * as pro from "@budibase/pro" import * as api from "../api" import sdk from "../sdk" @@ -114,6 +115,7 @@ export async function startup( // configure events to use the pro audit log write // can't integrate directly into backend-core due to cyclic issues queuePromises.push(events.processors.init(pro.sdk.auditLogs.write)) + queuePromises.push(appMigrations.init()) if (automationsEnabled()) { queuePromises.push(automations.init()) } From b286e2340b9756566dde889b9b4f44b7fd53a708 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 10 Jun 2024 21:48:02 +0100 Subject: [PATCH 7/7] Fixing an issue with test build. --- packages/server/src/appMigrations/queue.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/appMigrations/queue.ts b/packages/server/src/appMigrations/queue.ts index 37a87dc777..5c932bcb7f 100644 --- a/packages/server/src/appMigrations/queue.ts +++ b/packages/server/src/appMigrations/queue.ts @@ -17,7 +17,7 @@ export function init() { if (!apiEnabled()) { return } - const appMigrationQueue = queue.createQueue( + appMigrationQueue = queue.createQueue( queue.JobQueue.APP_MIGRATION, { jobOptions: {