diff --git a/.vscode/settings.json b/.vscode/settings.json index ece537efac..e22d5a8866 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,7 +1,7 @@ { "editor.formatOnSave": true, "editor.codeActionsOnSave": { - "source.fixAll": true + "source.fixAll": "explicit" }, "editor.defaultFormatter": "esbenp.prettier-vscode", "[json]": { diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts index 8ef34196ed..aee099e10a 100644 --- a/packages/backend-core/src/constants/misc.ts +++ b/packages/backend-core/src/constants/misc.ts @@ -11,24 +11,7 @@ export enum Cookie { OIDC_CONFIG = "budibase:oidc:config", } -export enum Header { - API_KEY = "x-budibase-api-key", - LICENSE_KEY = "x-budibase-license-key", - API_VER = "x-budibase-api-version", - APP_ID = "x-budibase-app-id", - SESSION_ID = "x-budibase-session-id", - TYPE = "x-budibase-type", - PREVIEW_ROLE = "x-budibase-role", - TENANT_ID = "x-budibase-tenant-id", - VERIFICATION_CODE = "x-budibase-verification-code", - RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code", - RESET_PASSWORD_CODE = "x-budibase-reset-password-code", - RETURN_RESET_PASSWORD_CODE = "x-budibase-return-reset-password-code", - TOKEN = "x-budibase-token", - CSRF_TOKEN = "x-csrf-token", - CORRELATION_ID = "x-budibase-correlation-id", - AUTHORIZATION = "authorization", -} +export { Header } from "@budibase/shared-core" export enum GlobalRole { OWNER = "owner", diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index ee1ef6da0c..0554737518 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -96,7 +96,7 @@ export async function getAppIdFromCtx(ctx: Ctx) { } // look in the path - const pathId = parseAppIdFromUrl(ctx.path) + const pathId = parseAppIdFromUrlPath(ctx.path) if (!appId && pathId) { appId = confirmAppId(pathId) } @@ -116,18 +116,21 @@ export async function getAppIdFromCtx(ctx: Ctx) { // referer header is present from a builder redirect const referer = ctx.request.headers.referer if (!appId && referer?.includes(BUILDER_APP_PREFIX)) { - const refererId = parseAppIdFromUrl(ctx.request.headers.referer) + const refererId = parseAppIdFromUrlPath(ctx.request.headers.referer) appId = confirmAppId(refererId) } return appId } -function parseAppIdFromUrl(url?: string) { +function parseAppIdFromUrlPath(url?: string) { if (!url) { return } - return url.split("/").find(subPath => subPath.startsWith(APP_PREFIX)) + return url + .split("?")[0] // Remove any possible query string + .split("/") + .find(subPath => subPath.startsWith(APP_PREFIX)) } /** diff --git a/packages/frontend-core/src/api/index.js b/packages/frontend-core/src/api/index.js index aefc3522a7..d4b4f3636e 100644 --- a/packages/frontend-core/src/api/index.js +++ b/packages/frontend-core/src/api/index.js @@ -1,4 +1,5 @@ import { Helpers } from "@budibase/bbui" +import { Header } from "@budibase/shared-core" import { ApiVersion } from "../constants" import { buildAnalyticsEndpoints } from "./analytics" import { buildAppEndpoints } from "./app" @@ -62,6 +63,11 @@ const defaultAPIClientConfig = { * invoked before the actual JS error is thrown up the stack. */ onError: null, + + /** + * A function can be passed to be called when an API call returns info about a migration running for a specific app + */ + onMigrationDetected: null, } /** @@ -133,9 +139,9 @@ export const createAPIClient = config => { // Build headers let headers = { Accept: "application/json" } - headers["x-budibase-session-id"] = APISessionID + headers[Header.SESSION_ID] = APISessionID if (!external) { - headers["x-budibase-api-version"] = ApiVersion + headers[Header.API_VER] = ApiVersion } if (json) { headers["Content-Type"] = "application/json" @@ -170,6 +176,7 @@ export const createAPIClient = config => { // Handle response if (response.status >= 200 && response.status < 400) { + handleMigrations(response) try { if (parseResponse) { return await parseResponse(response) @@ -186,7 +193,18 @@ export const createAPIClient = config => { } } - // Performs an API call to the server and caches the response. + const handleMigrations = response => { + if (!config.onMigrationDetected) { + return + } + const migration = response.headers.get(Header.MIGRATING_APP) + + if (migration) { + config.onMigrationDetected(migration) + } + } + + // Performs an API call to the server and caches the response. // Future invocation for this URL will return the cached result instead of // hitting the server again. const makeCachedApiCall = async params => { @@ -242,7 +260,7 @@ export const createAPIClient = config => { getAppID: () => { let headers = {} config?.attachHeaders(headers) - return headers?.["x-budibase-app-id"] + return headers?.[Header.APP_ID] }, } diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index bb4f447f79..70298c7172 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -340,7 +340,7 @@ async function performAppCreate(ctx: UserCtx) { // Initialise the app migration version as the latest one await appMigrations.updateAppMigrationMetadata({ appId, - version: appMigrations.latestMigration, + version: appMigrations.getLatestMigrationId(), }) await cache.app.invalidateAppMetadata(appId, newApplication) diff --git a/packages/server/src/api/controllers/migrations.ts b/packages/server/src/api/controllers/migrations.ts index 8f1bfa22db..c8f786578d 100644 --- a/packages/server/src/api/controllers/migrations.ts +++ b/packages/server/src/api/controllers/migrations.ts @@ -1,14 +1,34 @@ +import { context } from "@budibase/backend-core" import { migrate as migrationImpl, MIGRATIONS } from "../../migrations" -import { BBContext } from "@budibase/types" +import { Ctx } from "@budibase/types" +import { + getAppMigrationVersion, + getLatestMigrationId, +} from "../../appMigrations" -export async function migrate(ctx: BBContext) { +export async function migrate(ctx: Ctx) { const options = ctx.request.body // don't await as can take a while, just return migrationImpl(options) ctx.status = 200 } -export async function fetchDefinitions(ctx: BBContext) { +export async function fetchDefinitions(ctx: Ctx) { ctx.body = MIGRATIONS ctx.status = 200 } + +export async function getMigrationStatus(ctx: Ctx) { + const appId = context.getAppId() + + if (!appId) { + ctx.throw("AppId could not be found") + } + + const latestAppliedMigration = await getAppMigrationVersion(appId) + + const migrated = latestAppliedMigration === getLatestMigrationId() + + ctx.body = { migrated } + ctx.status = 200 +} diff --git a/packages/server/src/api/routes/migrations.ts b/packages/server/src/api/routes/migrations.ts index f530647c78..918b197de2 100644 --- a/packages/server/src/api/routes/migrations.ts +++ b/packages/server/src/api/routes/migrations.ts @@ -11,4 +11,6 @@ router auth.internalApi, migrationsController.fetchDefinitions ) + .get("/api/migrations/status", migrationsController.getMigrationStatus) + export default router diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts index a4ffe64604..b382d8b533 100644 --- a/packages/server/src/appMigrations/index.ts +++ b/packages/server/src/appMigrations/index.ts @@ -1,6 +1,9 @@ import queue from "./queue" +import { Next } from "koa" import { getAppMigrationVersion } from "./appMigrationMetadata" import { MIGRATIONS } from "./migrations" +import { UserCtx } from "@budibase/types" +import { Header } from "@budibase/backend-core" export * from "./appMigrationMetadata" @@ -9,14 +12,20 @@ export type AppMigration = { func: () => Promise } -export const latestMigration = MIGRATIONS.map(m => m.id) - .sort() - .reverse()[0] +export const getLatestMigrationId = () => + MIGRATIONS.map(m => m.id) + .sort() + .reverse()[0] const getTimestamp = (versionId: string) => versionId?.split("_")[0] -export async function checkMissingMigrations(appId: string) { +export async function checkMissingMigrations( + ctx: UserCtx, + next: Next, + appId: string +) { const currentVersion = await getAppMigrationVersion(appId) + const latestMigration = getLatestMigrationId() if (getTimestamp(currentVersion) < getTimestamp(latestMigration)) { await queue.add( @@ -29,5 +38,9 @@ export async function checkMissingMigrations(appId: string) { removeOnFail: true, } ) + + ctx.response.set(Header.MIGRATING_APP, appId) } + + return next() } diff --git a/packages/server/src/appMigrations/tests/migrations.integrity.spec.ts b/packages/server/src/appMigrations/tests/migrations.integrity.spec.ts new file mode 100644 index 0000000000..145a06d7f5 --- /dev/null +++ b/packages/server/src/appMigrations/tests/migrations.integrity.spec.ts @@ -0,0 +1,25 @@ +import { context } from "@budibase/backend-core" +import * as setup from "../../api/routes/tests/utilities" +import * as migrations from "../migrations" + +describe("migration integrity", () => { + // These test is checking that each migration is "idempotent". + // We should be able to rerun any migration, with any rerun not modifiying anything. The code should be aware that the migration already ran + it("each migration can rerun safely", async () => { + const config = setup.getConfig() + await config.init() + + await config.doInContext(config.getAppId(), async () => { + const db = context.getAppDB() + for (const migration of migrations.MIGRATIONS) { + await migration.func() + const docs = await db.allDocs({ include_docs: true }) + + await migration.func() + const latestDocs = await db.allDocs({ include_docs: true }) + + expect(docs).toEqual(latestDocs) + } + }) + }) +}) diff --git a/packages/server/src/appMigrations/tests/migrations.spec.ts b/packages/server/src/appMigrations/tests/migrations.spec.ts index 9d80cc5f99..5eb8535695 100644 --- a/packages/server/src/appMigrations/tests/migrations.spec.ts +++ b/packages/server/src/appMigrations/tests/migrations.spec.ts @@ -1,25 +1,53 @@ -import { context } from "@budibase/backend-core" +import { Header } from "@budibase/backend-core" import * as setup from "../../api/routes/tests/utilities" -import { MIGRATIONS } from "../migrations" +import * as migrations from "../migrations" +import { getAppMigrationVersion } from "../appMigrationMetadata" -describe("migration", () => { - // These test is checking that each migration is "idempotent". - // We should be able to rerun any migration, with any rerun not modifiying anything. The code should be aware that the migration already ran - it("each migration can rerun safely", async () => { +jest.mock("../migrations", () => ({ + MIGRATIONS: [ + { + id: "20231211101320_test", + func: async () => {}, + }, + ], +})) + +describe("migrations", () => { + it("new apps are created with the latest app migration version set", async () => { const config = setup.getConfig() await config.init() await config.doInContext(config.getAppId(), async () => { - const db = context.getAppDB() - for (const migration of MIGRATIONS) { - await migration.func() - const docs = await db.allDocs({ include_docs: true }) + const migrationVersion = await getAppMigrationVersion(config.getAppId()) - await migration.func() - const latestDocs = await db.allDocs({ include_docs: true }) - - expect(docs).toEqual(latestDocs) - } + expect(migrationVersion).toEqual("20231211101320_test") }) }) + + it("accessing an app that has no pending migrations will not attach the migrating header", async () => { + const config = setup.getConfig() + await config.init() + + const appId = config.getAppId() + + const response = await config.api.application.getRaw(appId) + + expect(response.headers[Header.MIGRATING_APP]).toBeUndefined() + }) + + it("accessing an app that has pending migrations will attach the migrating header", async () => { + const config = setup.getConfig() + await config.init() + + const appId = config.getAppId() + + migrations.MIGRATIONS.push({ + id: "20231211105812_new-test", + func: async () => {}, + }) + + const response = await config.api.application.getRaw(appId) + + expect(response.headers[Header.MIGRATING_APP]).toEqual(appId) + }) }) diff --git a/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts b/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts index 189f6c068b..3b8e90b526 100644 --- a/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts +++ b/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts @@ -4,12 +4,14 @@ import { getAppMigrationVersion } from "../appMigrationMetadata" import { context } from "@budibase/backend-core" import { AppMigration } from ".." +const futureTimestamp = `20500101174029` + describe("migrationsProcessor", () => { it("running migrations will update the latest applied migration", async () => { const testMigrations: AppMigration[] = [ - { id: "123", func: async () => {} }, - { id: "124", func: async () => {} }, - { id: "125", func: async () => {} }, + { id: `${futureTimestamp}_123`, func: async () => {} }, + { id: `${futureTimestamp}_124`, func: async () => {} }, + { id: `${futureTimestamp}_125`, func: async () => {} }, ] const config = setup.getConfig() @@ -23,13 +25,13 @@ describe("migrationsProcessor", () => { expect( await config.doInContext(appId, () => getAppMigrationVersion(appId)) - ).toBe("125") + ).toBe(`${futureTimestamp}_125`) }) it("no context can be initialised within a migration", async () => { const testMigrations: AppMigration[] = [ { - id: "123", + id: `${futureTimestamp}_123`, func: async () => { await context.doInAppMigrationContext("any", () => {}) }, diff --git a/packages/server/src/middleware/appMigrations.ts b/packages/server/src/middleware/appMigrations.ts index a94b8823e8..36e021c7ed 100644 --- a/packages/server/src/middleware/appMigrations.ts +++ b/packages/server/src/middleware/appMigrations.ts @@ -8,7 +8,5 @@ export default async (ctx: UserCtx, next: any) => { return next() } - await checkMissingMigrations(appId) - - return next() + return checkMissingMigrations(ctx, next, appId) } diff --git a/packages/server/src/tests/utilities/api/application.ts b/packages/server/src/tests/utilities/api/application.ts index 85bc4e4173..9c784bade1 100644 --- a/packages/server/src/tests/utilities/api/application.ts +++ b/packages/server/src/tests/utilities/api/application.ts @@ -1,3 +1,4 @@ +import { Response } from "supertest" import { App } from "@budibase/types" import TestConfiguration from "../TestConfiguration" import { TestAPI } from "./base" @@ -7,12 +8,17 @@ export class ApplicationAPI extends TestAPI { super(config) } - get = async (appId: string): Promise => { + getRaw = async (appId: string): Promise => { const result = await this.request .get(`/api/applications/${appId}/appPackage`) .set(this.config.defaultHeaders()) .expect("Content-Type", /json/) .expect(200) + return result + } + + get = async (appId: string): Promise => { + const result = await this.getRaw(appId) return result.body.application as App } } diff --git a/packages/shared-core/src/constants/api.ts b/packages/shared-core/src/constants/api.ts new file mode 100644 index 0000000000..d6633649e6 --- /dev/null +++ b/packages/shared-core/src/constants/api.ts @@ -0,0 +1,19 @@ +export enum Header { + API_KEY = "x-budibase-api-key", + LICENSE_KEY = "x-budibase-license-key", + API_VER = "x-budibase-api-version", + APP_ID = "x-budibase-app-id", + SESSION_ID = "x-budibase-session-id", + TYPE = "x-budibase-type", + PREVIEW_ROLE = "x-budibase-role", + TENANT_ID = "x-budibase-tenant-id", + VERIFICATION_CODE = "x-budibase-verification-code", + RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code", + RESET_PASSWORD_CODE = "x-budibase-reset-password-code", + RETURN_RESET_PASSWORD_CODE = "x-budibase-return-reset-password-code", + TOKEN = "x-budibase-token", + CSRF_TOKEN = "x-csrf-token", + CORRELATION_ID = "x-budibase-correlation-id", + AUTHORIZATION = "authorization", + MIGRATING_APP = "x-budibase-migrating-app", +} diff --git a/packages/shared-core/src/constants.ts b/packages/shared-core/src/constants/index.ts similarity index 99% rename from packages/shared-core/src/constants.ts rename to packages/shared-core/src/constants/index.ts index 0787b8bed1..a23913dd11 100644 --- a/packages/shared-core/src/constants.ts +++ b/packages/shared-core/src/constants/index.ts @@ -1,3 +1,5 @@ +export * from "./api" + export const OperatorOptions = { Equals: { value: "equal",