From 05b56925209eb4667276eb09c63dc9b7883bfe54 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 11 Dec 2023 10:47:53 +0100 Subject: [PATCH 01/28] Add app migration test --- .../src/appMigrations/tests/migrations.spec.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/server/src/appMigrations/tests/migrations.spec.ts b/packages/server/src/appMigrations/tests/migrations.spec.ts index 9d80cc5f99..9b1a0a9dd8 100644 --- a/packages/server/src/appMigrations/tests/migrations.spec.ts +++ b/packages/server/src/appMigrations/tests/migrations.spec.ts @@ -1,8 +1,21 @@ import { context } from "@budibase/backend-core" import * as setup from "../../api/routes/tests/utilities" import { MIGRATIONS } from "../migrations" +import { getAppMigrationVersion } from "../appMigrationMetadata" +import { latestMigration } from ".." + +describe("migrations", () => { + it("new apps are created with the latest app migration version set", async () => { + const config = setup.getConfig() + await config.init() + + await config.doInContext(config.getAppId(), async () => { + const migrationVersion = await getAppMigrationVersion(config.getAppId()) + + expect(migrationVersion).toEqual(latestMigration) + }) + }) -describe("migration", () => { // These test is checking that each migration is "idempotent". // We should be able to rerun any migration, with any rerun not modifiying anything. The code should be aware that the migration already ran it("each migration can rerun safely", async () => { From 34d3edc2dbc467d55347fa53b912dfa587347b73 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 11 Dec 2023 10:50:13 +0100 Subject: [PATCH 02/28] Add header --- packages/backend-core/src/constants/misc.ts | 1 + packages/server/src/appMigrations/index.ts | 13 ++++++++++++- packages/server/src/middleware/appMigrations.ts | 4 +--- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts index 8ef34196ed..139bd7c0d3 100644 --- a/packages/backend-core/src/constants/misc.ts +++ b/packages/backend-core/src/constants/misc.ts @@ -28,6 +28,7 @@ export enum Header { CSRF_TOKEN = "x-csrf-token", CORRELATION_ID = "x-budibase-correlation-id", AUTHORIZATION = "authorization", + MIGRATING_APP = "x-budibase-migrating-app", } export enum GlobalRole { diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts index a4ffe64604..9bc70f0abc 100644 --- a/packages/server/src/appMigrations/index.ts +++ b/packages/server/src/appMigrations/index.ts @@ -1,6 +1,9 @@ import queue from "./queue" +import { Next } from "koa" import { getAppMigrationVersion } from "./appMigrationMetadata" import { MIGRATIONS } from "./migrations" +import { UserCtx } from "@budibase/types" +import { Header } from "@budibase/backend-core" export * from "./appMigrationMetadata" @@ -15,7 +18,11 @@ export const latestMigration = MIGRATIONS.map(m => m.id) const getTimestamp = (versionId: string) => versionId?.split("_")[0] -export async function checkMissingMigrations(appId: string) { +export async function checkMissingMigrations( + ctx: UserCtx, + next: Next, + appId: string +) { const currentVersion = await getAppMigrationVersion(appId) if (getTimestamp(currentVersion) < getTimestamp(latestMigration)) { @@ -29,5 +36,9 @@ export async function checkMissingMigrations(appId: string) { removeOnFail: true, } ) + + ctx.response.set(Header.MIGRATING_APP, appId) } + + return next() } diff --git a/packages/server/src/middleware/appMigrations.ts b/packages/server/src/middleware/appMigrations.ts index a94b8823e8..36e021c7ed 100644 --- a/packages/server/src/middleware/appMigrations.ts +++ b/packages/server/src/middleware/appMigrations.ts @@ -8,7 +8,5 @@ export default async (ctx: UserCtx, next: any) => { return next() } - await checkMissingMigrations(appId) - - return next() + return checkMissingMigrations(ctx, next, appId) } From 9396c2fd6a5d5f67441fe889ed88e2940d0260ee Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 11 Dec 2023 13:05:30 +0100 Subject: [PATCH 03/28] Split tests --- .../tests/migrations.integrity.spec.ts | 25 ++++++++++++ .../appMigrations/tests/migrations.spec.ts | 38 +++++++------------ 2 files changed, 39 insertions(+), 24 deletions(-) create mode 100644 packages/server/src/appMigrations/tests/migrations.integrity.spec.ts diff --git a/packages/server/src/appMigrations/tests/migrations.integrity.spec.ts b/packages/server/src/appMigrations/tests/migrations.integrity.spec.ts new file mode 100644 index 0000000000..145a06d7f5 --- /dev/null +++ b/packages/server/src/appMigrations/tests/migrations.integrity.spec.ts @@ -0,0 +1,25 @@ +import { context } from "@budibase/backend-core" +import * as setup from "../../api/routes/tests/utilities" +import * as migrations from "../migrations" + +describe("migration integrity", () => { + // These test is checking that each migration is "idempotent". + // We should be able to rerun any migration, with any rerun not modifiying anything. The code should be aware that the migration already ran + it("each migration can rerun safely", async () => { + const config = setup.getConfig() + await config.init() + + await config.doInContext(config.getAppId(), async () => { + const db = context.getAppDB() + for (const migration of migrations.MIGRATIONS) { + await migration.func() + const docs = await db.allDocs({ include_docs: true }) + + await migration.func() + const latestDocs = await db.allDocs({ include_docs: true }) + + expect(docs).toEqual(latestDocs) + } + }) + }) +}) diff --git a/packages/server/src/appMigrations/tests/migrations.spec.ts b/packages/server/src/appMigrations/tests/migrations.spec.ts index 9b1a0a9dd8..d27a1ac37e 100644 --- a/packages/server/src/appMigrations/tests/migrations.spec.ts +++ b/packages/server/src/appMigrations/tests/migrations.spec.ts @@ -1,8 +1,18 @@ -import { context } from "@budibase/backend-core" import * as setup from "../../api/routes/tests/utilities" -import { MIGRATIONS } from "../migrations" +import * as migrations from "../migrations" import { getAppMigrationVersion } from "../appMigrationMetadata" -import { latestMigration } from ".." +import { AppMigration } from ".." + +const mockedMigrations: AppMigration[] = [ + { + id: "20231211101320_test", + func: async () => {}, + }, +] + +jest.doMock("../migrations", () => ({ + MIGRATIONS: mockedMigrations, +})) describe("migrations", () => { it("new apps are created with the latest app migration version set", async () => { @@ -12,27 +22,7 @@ describe("migrations", () => { await config.doInContext(config.getAppId(), async () => { const migrationVersion = await getAppMigrationVersion(config.getAppId()) - expect(migrationVersion).toEqual(latestMigration) - }) - }) - - // These test is checking that each migration is "idempotent". - // We should be able to rerun any migration, with any rerun not modifiying anything. The code should be aware that the migration already ran - it("each migration can rerun safely", async () => { - const config = setup.getConfig() - await config.init() - - await config.doInContext(config.getAppId(), async () => { - const db = context.getAppDB() - for (const migration of MIGRATIONS) { - await migration.func() - const docs = await db.allDocs({ include_docs: true }) - - await migration.func() - const latestDocs = await db.allDocs({ include_docs: true }) - - expect(docs).toEqual(latestDocs) - } + expect(migrationVersion).toEqual("20231211101320_test") }) }) }) From 2a92263df5cd106ee444a3038f025e61ce962a38 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 11 Dec 2023 13:19:07 +0100 Subject: [PATCH 04/28] Add tests --- .../appMigrations/tests/migrations.spec.ts | 45 ++++++++++++++----- .../src/tests/utilities/api/application.ts | 8 +++- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/packages/server/src/appMigrations/tests/migrations.spec.ts b/packages/server/src/appMigrations/tests/migrations.spec.ts index d27a1ac37e..5eb8535695 100644 --- a/packages/server/src/appMigrations/tests/migrations.spec.ts +++ b/packages/server/src/appMigrations/tests/migrations.spec.ts @@ -1,17 +1,15 @@ +import { Header } from "@budibase/backend-core" import * as setup from "../../api/routes/tests/utilities" import * as migrations from "../migrations" import { getAppMigrationVersion } from "../appMigrationMetadata" -import { AppMigration } from ".." -const mockedMigrations: AppMigration[] = [ - { - id: "20231211101320_test", - func: async () => {}, - }, -] - -jest.doMock("../migrations", () => ({ - MIGRATIONS: mockedMigrations, +jest.mock("../migrations", () => ({ + MIGRATIONS: [ + { + id: "20231211101320_test", + func: async () => {}, + }, + ], })) describe("migrations", () => { @@ -25,4 +23,31 @@ describe("migrations", () => { expect(migrationVersion).toEqual("20231211101320_test") }) }) + + it("accessing an app that has no pending migrations will not attach the migrating header", async () => { + const config = setup.getConfig() + await config.init() + + const appId = config.getAppId() + + const response = await config.api.application.getRaw(appId) + + expect(response.headers[Header.MIGRATING_APP]).toBeUndefined() + }) + + it("accessing an app that has pending migrations will attach the migrating header", async () => { + const config = setup.getConfig() + await config.init() + + const appId = config.getAppId() + + migrations.MIGRATIONS.push({ + id: "20231211105812_new-test", + func: async () => {}, + }) + + const response = await config.api.application.getRaw(appId) + + expect(response.headers[Header.MIGRATING_APP]).toEqual(appId) + }) }) diff --git a/packages/server/src/tests/utilities/api/application.ts b/packages/server/src/tests/utilities/api/application.ts index 85bc4e4173..9c784bade1 100644 --- a/packages/server/src/tests/utilities/api/application.ts +++ b/packages/server/src/tests/utilities/api/application.ts @@ -1,3 +1,4 @@ +import { Response } from "supertest" import { App } from "@budibase/types" import TestConfiguration from "../TestConfiguration" import { TestAPI } from "./base" @@ -7,12 +8,17 @@ export class ApplicationAPI extends TestAPI { super(config) } - get = async (appId: string): Promise => { + getRaw = async (appId: string): Promise => { const result = await this.request .get(`/api/applications/${appId}/appPackage`) .set(this.config.defaultHeaders()) .expect("Content-Type", /json/) .expect(200) + return result + } + + get = async (appId: string): Promise => { + const result = await this.getRaw(appId) return result.body.application as App } } From 144fbdf94676d8db3ad52702d289886497177e4b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 11 Dec 2023 13:30:04 +0100 Subject: [PATCH 05/28] Add tests --- packages/server/src/api/controllers/application.ts | 2 +- packages/server/src/appMigrations/index.ts | 8 +++++--- .../appMigrations/tests/migrationsProcessor.spec.ts | 12 +++++++----- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index bb4f447f79..70298c7172 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -340,7 +340,7 @@ async function performAppCreate(ctx: UserCtx) { // Initialise the app migration version as the latest one await appMigrations.updateAppMigrationMetadata({ appId, - version: appMigrations.latestMigration, + version: appMigrations.getLatestMigrationId(), }) await cache.app.invalidateAppMetadata(appId, newApplication) diff --git a/packages/server/src/appMigrations/index.ts b/packages/server/src/appMigrations/index.ts index 9bc70f0abc..b382d8b533 100644 --- a/packages/server/src/appMigrations/index.ts +++ b/packages/server/src/appMigrations/index.ts @@ -12,9 +12,10 @@ export type AppMigration = { func: () => Promise } -export const latestMigration = MIGRATIONS.map(m => m.id) - .sort() - .reverse()[0] +export const getLatestMigrationId = () => + MIGRATIONS.map(m => m.id) + .sort() + .reverse()[0] const getTimestamp = (versionId: string) => versionId?.split("_")[0] @@ -24,6 +25,7 @@ export async function checkMissingMigrations( appId: string ) { const currentVersion = await getAppMigrationVersion(appId) + const latestMigration = getLatestMigrationId() if (getTimestamp(currentVersion) < getTimestamp(latestMigration)) { await queue.add( diff --git a/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts b/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts index 189f6c068b..3b8e90b526 100644 --- a/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts +++ b/packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts @@ -4,12 +4,14 @@ import { getAppMigrationVersion } from "../appMigrationMetadata" import { context } from "@budibase/backend-core" import { AppMigration } from ".." +const futureTimestamp = `20500101174029` + describe("migrationsProcessor", () => { it("running migrations will update the latest applied migration", async () => { const testMigrations: AppMigration[] = [ - { id: "123", func: async () => {} }, - { id: "124", func: async () => {} }, - { id: "125", func: async () => {} }, + { id: `${futureTimestamp}_123`, func: async () => {} }, + { id: `${futureTimestamp}_124`, func: async () => {} }, + { id: `${futureTimestamp}_125`, func: async () => {} }, ] const config = setup.getConfig() @@ -23,13 +25,13 @@ describe("migrationsProcessor", () => { expect( await config.doInContext(appId, () => getAppMigrationVersion(appId)) - ).toBe("125") + ).toBe(`${futureTimestamp}_125`) }) it("no context can be initialised within a migration", async () => { const testMigrations: AppMigration[] = [ { - id: "123", + id: `${futureTimestamp}_123`, func: async () => { await context.doInAppMigrationContext("any", () => {}) }, From f3e046ba9824a1e8685e534562d54e6cda584815 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 12 Dec 2023 10:21:10 +0100 Subject: [PATCH 06/28] Move header constants to shared-core --- packages/backend-core/src/constants/misc.ts | 20 +------------------ packages/shared-core/src/constants/api.ts | 19 ++++++++++++++++++ .../src/{constants.ts => constants/index.ts} | 2 ++ 3 files changed, 22 insertions(+), 19 deletions(-) create mode 100644 packages/shared-core/src/constants/api.ts rename packages/shared-core/src/{constants.ts => constants/index.ts} (99%) diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts index 139bd7c0d3..aee099e10a 100644 --- a/packages/backend-core/src/constants/misc.ts +++ b/packages/backend-core/src/constants/misc.ts @@ -11,25 +11,7 @@ export enum Cookie { OIDC_CONFIG = "budibase:oidc:config", } -export enum Header { - API_KEY = "x-budibase-api-key", - LICENSE_KEY = "x-budibase-license-key", - API_VER = "x-budibase-api-version", - APP_ID = "x-budibase-app-id", - SESSION_ID = "x-budibase-session-id", - TYPE = "x-budibase-type", - PREVIEW_ROLE = "x-budibase-role", - TENANT_ID = "x-budibase-tenant-id", - VERIFICATION_CODE = "x-budibase-verification-code", - RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code", - RESET_PASSWORD_CODE = "x-budibase-reset-password-code", - RETURN_RESET_PASSWORD_CODE = "x-budibase-return-reset-password-code", - TOKEN = "x-budibase-token", - CSRF_TOKEN = "x-csrf-token", - CORRELATION_ID = "x-budibase-correlation-id", - AUTHORIZATION = "authorization", - MIGRATING_APP = "x-budibase-migrating-app", -} +export { Header } from "@budibase/shared-core" export enum GlobalRole { OWNER = "owner", diff --git a/packages/shared-core/src/constants/api.ts b/packages/shared-core/src/constants/api.ts new file mode 100644 index 0000000000..d6633649e6 --- /dev/null +++ b/packages/shared-core/src/constants/api.ts @@ -0,0 +1,19 @@ +export enum Header { + API_KEY = "x-budibase-api-key", + LICENSE_KEY = "x-budibase-license-key", + API_VER = "x-budibase-api-version", + APP_ID = "x-budibase-app-id", + SESSION_ID = "x-budibase-session-id", + TYPE = "x-budibase-type", + PREVIEW_ROLE = "x-budibase-role", + TENANT_ID = "x-budibase-tenant-id", + VERIFICATION_CODE = "x-budibase-verification-code", + RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code", + RESET_PASSWORD_CODE = "x-budibase-reset-password-code", + RETURN_RESET_PASSWORD_CODE = "x-budibase-return-reset-password-code", + TOKEN = "x-budibase-token", + CSRF_TOKEN = "x-csrf-token", + CORRELATION_ID = "x-budibase-correlation-id", + AUTHORIZATION = "authorization", + MIGRATING_APP = "x-budibase-migrating-app", +} diff --git a/packages/shared-core/src/constants.ts b/packages/shared-core/src/constants/index.ts similarity index 99% rename from packages/shared-core/src/constants.ts rename to packages/shared-core/src/constants/index.ts index 0787b8bed1..a23913dd11 100644 --- a/packages/shared-core/src/constants.ts +++ b/packages/shared-core/src/constants/index.ts @@ -1,3 +1,5 @@ +export * from "./api" + export const OperatorOptions = { Equals: { value: "equal", From 005dd2766366dee4d80b6b40a96a88b3ffef3797 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 12 Dec 2023 12:06:55 +0100 Subject: [PATCH 07/28] Use header consts from shared-core --- packages/frontend-core/src/api/index.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/frontend-core/src/api/index.js b/packages/frontend-core/src/api/index.js index aefc3522a7..85296f3121 100644 --- a/packages/frontend-core/src/api/index.js +++ b/packages/frontend-core/src/api/index.js @@ -1,4 +1,5 @@ import { Helpers } from "@budibase/bbui" +import { Header } from "@budibase/shared-core" import { ApiVersion } from "../constants" import { buildAnalyticsEndpoints } from "./analytics" import { buildAppEndpoints } from "./app" @@ -133,9 +134,9 @@ export const createAPIClient = config => { // Build headers let headers = { Accept: "application/json" } - headers["x-budibase-session-id"] = APISessionID + headers[Header.SESSION_ID] = APISessionID if (!external) { - headers["x-budibase-api-version"] = ApiVersion + headers[Header.API_VER] = ApiVersion } if (json) { headers["Content-Type"] = "application/json" @@ -242,7 +243,7 @@ export const createAPIClient = config => { getAppID: () => { let headers = {} config?.attachHeaders(headers) - return headers?.["x-budibase-app-id"] + return headers?.[Header.APP_ID] }, } From 50da7992168bf12b4dc5645dbb792e5fed692a8e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 12 Dec 2023 13:42:01 +0100 Subject: [PATCH 08/28] Add onMigrationDetected function --- packages/frontend-core/src/api/index.js | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/frontend-core/src/api/index.js b/packages/frontend-core/src/api/index.js index 85296f3121..d4b4f3636e 100644 --- a/packages/frontend-core/src/api/index.js +++ b/packages/frontend-core/src/api/index.js @@ -63,6 +63,11 @@ const defaultAPIClientConfig = { * invoked before the actual JS error is thrown up the stack. */ onError: null, + + /** + * A function can be passed to be called when an API call returns info about a migration running for a specific app + */ + onMigrationDetected: null, } /** @@ -171,6 +176,7 @@ export const createAPIClient = config => { // Handle response if (response.status >= 200 && response.status < 400) { + handleMigrations(response) try { if (parseResponse) { return await parseResponse(response) @@ -187,7 +193,18 @@ export const createAPIClient = config => { } } - // Performs an API call to the server and caches the response. + const handleMigrations = response => { + if (!config.onMigrationDetected) { + return + } + const migration = response.headers.get(Header.MIGRATING_APP) + + if (migration) { + config.onMigrationDetected(migration) + } + } + + // Performs an API call to the server and caches the response. // Future invocation for this URL will return the cached result instead of // hitting the server again. const makeCachedApiCall = async params => { From 46f8f4da58d844c2022d006d9c2603c59e36ac4e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 12 Dec 2023 18:37:03 +0100 Subject: [PATCH 09/28] Fix parsing appid from path with ? --- packages/backend-core/src/utils/utils.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index ee1ef6da0c..f55bd44073 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -127,7 +127,10 @@ function parseAppIdFromUrl(url?: string) { if (!url) { return } - return url.split("/").find(subPath => subPath.startsWith(APP_PREFIX)) + return url + .split("?")[0] + .split("/") + .find(subPath => subPath.startsWith(APP_PREFIX)) } /** From 165d86c24678229607abc4e127b9b137100b1569 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 12 Dec 2023 18:38:29 +0100 Subject: [PATCH 10/28] Create updating page --- .../server/src/api/controllers/migrations.ts | 26 ++++++++++++++++--- packages/server/src/api/routes/migrations.ts | 2 ++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/migrations.ts b/packages/server/src/api/controllers/migrations.ts index 8f1bfa22db..74e573dc6a 100644 --- a/packages/server/src/api/controllers/migrations.ts +++ b/packages/server/src/api/controllers/migrations.ts @@ -1,14 +1,34 @@ +import { context } from "@budibase/backend-core" import { migrate as migrationImpl, MIGRATIONS } from "../../migrations" -import { BBContext } from "@budibase/types" +import { Ctx } from "@budibase/types" +import { + getAppMigrationVersion, + getLatestMigrationId, +} from "../../appMigrations" -export async function migrate(ctx: BBContext) { +export async function migrate(ctx: Ctx) { const options = ctx.request.body // don't await as can take a while, just return migrationImpl(options) ctx.status = 200 } -export async function fetchDefinitions(ctx: BBContext) { +export async function fetchDefinitions(ctx: Ctx) { ctx.body = MIGRATIONS ctx.status = 200 } + +export async function migrationCompleted(ctx: Ctx) { + const appId = context.getAppId() + + if (!appId) { + ctx.throw("AppId could not be found") + } + + const latestAppliedMigration = await getAppMigrationVersion(appId) + + const migrated = latestAppliedMigration === getLatestMigrationId() + + ctx.body = { migrated } + ctx.status = 200 +} diff --git a/packages/server/src/api/routes/migrations.ts b/packages/server/src/api/routes/migrations.ts index f530647c78..8f5cc63f82 100644 --- a/packages/server/src/api/routes/migrations.ts +++ b/packages/server/src/api/routes/migrations.ts @@ -11,4 +11,6 @@ router auth.internalApi, migrationsController.fetchDefinitions ) + .get("/api/migrations/status", migrationsController.migrationCompleted) + export default router From 251663f38c62eb4de2fce99c6ed4741ec543ebab Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 13 Dec 2023 09:57:12 +0100 Subject: [PATCH 11/28] Renames --- packages/server/src/api/controllers/migrations.ts | 2 +- packages/server/src/api/routes/migrations.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/controllers/migrations.ts b/packages/server/src/api/controllers/migrations.ts index 74e573dc6a..c8f786578d 100644 --- a/packages/server/src/api/controllers/migrations.ts +++ b/packages/server/src/api/controllers/migrations.ts @@ -18,7 +18,7 @@ export async function fetchDefinitions(ctx: Ctx) { ctx.status = 200 } -export async function migrationCompleted(ctx: Ctx) { +export async function getMigrationStatus(ctx: Ctx) { const appId = context.getAppId() if (!appId) { diff --git a/packages/server/src/api/routes/migrations.ts b/packages/server/src/api/routes/migrations.ts index 8f5cc63f82..918b197de2 100644 --- a/packages/server/src/api/routes/migrations.ts +++ b/packages/server/src/api/routes/migrations.ts @@ -11,6 +11,6 @@ router auth.internalApi, migrationsController.fetchDefinitions ) - .get("/api/migrations/status", migrationsController.migrationCompleted) + .get("/api/migrations/status", migrationsController.getMigrationStatus) export default router From a0dd71f990d80bf9efb2b4b4114dbb2922ad9528 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 10:31:04 +0100 Subject: [PATCH 12/28] Renames --- packages/backend-core/src/utils/utils.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index f55bd44073..0554737518 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -96,7 +96,7 @@ export async function getAppIdFromCtx(ctx: Ctx) { } // look in the path - const pathId = parseAppIdFromUrl(ctx.path) + const pathId = parseAppIdFromUrlPath(ctx.path) if (!appId && pathId) { appId = confirmAppId(pathId) } @@ -116,19 +116,19 @@ export async function getAppIdFromCtx(ctx: Ctx) { // referer header is present from a builder redirect const referer = ctx.request.headers.referer if (!appId && referer?.includes(BUILDER_APP_PREFIX)) { - const refererId = parseAppIdFromUrl(ctx.request.headers.referer) + const refererId = parseAppIdFromUrlPath(ctx.request.headers.referer) appId = confirmAppId(refererId) } return appId } -function parseAppIdFromUrl(url?: string) { +function parseAppIdFromUrlPath(url?: string) { if (!url) { return } return url - .split("?")[0] + .split("?")[0] // Remove any possible query string .split("/") .find(subPath => subPath.startsWith(APP_PREFIX)) } From 46abfdd1eab1277add97b71107a155bf731b293b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 10:48:57 +0100 Subject: [PATCH 13/28] Add label for deployments --- .github/workflows/deploy-featurebranch.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index c70f2fff20..b572908505 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -7,7 +7,10 @@ on: jobs: release: - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' + if: | + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == 'Budibase/budibase'|| + contains(github.event.issue.labels.*.name, 'feature-branch') runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 5f62004675c292805a513977859c2fcda62aeed9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 10:52:29 +0100 Subject: [PATCH 14/28] Fix condition --- .github/workflows/deploy-featurebranch.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index b572908505..9bbab42436 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -8,8 +8,7 @@ on: jobs: release: if: | - github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == 'Budibase/budibase'|| + (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') && contains(github.event.issue.labels.*.name, 'feature-branch') runs-on: ubuntu-latest steps: From 934611029fd51a0f0976d33c3921530a6f62b984 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 11:04:28 +0100 Subject: [PATCH 15/28] Add types --- .github/workflows/deploy-featurebranch.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index 9bbab42436..d177d88a2f 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -2,6 +2,13 @@ name: deploy-featurebranch on: pull_request: + types: [ + labeled, + # default types + opened, + synchronize, + reopened, + ] branches: - master From 651b42263be270282dee6689a548b109ab369ea4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 11:05:05 +0100 Subject: [PATCH 16/28] Add comment --- .github/workflows/deploy-featurebranch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index d177d88a2f..b3463b4ccd 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -4,7 +4,7 @@ on: pull_request: types: [ labeled, - # default types + # default types (https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request) opened, synchronize, reopened, From ad7be462faae3d34308b08eaba932137ddfc773f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 11:09:04 +0100 Subject: [PATCH 17/28] Fix --- .github/workflows/deploy-featurebranch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index b3463b4ccd..10d57bb6e3 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -16,7 +16,7 @@ jobs: release: if: | (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') && - contains(github.event.issue.labels.*.name, 'feature-branch') + contains(github.event.pull_request.labels.*.name, 'feature-branch') runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 4e934a56b7c1bd45822a6bae0fa168439876fec1 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 11:14:05 +0100 Subject: [PATCH 18/28] Delete fb on label removed --- .github/workflows/close-featurebranch.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/close-featurebranch.yml b/.github/workflows/close-featurebranch.yml index 46cb781730..20bf2af5d3 100644 --- a/.github/workflows/close-featurebranch.yml +++ b/.github/workflows/close-featurebranch.yml @@ -2,9 +2,7 @@ name: close-featurebranch on: pull_request: - types: [closed] - branches: - - master + types: [closed, unlabeled] workflow_dispatch: inputs: BRANCH: @@ -14,6 +12,9 @@ on: jobs: release: + if: | + (contains(github.event.pull_request.labels.*.name, 'feature-branch') || + contains(github.event.changes.labels.removed.*.name, 'feature-branch')) runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From f1190c47f8aed73fac56c805b18e64a65826d00a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 11:14:26 +0100 Subject: [PATCH 19/28] Do not check base on fb --- .github/workflows/deploy-featurebranch.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index 10d57bb6e3..a5636fe912 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -4,13 +4,11 @@ on: pull_request: types: [ labeled, - # default types (https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request) + # default types below (https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request) opened, synchronize, reopened, ] - branches: - - master jobs: release: From c07914a0bf6cdf9ede536da37e58ad60f61a5411 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 11:18:35 +0100 Subject: [PATCH 20/28] Attempt close --- .github/workflows/close-featurebranch.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/close-featurebranch.yml b/.github/workflows/close-featurebranch.yml index 20bf2af5d3..e805cff0e0 100644 --- a/.github/workflows/close-featurebranch.yml +++ b/.github/workflows/close-featurebranch.yml @@ -13,8 +13,8 @@ on: jobs: release: if: | - (contains(github.event.pull_request.labels.*.name, 'feature-branch') || - contains(github.event.changes.labels.removed.*.name, 'feature-branch')) + contains(github.event.pull_request.labels.*.name, 'feature-branch') || + github.event.label.name == 'feature-branch' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 8542b8d87f61fef28172f441360b67887044d5ca Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Dec 2023 11:26:28 +0100 Subject: [PATCH 21/28] Do check labels --- .github/workflows/close-featurebranch.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/close-featurebranch.yml b/.github/workflows/close-featurebranch.yml index e805cff0e0..5da3eb52cd 100644 --- a/.github/workflows/close-featurebranch.yml +++ b/.github/workflows/close-featurebranch.yml @@ -13,8 +13,8 @@ on: jobs: release: if: | - contains(github.event.pull_request.labels.*.name, 'feature-branch') || - github.event.label.name == 'feature-branch' + (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'feature-branch')) || + github.event.label.name == 'feature-branch' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 7b4ab36e39ad48d93be24e7725f67e9f91dbbc7d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 14 Dec 2023 10:30:17 +0000 Subject: [PATCH 22/28] Update VSCode config to adapt to new format of source.fixAll --- .vscode/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index ece537efac..e22d5a8866 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,7 +1,7 @@ { "editor.formatOnSave": true, "editor.codeActionsOnSave": { - "source.fixAll": true + "source.fixAll": "explicit" }, "editor.defaultFormatter": "esbenp.prettier-vscode", "[json]": { From 598ab03fd070f4c9e2569abbe439d983bff79746 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 14 Dec 2023 10:40:22 +0000 Subject: [PATCH 23/28] Optimise getUniqueByProp --- packages/server/src/db/linkedRows/linkUtils.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts index 5942e7e5a1..0d2e99fc6c 100644 --- a/packages/server/src/db/linkedRows/linkUtils.ts +++ b/packages/server/src/db/linkedRows/linkUtils.ts @@ -99,9 +99,15 @@ export async function getLinkDocuments(args: { } export function getUniqueByProp(array: any[], prop: string) { - return array.filter((obj, pos, arr) => { - return arr.map(mapObj => mapObj[prop]).indexOf(obj[prop]) === pos - }) + const seen = new Set() + const ret = [] + for (const item of array) { + if (!seen.has(item[prop])) { + seen.add(item[prop]) + ret.push(item) + } + } + return ret } export function getLinkedTableIDs(table: Table): string[] { From 0597c0efc6246a2a2388ce9235a42f939d550253 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 14 Dec 2023 11:08:55 +0000 Subject: [PATCH 24/28] Respond to PR feedback. --- packages/server/src/db/linkedRows/linkUtils.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts index 0d2e99fc6c..faaf9c1f21 100644 --- a/packages/server/src/db/linkedRows/linkUtils.ts +++ b/packages/server/src/db/linkedRows/linkUtils.ts @@ -100,14 +100,14 @@ export async function getLinkDocuments(args: { export function getUniqueByProp(array: any[], prop: string) { const seen = new Set() - const ret = [] + const filteredArray = [] for (const item of array) { if (!seen.has(item[prop])) { seen.add(item[prop]) - ret.push(item) + filteredArray.push(item) } } - return ret + return filteredArray } export function getLinkedTableIDs(table: Table): string[] { From 359679dfc4d1233995eb11629df48667e0c02cfe Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 14 Dec 2023 11:34:20 +0000 Subject: [PATCH 25/28] Bump version to 2.13.41 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 5e12b57763..962954db39 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.40", + "version": "2.13.41", "npmClient": "yarn", "packages": [ "packages/*", From 995f24c64f21e551ee14e61c3028ae869e43664a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 14 Dec 2023 15:35:45 +0000 Subject: [PATCH 26/28] Allow cluster internal traffic to hit the nginx status endpoint. --- hosting/proxy/nginx.prod.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/hosting/proxy/nginx.prod.conf b/hosting/proxy/nginx.prod.conf index 88f9645f80..65cc3ff390 100644 --- a/hosting/proxy/nginx.prod.conf +++ b/hosting/proxy/nginx.prod.conf @@ -257,6 +257,7 @@ http { access_log off; allow 127.0.0.1; + allow 10.0.0.0/8; deny all; location /nginx_status { From 7ec2c3860bfc2bb69f39d31a6b775003af39acba Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 14 Dec 2023 16:20:31 +0000 Subject: [PATCH 27/28] Optimise the performance of Writethrough.put. --- packages/backend-core/src/cache/writethrough.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index 24e519dc7f..5cafe418d7 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -56,11 +56,8 @@ async function put( const writeDb = async (toWrite: any) => { // doc should contain the _id and _rev const response = await db.put(toWrite, { force: true }) - output = { - ...doc, - _id: response.id, - _rev: response.rev, - } + output._id = response.id + output._rev = response.rev } try { await writeDb(doc) From e7606125b61647fbac2910bff229fcb9ee61c8cf Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 14 Dec 2023 16:33:31 +0000 Subject: [PATCH 28/28] Bump version to 2.13.42 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 962954db39..7d9ea19476 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.41", + "version": "2.13.42", "npmClient": "yarn", "packages": [ "packages/*",