Merge branch 'master' into optimise-get-unique-by-prod

.vscode/settings.json

@@ -1,7 +1,7 @@
 {
   "editor.formatOnSave": true,
   "editor.codeActionsOnSave": {
-    "source.fixAll": true
+    "source.fixAll": "explicit"
   },
   "editor.defaultFormatter": "esbenp.prettier-vscode",
   "[json]": {

packages/backend-core/src/constants/misc.ts

@@ -11,24 +11,7 @@ export enum Cookie {
   OIDC_CONFIG = "budibase:oidc:config",
 }
 
-export enum Header {
+export { Header } from "@budibase/shared-core"
-  API_KEY = "x-budibase-api-key",
-  LICENSE_KEY = "x-budibase-license-key",
-  API_VER = "x-budibase-api-version",
-  APP_ID = "x-budibase-app-id",
-  SESSION_ID = "x-budibase-session-id",
-  TYPE = "x-budibase-type",
-  PREVIEW_ROLE = "x-budibase-role",
-  TENANT_ID = "x-budibase-tenant-id",
-  VERIFICATION_CODE = "x-budibase-verification-code",
-  RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code",
-  RESET_PASSWORD_CODE = "x-budibase-reset-password-code",
-  RETURN_RESET_PASSWORD_CODE = "x-budibase-return-reset-password-code",
-  TOKEN = "x-budibase-token",
-  CSRF_TOKEN = "x-csrf-token",
-  CORRELATION_ID = "x-budibase-correlation-id",
-  AUTHORIZATION = "authorization",
-}
 
 export enum GlobalRole {
   OWNER = "owner",

packages/backend-core/src/utils/utils.ts

@@ -96,7 +96,7 @@ export async function getAppIdFromCtx(ctx: Ctx) {
   }
 
   // look in the path
-  const pathId = parseAppIdFromUrl(ctx.path)
+  const pathId = parseAppIdFromUrlPath(ctx.path)
   if (!appId && pathId) {
     appId = confirmAppId(pathId)
   }
@@ -116,18 +116,21 @@ export async function getAppIdFromCtx(ctx: Ctx) {
   // referer header is present from a builder redirect
   const referer = ctx.request.headers.referer
   if (!appId && referer?.includes(BUILDER_APP_PREFIX)) {
-    const refererId = parseAppIdFromUrl(ctx.request.headers.referer)
+    const refererId = parseAppIdFromUrlPath(ctx.request.headers.referer)
     appId = confirmAppId(refererId)
   }
 
   return appId
 }
 
-function parseAppIdFromUrl(url?: string) {
+function parseAppIdFromUrlPath(url?: string) {
   if (!url) {
     return
   }
-  return url.split("/").find(subPath => subPath.startsWith(APP_PREFIX))
+  return url
+    .split("?")[0] // Remove any possible query string
+    .split("/")
+    .find(subPath => subPath.startsWith(APP_PREFIX))
 }
 
 /**

packages/frontend-core/src/api/index.js

@@ -1,4 +1,5 @@
 import { Helpers } from "@budibase/bbui"
+import { Header } from "@budibase/shared-core"
 import { ApiVersion } from "../constants"
 import { buildAnalyticsEndpoints } from "./analytics"
 import { buildAppEndpoints } from "./app"
@@ -62,6 +63,11 @@ const defaultAPIClientConfig = {
    * invoked before the actual JS error is thrown up the stack.
    */
   onError: null,
+
+  /**
+   * A function can be passed to be called when an API call returns info about a migration running for a specific app
+   */
+  onMigrationDetected: null,
 }
 
 /**
@@ -133,9 +139,9 @@ export const createAPIClient = config => {
 
     // Build headers
     let headers = { Accept: "application/json" }
-    headers["x-budibase-session-id"] = APISessionID
+    headers[Header.SESSION_ID] = APISessionID
     if (!external) {
-      headers["x-budibase-api-version"] = ApiVersion
+      headers[Header.API_VER] = ApiVersion
     }
     if (json) {
       headers["Content-Type"] = "application/json"
@@ -170,6 +176,7 @@ export const createAPIClient = config => {
 
     // Handle response
     if (response.status >= 200 && response.status < 400) {
+      handleMigrations(response)
       try {
         if (parseResponse) {
           return await parseResponse(response)
@@ -186,6 +193,17 @@ export const createAPIClient = config => {
     }
   }
 
+  const handleMigrations = response => {
+    if (!config.onMigrationDetected) {
+      return
+    }
+    const migration = response.headers.get(Header.MIGRATING_APP)
+
+    if (migration) {
+      config.onMigrationDetected(migration)
+    }
+  }
+
   // Performs an API call to the server  and caches the response.
   // Future invocation for this URL will return the cached result instead of
   // hitting the server again.
@@ -242,7 +260,7 @@ export const createAPIClient = config => {
     getAppID: () => {
       let headers = {}
       config?.attachHeaders(headers)
-      return headers?.["x-budibase-app-id"]
+      return headers?.[Header.APP_ID]
     },
   }
 

packages/server/src/api/controllers/application.ts

@@ -340,7 +340,7 @@ async function performAppCreate(ctx: UserCtx) {
     // Initialise the app migration version as the latest one
     await appMigrations.updateAppMigrationMetadata({
       appId,
-      version: appMigrations.latestMigration,
+      version: appMigrations.getLatestMigrationId(),
     })
 
     await cache.app.invalidateAppMetadata(appId, newApplication)

packages/server/src/api/controllers/migrations.ts

@@ -1,14 +1,34 @@
+import { context } from "@budibase/backend-core"
 import { migrate as migrationImpl, MIGRATIONS } from "../../migrations"
-import { BBContext } from "@budibase/types"
+import { Ctx } from "@budibase/types"
+import {
+  getAppMigrationVersion,
+  getLatestMigrationId,
+} from "../../appMigrations"
 
-export async function migrate(ctx: BBContext) {
+export async function migrate(ctx: Ctx) {
   const options = ctx.request.body
   // don't await as can take a while, just return
   migrationImpl(options)
   ctx.status = 200
 }
 
-export async function fetchDefinitions(ctx: BBContext) {
+export async function fetchDefinitions(ctx: Ctx) {
   ctx.body = MIGRATIONS
   ctx.status = 200
 }
+
+export async function getMigrationStatus(ctx: Ctx) {
+  const appId = context.getAppId()
+
+  if (!appId) {
+    ctx.throw("AppId could not be found")
+  }
+
+  const latestAppliedMigration = await getAppMigrationVersion(appId)
+
+  const migrated = latestAppliedMigration === getLatestMigrationId()
+
+  ctx.body = { migrated }
+  ctx.status = 200
+}

packages/server/src/api/routes/migrations.ts

@@ -11,4 +11,6 @@ router
     auth.internalApi,
     migrationsController.fetchDefinitions
   )
+  .get("/api/migrations/status", migrationsController.getMigrationStatus)
+
 export default router

packages/server/src/appMigrations/index.ts

@@ -1,6 +1,9 @@
 import queue from "./queue"
+import { Next } from "koa"
 import { getAppMigrationVersion } from "./appMigrationMetadata"
 import { MIGRATIONS } from "./migrations"
+import { UserCtx } from "@budibase/types"
+import { Header } from "@budibase/backend-core"
 
 export * from "./appMigrationMetadata"
 
@@ -9,14 +12,20 @@ export type AppMigration = {
   func: () => Promise<void>
 }
 
-export const latestMigration = MIGRATIONS.map(m => m.id)
+export const getLatestMigrationId = () =>
+  MIGRATIONS.map(m => m.id)
     .sort()
     .reverse()[0]
 
 const getTimestamp = (versionId: string) => versionId?.split("_")[0]
 
-export async function checkMissingMigrations(appId: string) {
+export async function checkMissingMigrations(
+  ctx: UserCtx,
+  next: Next,
+  appId: string
+) {
   const currentVersion = await getAppMigrationVersion(appId)
+  const latestMigration = getLatestMigrationId()
 
   if (getTimestamp(currentVersion) < getTimestamp(latestMigration)) {
     await queue.add(
@@ -29,5 +38,9 @@ export async function checkMissingMigrations(appId: string) {
         removeOnFail: true,
       }
     )
+
+    ctx.response.set(Header.MIGRATING_APP, appId)
   }
+
+  return next()
 }

packages/server/src/appMigrations/tests/migrations.integrity.spec.ts

@@ -0,0 +1,25 @@
+import { context } from "@budibase/backend-core"
+import * as setup from "../../api/routes/tests/utilities"
+import * as migrations from "../migrations"
+
+describe("migration integrity", () => {
+  // These test is checking that each migration is "idempotent".
+  // We should be able to rerun any migration, with any rerun not modifiying anything. The code should be aware that the migration already ran
+  it("each migration can rerun safely", async () => {
+    const config = setup.getConfig()
+    await config.init()
+
+    await config.doInContext(config.getAppId(), async () => {
+      const db = context.getAppDB()
+      for (const migration of migrations.MIGRATIONS) {
+        await migration.func()
+        const docs = await db.allDocs({ include_docs: true })
+
+        await migration.func()
+        const latestDocs = await db.allDocs({ include_docs: true })
+
+        expect(docs).toEqual(latestDocs)
+      }
+    })
+  })
+})

packages/server/src/appMigrations/tests/migrations.spec.ts

@@ -1,25 +1,53 @@
-import { context } from "@budibase/backend-core"
+import { Header } from "@budibase/backend-core"
 import * as setup from "../../api/routes/tests/utilities"
-import { MIGRATIONS } from "../migrations"
+import * as migrations from "../migrations"
+import { getAppMigrationVersion } from "../appMigrationMetadata"
 
-describe("migration", () => {
+jest.mock<typeof migrations>("../migrations", () => ({
-  // These test is checking that each migration is "idempotent".
+  MIGRATIONS: [
-  // We should be able to rerun any migration, with any rerun not modifiying anything. The code should be aware that the migration already ran
+    {
-  it("each migration can rerun safely", async () => {
+      id: "20231211101320_test",
+      func: async () => {},
+    },
+  ],
+}))
+
+describe("migrations", () => {
+  it("new apps are created with the latest app migration version set", async () => {
     const config = setup.getConfig()
     await config.init()
 
     await config.doInContext(config.getAppId(), async () => {
-      const db = context.getAppDB()
+      const migrationVersion = await getAppMigrationVersion(config.getAppId())
-      for (const migration of MIGRATIONS) {
-        await migration.func()
-        const docs = await db.allDocs({ include_docs: true })
 
-        await migration.func()
+      expect(migrationVersion).toEqual("20231211101320_test")
-        const latestDocs = await db.allDocs({ include_docs: true })
-
-        expect(docs).toEqual(latestDocs)
-      }
     })
   })
+
+  it("accessing an app that has no pending migrations will not attach the migrating header", async () => {
+    const config = setup.getConfig()
+    await config.init()
+
+    const appId = config.getAppId()
+
+    const response = await config.api.application.getRaw(appId)
+
+    expect(response.headers[Header.MIGRATING_APP]).toBeUndefined()
+  })
+
+  it("accessing an app that has pending migrations will attach the migrating header", async () => {
+    const config = setup.getConfig()
+    await config.init()
+
+    const appId = config.getAppId()
+
+    migrations.MIGRATIONS.push({
+      id: "20231211105812_new-test",
+      func: async () => {},
+    })
+
+    const response = await config.api.application.getRaw(appId)
+
+    expect(response.headers[Header.MIGRATING_APP]).toEqual(appId)
+  })
 })

packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts

@@ -4,12 +4,14 @@ import { getAppMigrationVersion } from "../appMigrationMetadata"
 import { context } from "@budibase/backend-core"
 import { AppMigration } from ".."
 
+const futureTimestamp = `20500101174029`
+
 describe("migrationsProcessor", () => {
   it("running migrations will update the latest applied migration", async () => {
     const testMigrations: AppMigration[] = [
-      { id: "123", func: async () => {} },
+      { id: `${futureTimestamp}_123`, func: async () => {} },
-      { id: "124", func: async () => {} },
+      { id: `${futureTimestamp}_124`, func: async () => {} },
-      { id: "125", func: async () => {} },
+      { id: `${futureTimestamp}_125`, func: async () => {} },
     ]
 
     const config = setup.getConfig()
@@ -23,13 +25,13 @@ describe("migrationsProcessor", () => {
 
     expect(
       await config.doInContext(appId, () => getAppMigrationVersion(appId))
-    ).toBe("125")
+    ).toBe(`${futureTimestamp}_125`)
   })
 
   it("no context can be initialised within a migration", async () => {
     const testMigrations: AppMigration[] = [
       {
-        id: "123",
+        id: `${futureTimestamp}_123`,
         func: async () => {
           await context.doInAppMigrationContext("any", () => {})
         },

packages/server/src/middleware/appMigrations.ts

@@ -8,7 +8,5 @@ export default async (ctx: UserCtx, next: any) => {
     return next()
   }
 
-  await checkMissingMigrations(appId)
+  return checkMissingMigrations(ctx, next, appId)
-
-  return next()
 }

packages/server/src/tests/utilities/api/application.ts

@@ -1,3 +1,4 @@
+import { Response } from "supertest"
 import { App } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
@@ -7,12 +8,17 @@ export class ApplicationAPI extends TestAPI {
     super(config)
   }
 
-  get = async (appId: string): Promise<App> => {
+  getRaw = async (appId: string): Promise<Response> => {
     const result = await this.request
       .get(`/api/applications/${appId}/appPackage`)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
       .expect(200)
+    return result
+  }
+
+  get = async (appId: string): Promise<App> => {
+    const result = await this.getRaw(appId)
     return result.body.application as App
   }
 }

packages/shared-core/src/constants/api.ts

@@ -0,0 +1,19 @@
+export enum Header {
+  API_KEY = "x-budibase-api-key",
+  LICENSE_KEY = "x-budibase-license-key",
+  API_VER = "x-budibase-api-version",
+  APP_ID = "x-budibase-app-id",
+  SESSION_ID = "x-budibase-session-id",
+  TYPE = "x-budibase-type",
+  PREVIEW_ROLE = "x-budibase-role",
+  TENANT_ID = "x-budibase-tenant-id",
+  VERIFICATION_CODE = "x-budibase-verification-code",
+  RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code",
+  RESET_PASSWORD_CODE = "x-budibase-reset-password-code",
+  RETURN_RESET_PASSWORD_CODE = "x-budibase-return-reset-password-code",
+  TOKEN = "x-budibase-token",
+  CSRF_TOKEN = "x-csrf-token",
+  CORRELATION_ID = "x-budibase-correlation-id",
+  AUTHORIZATION = "authorization",
+  MIGRATING_APP = "x-budibase-migrating-app",
+}

packages/shared-core/src/constants/index.ts

@@ -1,3 +1,5 @@
+export * from "./api"
+
 export const OperatorOptions = {
   Equals: {
     value: "equal",