Merge branch 'master' of github.com:Budibase/budibase into labday/sqs

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.13.35",
+  "version": "2.13.36",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -17,14 +17,14 @@
     "kill-port": "^1.6.1",
     "lerna": "7.1.1",
     "madge": "^6.0.0",
-    "minimist": "^1.2.8",
     "nx": "16.4.3",
     "nx-cloud": "16.0.5",
     "prettier": "2.8.8",
     "prettier-plugin-svelte": "^2.3.0",
     "svelte": "3.49.0",
     "svelte-eslint-parser": "^0.32.0",
-    "typescript": "5.2.2"
+    "typescript": "5.2.2",
+    "yargs": "^17.7.2"
   },
   "scripts": {
     "preinstall": "node scripts/syncProPackage.js",
@@ -79,7 +79,8 @@
     "security:audit": "node scripts/audit.js",
     "postinstall": "husky install",
     "submodules:load": "git submodule init && git submodule update && yarn",
-    "submodules:unload": "git submodule deinit --all && yarn"
+    "submodules:unload": "git submodule deinit --all && yarn",
+    "add-app-migration": "node scripts/add-app-migration.js --title"
   },
   "workspaces": {
     "packages": [

packages/backend-core/src/queue/constants.ts

@@ -3,4 +3,5 @@ export enum JobQueue {
   APP_BACKUP = "appBackupQueue",
   AUDIT_LOG = "auditLogQueue",
   SYSTEM_EVENT_QUEUE = "systemEventQueue",
+  APP_MIGRATION = "appMigration",
 }

packages/backend-core/src/queue/listeners.ts

@@ -87,6 +87,7 @@ enum QueueEventType {
   APP_BACKUP_EVENT = "app-backup-event",
   AUDIT_LOG_EVENT = "audit-log-event",
   SYSTEM_EVENT = "system-event",
+  APP_MIGRATION = "app-migration",
 }
 
 const EventTypeMap: { [key in JobQueue]: QueueEventType } = {
@@ -94,6 +95,7 @@ const EventTypeMap: { [key in JobQueue]: QueueEventType } = {
   [JobQueue.APP_BACKUP]: QueueEventType.APP_BACKUP_EVENT,
   [JobQueue.AUDIT_LOG]: QueueEventType.AUDIT_LOG_EVENT,
   [JobQueue.SYSTEM_EVENT_QUEUE]: QueueEventType.SYSTEM_EVENT,
+  [JobQueue.APP_MIGRATION]: QueueEventType.APP_MIGRATION,
 }
 
 function logging(queue: Queue, jobQueue: JobQueue) {

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -1,6 +1,5 @@
 <script>
   import {
-    banner,
     Heading,
     Layout,
     Button,
@@ -11,7 +10,6 @@
     Notification,
     Body,
     Search,
-    BANNER_TYPES,
   } from "@budibase/bbui"
   import Spinner from "components/common/Spinner.svelte"
   import CreateAppModal from "components/start/CreateAppModal.svelte"
@@ -200,20 +198,6 @@
       if (usersLimitLockAction) {
         usersLimitLockAction()
       }
-      if (!$admin.isDev) {
-        await banner.show({
-          messages: [
-            {
-              message:
-                "We've updated our pricing - see our website to learn more.",
-              type: BANNER_TYPES.NEUTRAL,
-              extraButtonText: "Learn More",
-              extraButtonAction: () =>
-                window.open("https://budibase.com/pricing"),
-            },
-          ],
-        })
-      }
     } catch (error) {
       notifications.error("Error getting init info")
     }

packages/server/README.md

@@ -0,0 +1,14 @@
+# Budibase server project
+
+This project contains all the server specific logic required to run a Budibase app
+
+## App migrations
+
+A migration system has been created in order to modify existing apps when breaking changes are added. These migrations will run on the app startup (both from the client side or the builder side), blocking the access until they are correctly applied.
+
+### Create a new migration
+
+In order to add a new migration:
+
+1. Run `yarn add-app-migration [title]`
+2. Write your code on the newly created file

packages/server/src/api/controllers/application.ts

@@ -52,6 +52,7 @@ import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
 import { sdk as sharedCoreSDK } from "@budibase/shared-core"
+import * as appMigrations from "../../appMigrations"
 
 // utility function, need to do away with this
 async function getLayouts() {
@@ -336,6 +337,12 @@ async function performAppCreate(ctx: UserCtx) {
       await createApp(appId)
     }
 
+    // Initialise the app migration version as the latest one
+    await appMigrations.updateAppMigrationMetadata({
+      appId,
+      version: appMigrations.latestMigration,
+    })
+
     await cache.app.invalidateAppMetadata(appId, newApplication)
     return newApplication
   })

packages/server/src/api/index.ts

@@ -4,6 +4,7 @@ import currentApp from "../middleware/currentapp"
 import zlib from "zlib"
 import { mainRoutes, staticRoutes, publicRoutes } from "./routes"
 import { middleware as pro } from "@budibase/pro"
+import migrations from "../middleware/appMigrations"
 
 export { shutdown } from "./routes/public"
 const compress = require("koa-compress")
@@ -47,6 +48,8 @@ router
   // @ts-ignore
   .use(currentApp)
   .use(auth.auditLog)
+  // @ts-ignore
+  .use(migrations)
 
 // authenticated routes
 for (let route of mainRoutes) {

packages/server/src/appMigrations/appMigrationMetadata.ts

@@ -0,0 +1,90 @@
+import { Duration, cache, context, db, env } from "@budibase/backend-core"
+import { Database, DocumentType, Document } from "@budibase/types"
+
+export interface AppMigrationDoc extends Document {
+  version: string
+  history: Record<string, { runAt: string }>
+}
+
+const EXPIRY_SECONDS = Duration.fromDays(1).toSeconds()
+
+async function getFromDB(appId: string) {
+  return db.doWithDB(
+    appId,
+    (db: Database) => {
+      return db.get<AppMigrationDoc>(DocumentType.APP_MIGRATION_METADATA)
+    },
+    { skip_setup: true }
+  )
+}
+
+const getCacheKey = (appId: string) => `appmigrations_${env.VERSION}_${appId}`
+
+export async function getAppMigrationVersion(appId: string): Promise<string> {
+  const cacheKey = getCacheKey(appId)
+
+  let metadata: AppMigrationDoc | undefined = await cache.get(cacheKey)
+
+  // We don't want to cache in dev, in order to be able to tweak it
+  if (metadata && !env.isDev()) {
+    return metadata.version
+  }
+
+  let version
+  try {
+    metadata = await getFromDB(appId)
+    version = metadata.version
+  } catch (err: any) {
+    if (err.status !== 404) {
+      throw err
+    }
+
+    version = ""
+  }
+
+  await cache.store(cacheKey, version, EXPIRY_SECONDS)
+
+  return version
+}
+
+export async function updateAppMigrationMetadata({
+  appId,
+  version,
+}: {
+  appId: string
+  version: string
+}): Promise<void> {
+  const db = context.getAppDB()
+
+  let appMigrationDoc: AppMigrationDoc
+
+  try {
+    appMigrationDoc = await getFromDB(appId)
+  } catch (err: any) {
+    if (err.status !== 404) {
+      throw err
+    }
+
+    appMigrationDoc = {
+      _id: DocumentType.APP_MIGRATION_METADATA,
+      version: "",
+      history: {},
+    }
+    await db.put(appMigrationDoc)
+    appMigrationDoc = await getFromDB(appId)
+  }
+
+  const updatedMigrationDoc: AppMigrationDoc = {
+    ...appMigrationDoc,
+    version: version || "",
+    history: {
+      ...appMigrationDoc.history,
+      [version]: { runAt: new Date().toISOString() },
+    },
+  }
+  await db.put(updatedMigrationDoc)
+
+  const cacheKey = getCacheKey(appId)
+
+  await cache.destroy(cacheKey)
+}

packages/server/src/appMigrations/index.ts

@@ -0,0 +1,33 @@
+import queue from "./queue"
+import { getAppMigrationVersion } from "./appMigrationMetadata"
+import { MIGRATIONS } from "./migrations"
+
+export * from "./appMigrationMetadata"
+
+export type AppMigration = {
+  id: string
+  func: () => Promise<void>
+}
+
+export const latestMigration = MIGRATIONS.map(m => m.id)
+  .sort()
+  .reverse()[0]
+
+const getTimestamp = (versionId: string) => versionId?.split("_")[0]
+
+export async function checkMissingMigrations(appId: string) {
+  const currentVersion = await getAppMigrationVersion(appId)
+
+  if (getTimestamp(currentVersion) < getTimestamp(latestMigration)) {
+    await queue.add(
+      {
+        appId,
+      },
+      {
+        jobId: `${appId}_${latestMigration}`,
+        removeOnComplete: true,
+        removeOnFail: true,
+      }
+    )
+  }
+}

packages/server/src/appMigrations/migrations.ts

@@ -0,0 +1,7 @@
+// This file should never be manually modified, use `yarn add-app-migration` in order to add a new one
+
+import { AppMigration } from "."
+
+export const MIGRATIONS: AppMigration[] = [
+  // Migrations will be executed sorted by id
+]

packages/server/src/appMigrations/migrationsProcessor.ts

@@ -0,0 +1,58 @@
+import { context, locks } from "@budibase/backend-core"
+import { LockName, LockType } from "@budibase/types"
+
+import {
+  getAppMigrationVersion,
+  updateAppMigrationMetadata,
+} from "./appMigrationMetadata"
+import { AppMigration } from "."
+
+export async function processMigrations(
+  appId: string,
+  migrations: AppMigration[]
+) {
+  console.log(`Processing app migration for "${appId}"`)
+
+  await locks.doWithLock(
+    {
+      name: LockName.APP_MIGRATION,
+      type: LockType.AUTO_EXTEND,
+      resource: appId,
+    },
+    async () => {
+      await context.doInAppMigrationContext(appId, async () => {
+        let currentVersion = await getAppMigrationVersion(appId)
+
+        const pendingMigrations = migrations
+          .filter(m => m.id > currentVersion)
+          .sort((a, b) => a.id.localeCompare(b.id))
+
+        const migrationIds = migrations.map(m => m.id).sort()
+
+        let index = 0
+        for (const { id, func } of pendingMigrations) {
+          const expectedMigration =
+            migrationIds[migrationIds.indexOf(currentVersion) + 1]
+
+          if (expectedMigration !== id) {
+            throw `Migration ${id} could not run, update for "${id}" is running but ${expectedMigration} is expected`
+          }
+
+          const counter = `(${++index}/${pendingMigrations.length})`
+          console.info(`Running migration ${id}... ${counter}`, {
+            migrationId: id,
+            appId,
+          })
+          await func()
+          await updateAppMigrationMetadata({
+            appId,
+            version: id,
+          })
+          currentVersion = id
+        }
+      })
+    }
+  )
+
+  console.log(`App migration for "${appId}" processed`)
+}

packages/server/src/appMigrations/queue.ts

@@ -0,0 +1,15 @@
+import { queue } from "@budibase/backend-core"
+import { Job } from "bull"
+import { MIGRATIONS } from "./migrations"
+import { processMigrations } from "./migrationsProcessor"
+
+const appMigrationQueue = queue.createQueue(queue.JobQueue.APP_MIGRATION)
+appMigrationQueue.process(processMessage)
+
+async function processMessage(job: Job) {
+  const { appId } = job.data
+
+  await processMigrations(appId, MIGRATIONS)
+}
+
+export default appMigrationQueue

packages/server/src/appMigrations/tests/migrations.spec.ts

@@ -0,0 +1,25 @@
+import { context } from "@budibase/backend-core"
+import * as setup from "../../api/routes/tests/utilities"
+import { MIGRATIONS } from "../migrations"
+
+describe("migration", () => {
+  // These test is checking that each migration is "idempotent".
+  // We should be able to rerun any migration, with any rerun not modifiying anything. The code should be aware that the migration already ran
+  it("each migration can rerun safely", async () => {
+    const config = setup.getConfig()
+    await config.init()
+
+    await config.doInContext(config.getAppId(), async () => {
+      const db = context.getAppDB()
+      for (const migration of MIGRATIONS) {
+        await migration.func()
+        const docs = await db.allDocs({ include_docs: true })
+
+        await migration.func()
+        const latestDocs = await db.allDocs({ include_docs: true })
+
+        expect(docs).toEqual(latestDocs)
+      }
+    })
+  })
+})

packages/server/src/appMigrations/tests/migrationsProcessor.spec.ts

@@ -0,0 +1,50 @@
+import * as setup from "../../api/routes/tests/utilities"
+import { processMigrations } from "../migrationsProcessor"
+import { getAppMigrationVersion } from "../appMigrationMetadata"
+import { context } from "@budibase/backend-core"
+import { AppMigration } from ".."
+
+describe("migrationsProcessor", () => {
+  it("running migrations will update the latest applied migration", async () => {
+    const testMigrations: AppMigration[] = [
+      { id: "123", func: async () => {} },
+      { id: "124", func: async () => {} },
+      { id: "125", func: async () => {} },
+    ]
+
+    const config = setup.getConfig()
+    await config.init()
+
+    const appId = config.getAppId()
+
+    await config.doInContext(appId, () =>
+      processMigrations(appId, testMigrations)
+    )
+
+    expect(
+      await config.doInContext(appId, () => getAppMigrationVersion(appId))
+    ).toBe("125")
+  })
+
+  it("no context can be initialised within a migration", async () => {
+    const testMigrations: AppMigration[] = [
+      {
+        id: "123",
+        func: async () => {
+          await context.doInAppMigrationContext("any", () => {})
+        },
+      },
+    ]
+
+    const config = setup.getConfig()
+    await config.init()
+
+    const appId = config.getAppId()
+
+    await expect(
+      config.doInContext(appId, () => processMigrations(appId, testMigrations))
+    ).rejects.toThrowError(
+      "The context cannot be changed, a migration is currently running"
+    )
+  })
+})

packages/server/src/environment.ts

@@ -88,6 +88,7 @@ const environment = {
   },
   TOP_LEVEL_PATH:
     process.env.TOP_LEVEL_PATH || process.env.SERVER_TOP_LEVEL_PATH,
+  APP_MIGRATION_TIMEOUT: parseIntSafe(process.env.APP_MIGRATION_TIMEOUT),
 }
 
 // threading can cause memory issues with node-ts in development

packages/server/src/middleware/appMigrations.ts

@@ -0,0 +1,14 @@
+import { UserCtx } from "@budibase/types"
+import { checkMissingMigrations } from "../appMigrations"
+
+export default async (ctx: UserCtx, next: any) => {
+  const { appId } = ctx
+
+  if (!appId) {
+    return next()
+  }
+
+  await checkMissingMigrations(appId)
+
+  return next()
+}

packages/types/src/documents/document.ts

@@ -37,6 +37,7 @@ export enum DocumentType {
   USER_FLAG = "flag",
   AUTOMATION_METADATA = "meta_au",
   AUDIT_LOG = "al",
+  APP_MIGRATION_METADATA = "_design/migrations",
 }
 
 // these are the core documents that make up the data, design

packages/types/src/sdk/locks.ts

@@ -20,6 +20,7 @@ export enum LockName {
   UPDATE_TENANTS_DOC = "update_tenants_doc",
   PERSIST_WRITETHROUGH = "persist_writethrough",
   QUOTA_USAGE_EVENT = "quota_usage_event",
+  APP_MIGRATION = "app_migrations",
 }
 
 export type LockOptions = {

scripts/add-app-migration.js

@@ -0,0 +1,80 @@
+const fs = require("fs")
+const path = require("path")
+
+const argv = require("yargs").demandOption(
+  ["title"],
+  "Please provide the required parameter: --title=[title]"
+).argv
+
+const { title } = argv
+
+const generateTimestamp = () => {
+  const now = new Date()
+  const year = now.getFullYear()
+  const month = String(now.getMonth() + 1).padStart(2, "0")
+  const day = String(now.getDate()).padStart(2, "0")
+  const hours = String(now.getHours()).padStart(2, "0")
+  const minutes = String(now.getMinutes()).padStart(2, "0")
+  const seconds = String(now.getSeconds()).padStart(2, "0")
+
+  return `${year}${month}${day}${hours}${minutes}${seconds}`
+}
+
+const createMigrationFile = () => {
+  const migrationFilename = `${generateTimestamp()}_${title}`
+  const migrationsDir = "../packages/server/src/appMigrations"
+
+  const template = `const migration = async () => {
+  // Add your migration logic here
+}
+
+export default migration
+`
+
+  const newMigrationPath = path.join(
+    migrationsDir,
+    "migrations",
+    `${migrationFilename}.ts`
+  )
+  fs.writeFileSync(path.resolve(__dirname, newMigrationPath), template)
+
+  console.log(`New migration created: ${newMigrationPath}`)
+
+  // Append the new migration to the main migrations file
+  const migrationsFilePath = path.join(migrationsDir, "migrations.ts")
+
+  const migrationDir = fs.readdirSync(
+    path.join(__dirname, migrationsDir, "migrations")
+  )
+  const migrations = migrationDir
+    .filter(m => m.endsWith(".ts"))
+    .map(m => m.substring(0, m.length - 3))
+
+  let migrationFileContent =
+    '// This file should never be manually modified, use `yarn add-app-migration` in order to add a new one\n\nimport { AppMigration } from "."\n\n'
+
+  for (const migration of migrations) {
+    migrationFileContent += `import m${migration} from "./migrations/${migration}"\n`
+  }
+
+  migrationFileContent += `\nexport const MIGRATIONS: AppMigration[] = [
+  // Migrations will be executed sorted by id\n`
+
+  for (const migration of migrations) {
+    migrationFileContent += `  {
+    id: "${migration}",
+    func: m${migration}
+  },\n`
+  }
+
+  migrationFileContent += `]\n`
+
+  fs.writeFileSync(
+    path.resolve(__dirname, migrationsFilePath),
+    migrationFileContent
+  )
+
+  console.log(`Main migrations file updated: ${migrationsFilePath}`)
+}
+
+createMigrationFile()

scripts/build.js

@@ -13,7 +13,7 @@ const {
 } = require("@esbuild-plugins/tsconfig-paths")
 const { nodeExternalsPlugin } = require("esbuild-node-externals")
 
-var argv = require("minimist")(process.argv.slice(2))
+var { argv } = require("yargs")
 
 function runBuild(entry, outfile) {
   const isDev = process.env.NODE_ENV !== "production"

yarn.lock

@@ -15702,7 +15702,7 @@ minimist-options@4.1.0:
     is-plain-obj "^1.1.0"
     kind-of "^6.0.3"
 
-minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.5, minimist@^1.2.6, minimist@^1.2.8:
+minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.5, minimist@^1.2.6:
   version "1.2.8"
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c"
   integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==
@@ -23021,6 +23021,19 @@ yargs@^17.3.1, yargs@^17.5.1, yargs@^17.6.2:
     y18n "^5.0.5"
     yargs-parser "^21.1.1"
 
+yargs@^17.7.2:
+  version "17.7.2"
+  resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269"
+  integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==
+  dependencies:
+    cliui "^8.0.1"
+    escalade "^3.1.1"
+    get-caller-file "^2.0.5"
+    require-directory "^2.1.1"
+    string-width "^4.2.3"
+    y18n "^5.0.5"
+    yargs-parser "^21.1.1"
+
 yauzl@^2.4.2:
   version "2.10.0"
   resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9"