Migrations 2.0

packages/backend-core/db.js

@@ -1,4 +1,5 @@
 module.exports = {
   ...require("./src/db/utils"),
   ...require("./src/db/constants"),
+  ...require("./src/db/views"),
 }

packages/backend-core/src/auth.js

@@ -12,6 +12,7 @@ const {
   tenancy,
   appTenancy,
   authError,
+  internalApi,
 } = require("./middleware")
 
 // Strategies
@@ -42,4 +43,5 @@ module.exports = {
   buildAppTenancyMiddleware: appTenancy,
   auditLog,
   authError,
+  internalApi,
 }

packages/backend-core/src/middleware/index.js

@@ -7,6 +7,7 @@ const authenticated = require("./authenticated")
 const auditLog = require("./auditLog")
 const tenancy = require("./tenancy")
 const appTenancy = require("./appTenancy")
+const internalApi = require("./internalApi")
 
 module.exports = {
   google,
@@ -18,4 +19,5 @@ module.exports = {
   tenancy,
   appTenancy,
   authError,
+  internalApi,
 }

packages/backend-core/src/middleware/internalApi.js

@@ -0,0 +1,14 @@
+const env = require("../environment")
+const { Headers } = require("../constants")
+
+/**
+ * API Key only endpoint.
+ */
+module.exports = async (ctx, next) => {
+  const apiKey = ctx.request.headers[Headers.API_KEY]
+  if (apiKey !== env.INTERNAL_API_KEY) {
+    ctx.throw(403, "Unauthorized")
+  }
+
+  return next()
+}

packages/backend-core/src/migrations/index.js

@@ -1,20 +1,12 @@
+const { DEFAULT_TENANT_ID } = require("../constants")
 const { DocumentTypes } = require("../db/constants")
-const { getGlobalDB, getTenantId } = require("../tenancy")
+const { getAllApps } = require("../db/utils")
+const environment = require("../environment")
+const { doInTenant, getTenantIds, getGlobalDBName } = require("../tenancy")
 
-exports.MIGRATION_DBS = {
-  GLOBAL_DB: "GLOBAL_DB",
-}
-
-exports.MIGRATIONS = {
-  USER_EMAIL_VIEW_CASING: "user_email_view_casing",
-  QUOTAS_1: "quotas_1",
-}
-
-const DB_LOOKUP = {
-  [exports.MIGRATION_DBS.GLOBAL_DB]: [
-    exports.MIGRATIONS.USER_EMAIL_VIEW_CASING,
-    exports.MIGRATIONS.QUOTAS_1,
-  ],
+exports.MIGRATION_TYPES = {
+  GLOBAL: "global", // run once, recorded in global db, global db is provided as an argument
+  APP: "app", // run per app, recorded in each app db, app db is provided as an argument
 }
 
 exports.getMigrationsDoc = async db => {
@@ -28,40 +20,84 @@ exports.getMigrationsDoc = async db => {
   }
 }
 
-exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
-  const tenantId = getTenantId()
-  try {
-    let db
-    if (migrationDb === exports.MIGRATION_DBS.GLOBAL_DB) {
-      db = getGlobalDB()
-    } else {
-      throw new Error(`Unrecognised migration db [${migrationDb}]`)
-    }
+const runMigration = async (tenantId, CouchDB, migration, options = {}) => {
+  const migrationType = migration.type
+  const migrationName = migration.name
 
-    if (!DB_LOOKUP[migrationDb].includes(migrationName)) {
-      throw new Error(
-        `Unrecognised migration name [${migrationName}] for db [${migrationDb}]`
-      )
-    }
-
-    const doc = await exports.getMigrationsDoc(db)
-    // exit if the migration has been performed
-    if (doc[migrationName]) {
-      return
-    }
-
-    console.log(`[Tenant: ${tenantId}] Performing migration: ${migrationName}`)
-    await migrateFn()
-    console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`)
-
-    // mark as complete
-    doc[migrationName] = Date.now()
-    await db.put(doc)
-  } catch (err) {
-    console.error(
-      `[Tenant: ${tenantId}] Error performing migration: ${migrationName}: `,
-      err
+  // get the db to store the migration in
+  let dbNames
+  if (migrationType === exports.MIGRATION_TYPES.GLOBAL) {
+    dbNames = [getGlobalDBName(tenantId)]
+  } else if (migrationType === exports.MIGRATION_TYPES.APP) {
+    dbNames = await getAllApps(CouchDB, { all: true })
+  } else {
+    throw new Error(
+      `[Tenant: ${tenantId}] Unrecognised migration type [${migrationType}]`
     )
-    throw err
+  }
+
+  // run the migration against each db
+  for (const dbName of dbNames) {
+    const db = new CouchDB(dbName)
+    try {
+      const doc = await exports.getMigrationsDoc(db)
+
+      // exit if the migration has been performed already
+      if (doc[migrationName]) {
+        if (
+          options.force &&
+          options.force[migrationType] &&
+          options.force[migrationType].includes(migrationName)
+        ) {
+          console.log(
+            `[Tenant: ${tenantId}] Forcing migration [${migrationName}]`
+          )
+        } else {
+          // the migration has already been performed
+          return
+        }
+      }
+
+      console.log(
+        `[Tenant: ${tenantId}] Performing migration: ${migrationName}`
+      )
+      // run the migration with tenant context
+      await doInTenant(tenantId, () => migration.fn(db))
+      console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`)
+
+      // mark as complete
+      doc[migrationName] = Date.now()
+      await db.put(doc)
+    } catch (err) {
+      console.error(
+        `[Tenant: ${tenantId}] Error performing migration: ${migrationName} on db: ${db.name}: `,
+        err
+      )
+      throw err
+    }
   }
 }
+
+exports.runMigrations = async (CouchDB, migrations, options = {}) => {
+  console.log("Running migrations")
+  let tenantIds
+  if (environment.MULTI_TENANCY) {
+    if (!options.tenantIds || !options.tenantIds.length) {
+      // run for all tenants
+      tenantIds = await getTenantIds()
+    }
+  } else {
+    // single tenancy
+    tenantIds = [DEFAULT_TENANT_ID]
+  }
+
+  // for all tenants
+  for (const tenantId of tenantIds) {
+    // for all migrations
+    for (const migration of migrations) {
+      // run the migration
+      await runMigration(tenantId, CouchDB, migration, options)
+    }
+  }
+  console.log("Migrations complete")
+}

packages/backend-core/src/tenancy/tenancy.js

@@ -148,3 +148,15 @@ exports.isUserInAppTenant = (appId, user = null) => {
   const tenantId = exports.getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID
   return tenantId === userTenantId
 }
+
+exports.getTenantIds = async () => {
+  const db = getDB(PLATFORM_INFO_DB)
+  let tenants
+  try {
+    tenants = await db.get(TENANT_DOC)
+  } catch (err) {
+    // if theres an error the doc doesn't exist, no tenants exist
+    return []
+  }
+  return (tenants && tenants.tenantIds) || []
+}

packages/backend-core/src/utils.js

@@ -20,9 +20,6 @@ const { hash } = require("./hashing")
 const userCache = require("./cache/user")
 const env = require("./environment")
 const { getUserSessions, invalidateSessions } = require("./security/sessions")
-const { migrateIfRequired } = require("./migrations")
-const { USER_EMAIL_VIEW_CASING } = require("./migrations").MIGRATIONS
-const { GLOBAL_DB } = require("./migrations").MIGRATION_DBS
 
 const APP_PREFIX = DocumentTypes.APP + SEPARATOR
 
@@ -149,11 +146,6 @@ exports.getGlobalUserByEmail = async email => {
   }
   const db = getGlobalDB()
 
-  await migrateIfRequired(GLOBAL_DB, USER_EMAIL_VIEW_CASING, async () => {
-    // re-create the view with latest changes
-    await createUserEmailView(db)
-  })
-
   try {
     let users = (
       await db.query(`database/${ViewNames.USER_BY_EMAIL}`, {

packages/server/src/api/controllers/migrations.js

@@ -0,0 +1,8 @@
+const { migrate } = require("../../migrations")
+
+exports.migrate = async ctx => {
+  const options = ctx.request.body
+  // don't await as can take a while, just return
+  migrate(options)
+  ctx.status = 200
+}

packages/server/src/api/routes/index.js

@@ -24,6 +24,7 @@ const backupRoutes = require("./backup")
 const metadataRoutes = require("./metadata")
 const devRoutes = require("./dev")
 const cloudRoutes = require("./cloud")
+const migrationRoutes = require("./migrations")
 
 exports.mainRoutes = [
   authRoutes,
@@ -53,6 +54,7 @@ exports.mainRoutes = [
   // this could be breaking as koa may recognise other routes as this
   tableRoutes,
   rowRoutes,
+  migrationRoutes,
 ]
 
 exports.staticRoutes = staticRoutes

packages/server/src/api/routes/migrations.js

@@ -0,0 +1,8 @@
+const Router = require("@koa/router")
+const migrationsController = require("../controllers/migrations")
+const router = Router()
+const { internalApi } = require("@budibase/backend-core/auth")
+
+router.post("/api/migrations/run", internalApi, migrationsController.migrate)
+
+module.exports = router

packages/server/src/app.ts

@@ -1,7 +1,7 @@
 // need to load environment first
 import { ExtendableContext } from "koa"
 
-const env = require("./environment")
+import * as env from "./environment"
 const CouchDB = require("./db")
 require("@budibase/backend-core").init(CouchDB)
 const Koa = require("koa")
@@ -16,6 +16,7 @@ const Sentry = require("@sentry/node")
 const fileSystem = require("./utilities/fileSystem")
 const bullboard = require("./automations/bullboard")
 const redis = require("./utilities/redis")
+import * as migrations from "./migrations"
 
 const app = new Koa()
 
@@ -84,13 +85,25 @@ module.exports = server.listen(env.PORT || 0, async () => {
   await automations.init()
 })
 
-process.on("uncaughtException", err => {
-  console.error(err)
+const shutdown = () => {
   server.close()
   server.destroy()
+}
+
+process.on("uncaughtException", err => {
+  console.error(err)
+  shutdown()
 })
 
 process.on("SIGTERM", () => {
-  server.close()
-  server.destroy()
+  shutdown()
 })
+
+// run migrations on startup if not done via http
+// not recommended in a clustered environment
+if (!env.HTTP_MIGRATIONS) {
+  migrations.migrate().catch(err => {
+    console.error("Error performing migrations. Exiting.\n", err)
+    shutdown()
+  })
+}

packages/server/src/environment.js

@@ -43,6 +43,7 @@ module.exports = {
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
+  HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
   // environment
   NODE_ENV: process.env.NODE_ENV,
   JEST_WORKER_ID: process.env.JEST_WORKER_ID,

packages/server/src/middleware/usageQuota.js

@@ -5,7 +5,6 @@ const {
   isExternalTable,
   isRowId: isExternalRowId,
 } = require("../integrations/utils")
-const migration = require("../migrations/usageQuotas")
 
 // currently only counting new writes and deletes
 const METHOD_MAP = {
@@ -74,7 +73,6 @@ module.exports = async (ctx, next) => {
     usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
   try {
-    await migration.run()
     await performRequest(ctx, next, property, usage)
   } catch (err) {
     ctx.throw(400, err)

packages/server/src/migrations/functions/quotas1.ts

@@ -0,0 +1,20 @@
+import { runQuotaMigration } from "./usageQuotas"
+import * as syncApps from "./usageQuotas/syncApps"
+import * as syncRows from "./usageQuotas/syncRows"
+
+/**
+ * Date:
+ * January 2022
+ *
+ * Description:
+ * Synchronise the app and row quotas to the state of the db after it was
+ * discovered that the quota resets were still in place and the row quotas
+ * weren't being decremented correctly.
+ */
+
+export const run = async () => {
+  await runQuotaMigration(async () => {
+    await syncApps.run()
+    await syncRows.run()
+  })
+}

packages/server/src/migrations/functions/usageQuotas/index.ts

@@ -0,0 +1,8 @@
+const { useQuotas } = require("../../../utilities/usageQuota")
+
+export const runQuotaMigration = async (migration: Function) => {
+  if (!useQuotas()) {
+    return
+  }
+  await migration()
+}

packages/server/src/migrations/functions/usageQuotas/syncApps.ts

@@ -1,9 +1,9 @@
 const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
 const { getAllApps } = require("@budibase/backend-core/db")
-const CouchDB = require("../../db")
-const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+import CouchDB from "../../../db"
+import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
 
-exports.run = async () => {
+export const run = async () => {
   const db = getGlobalDB()
   // get app count
   const devApps = await getAllApps(CouchDB, { dev: true })

packages/server/src/migrations/functions/usageQuotas/syncRows.ts

@@ -1,14 +1,14 @@
 const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
 const { getAllApps } = require("@budibase/backend-core/db")
-const CouchDB = require("../../db")
-const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
-const { getUniqueRows } = require("../../utilities/usageQuota/rows")
+import CouchDB from "../../../db"
+import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
+import { getUniqueRows } from "../../../utilities/usageQuota/rows"
 
-exports.run = async () => {
+export const run = async () => {
   const db = getGlobalDB()
   // get all rows in all apps
   const allApps = await getAllApps(CouchDB, { all: true })
-  const appIds = allApps ? allApps.map(app => app.appId) : []
+  const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : []
   const rows = await getUniqueRows(appIds)
   const rowCount = rows ? rows.length : 0
 

packages/server/src/migrations/functions/usageQuotas/tests/index.spec.js

@@ -1,5 +1,5 @@
-const env = require("../../../environment")
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const env = require("../../../../environment")
+const TestConfig = require("../../../../tests/utilities/TestConfiguration")
 
 const syncApps = jest.fn()
 const syncRows = jest.fn()
@@ -7,7 +7,7 @@ const syncRows = jest.fn()
 jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) )
 jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) )
 
-const migrations = require("../../usageQuotas")
+const migrations = require("..")
 
 describe("run", () => {
   let config = new TestConfig(false)

packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js

@@ -1,7 +1,7 @@
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 const TestConfig = require("../../../tests/utilities/TestConfiguration")
 const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
-const syncApps = require("../../usageQuotas/syncApps")
+const syncApps = require("../syncApps")
 const env = require("../../../environment")
 
 describe("syncApps", () => {

packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js

@@ -1,7 +1,7 @@
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 const TestConfig = require("../../../tests/utilities/TestConfiguration")
 const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
-const syncRows = require("../../usageQuotas/syncRows")
+const syncRows = require("../syncRows")
 const env = require("../../../environment")
 
 describe("syncRows", () => {

packages/server/src/migrations/functions/userEmailViewCasing.ts

@@ -0,0 +1,13 @@
+const { createUserEmailView } = require("@budibase/backend-core/db")
+
+/**
+ * Date:
+ * October 2021
+ *
+ * Description:
+ * Recreate the user email view to include latest changes i.e. lower casing the email address
+ */
+
+export const run = async (db: any) => {
+  await createUserEmailView(db)
+}

packages/server/src/migrations/index.ts

@@ -0,0 +1,48 @@
+import CouchDB from "../db"
+const {
+  MIGRATION_TYPES,
+  runMigrations,
+} = require("@budibase/backend-core/migrations")
+
+// migration functions
+import * as userEmailViewCasing from "./functions/userEmailViewCasing"
+import * as quota1 from "./functions/quotas1"
+
+export interface Migration {
+  type: string
+  name: string
+  fn: Function
+}
+
+/**
+ * e.g.
+ * {
+ *   tenantIds: ['bb'],
+ *   force: {
+ *    global: ['quota_1']
+ *   }
+ * }
+ */
+export interface MigrationOptions {
+  tenantIds?: string[]
+  forced?: {
+    [type: string]: string[]
+  }
+}
+
+const MIGRATIONS: Migration[] = [
+  {
+    type: MIGRATION_TYPES.GLOBAL,
+    name: "user_email_view_casing",
+    fn: userEmailViewCasing.run,
+  },
+  {
+    type: MIGRATION_TYPES.GLOBAL,
+    name: "quotas_1",
+    fn: quota1.run,
+  },
+]
+
+export const migrate = async (options?: MigrationOptions) => {
+  await runMigrations(CouchDB, MIGRATIONS, options)
+}

packages/server/src/migrations/usageQuotas/index.js

@@ -1,24 +0,0 @@
-const {
-  MIGRATIONS,
-  MIGRATION_DBS,
-  migrateIfRequired,
-} = require("@budibase/backend-core/migrations")
-const { useQuotas } = require("../../utilities/usageQuota")
-const syncApps = require("./syncApps")
-const syncRows = require("./syncRows")
-
-exports.run = async () => {
-  if (!useQuotas()) {
-    return
-  }
-
-  // Jan 2022
-  await migrateIfRequired(
-    MIGRATION_DBS.GLOBAL_DB,
-    MIGRATIONS.QUOTAS_1,
-    async () => {
-      await syncApps.run()
-      await syncRows.run()
-    }
-  )
-}