From 3fdce44d5635d9b567f6d889e4281d2dc11d7020 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Mon, 24 Jan 2022 10:48:59 +0000
Subject: [PATCH 1/7] Migrations 2.0

---
 packages/backend-core/db.js                   |   1 +
 packages/backend-core/src/auth.js             |   2 +
 packages/backend-core/src/middleware/index.js |   2 +
 .../src/middleware/internalApi.js             |  14 ++
 packages/backend-core/src/migrations/index.js | 132 +++++++++++-------
 packages/backend-core/src/tenancy/tenancy.js  |  12 ++
 packages/backend-core/src/utils.js            |   8 --
 .../server/src/api/controllers/migrations.js  |   8 ++
 packages/server/src/api/routes/index.js       |   2 +
 packages/server/src/api/routes/migrations.js  |   8 ++
 packages/server/src/app.ts                    |  23 ++-
 packages/server/src/environment.js            |   1 +
 packages/server/src/middleware/usageQuota.js  |   2 -
 .../src/migrations/functions/quotas1.ts       |  20 +++
 .../migrations/functions/usageQuotas/index.ts |   8 ++
 .../usageQuotas/syncApps.ts}                  |   6 +-
 .../usageQuotas/syncRows.ts}                  |  10 +-
 .../usageQuotas/tests}/index.spec.js          |   6 +-
 .../usageQuotas/tests}/syncApps.spec.js       |   2 +-
 .../usageQuotas/tests}/syncRows.spec.js       |   2 +-
 .../functions/userEmailViewCasing.ts          |  13 ++
 packages/server/src/migrations/index.ts       |  48 +++++++
 .../src/migrations/usageQuotas/index.js       |  24 ----
 23 files changed, 254 insertions(+), 100 deletions(-)
 create mode 100644 packages/backend-core/src/middleware/internalApi.js
 create mode 100644 packages/server/src/api/controllers/migrations.js
 create mode 100644 packages/server/src/api/routes/migrations.js
 create mode 100644 packages/server/src/migrations/functions/quotas1.ts
 create mode 100644 packages/server/src/migrations/functions/usageQuotas/index.ts
 rename packages/server/src/migrations/{usageQuotas/syncApps.js => functions/usageQuotas/syncApps.ts} (80%)
 rename packages/server/src/migrations/{usageQuotas/syncRows.js => functions/usageQuotas/syncRows.ts} (67%)
 rename packages/server/src/migrations/{tests/usageQuotas => functions/usageQuotas/tests}/index.spec.js (76%)
 rename packages/server/src/migrations/{tests/usageQuotas => functions/usageQuotas/tests}/syncApps.spec.js (94%)
 rename packages/server/src/migrations/{tests/usageQuotas => functions/usageQuotas/tests}/syncRows.spec.js (95%)
 create mode 100644 packages/server/src/migrations/functions/userEmailViewCasing.ts
 create mode 100644 packages/server/src/migrations/index.ts
 delete mode 100644 packages/server/src/migrations/usageQuotas/index.js

diff --git a/packages/backend-core/db.js b/packages/backend-core/db.js
index a7b38821a7..47854ca9c7 100644
--- a/packages/backend-core/db.js
+++ b/packages/backend-core/db.js
@@ -1,4 +1,5 @@
 module.exports = {
   ...require("./src/db/utils"),
   ...require("./src/db/constants"),
+  ...require("./src/db/views"),
 }
diff --git a/packages/backend-core/src/auth.js b/packages/backend-core/src/auth.js
index 7f66d887ae..068dd99a95 100644
--- a/packages/backend-core/src/auth.js
+++ b/packages/backend-core/src/auth.js
@@ -12,6 +12,7 @@ const {
   tenancy,
   appTenancy,
   authError,
+  internalApi,
 } = require("./middleware")
 
 // Strategies
@@ -42,4 +43,5 @@ module.exports = {
   buildAppTenancyMiddleware: appTenancy,
   auditLog,
   authError,
+  internalApi,
 }
diff --git a/packages/backend-core/src/middleware/index.js b/packages/backend-core/src/middleware/index.js
index cf8676a2bc..794aa707a7 100644
--- a/packages/backend-core/src/middleware/index.js
+++ b/packages/backend-core/src/middleware/index.js
@@ -7,6 +7,7 @@ const authenticated = require("./authenticated")
 const auditLog = require("./auditLog")
 const tenancy = require("./tenancy")
 const appTenancy = require("./appTenancy")
+const internalApi = require("./internalApi")
 
 module.exports = {
   google,
@@ -18,4 +19,5 @@ module.exports = {
   tenancy,
   appTenancy,
   authError,
+  internalApi,
 }
diff --git a/packages/backend-core/src/middleware/internalApi.js b/packages/backend-core/src/middleware/internalApi.js
new file mode 100644
index 0000000000..275d559a9e
--- /dev/null
+++ b/packages/backend-core/src/middleware/internalApi.js
@@ -0,0 +1,14 @@
+const env = require("../environment")
+const { Headers } = require("../constants")
+
+/**
+ * API Key only endpoint.
+ */
+module.exports = async (ctx, next) => {
+  const apiKey = ctx.request.headers[Headers.API_KEY]
+  if (apiKey !== env.INTERNAL_API_KEY) {
+    ctx.throw(403, "Unauthorized")
+  }
+
+  return next()
+}
diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js
index 6b8eb3a95c..eae8f150b0 100644
--- a/packages/backend-core/src/migrations/index.js
+++ b/packages/backend-core/src/migrations/index.js
@@ -1,20 +1,12 @@
+const { DEFAULT_TENANT_ID } = require("../constants")
 const { DocumentTypes } = require("../db/constants")
-const { getGlobalDB, getTenantId } = require("../tenancy")
+const { getAllApps } = require("../db/utils")
+const environment = require("../environment")
+const { doInTenant, getTenantIds, getGlobalDBName } = require("../tenancy")
 
-exports.MIGRATION_DBS = {
-  GLOBAL_DB: "GLOBAL_DB",
-}
-
-exports.MIGRATIONS = {
-  USER_EMAIL_VIEW_CASING: "user_email_view_casing",
-  QUOTAS_1: "quotas_1",
-}
-
-const DB_LOOKUP = {
-  [exports.MIGRATION_DBS.GLOBAL_DB]: [
-    exports.MIGRATIONS.USER_EMAIL_VIEW_CASING,
-    exports.MIGRATIONS.QUOTAS_1,
-  ],
+exports.MIGRATION_TYPES = {
+  GLOBAL: "global", // run once, recorded in global db, global db is provided as an argument
+  APP: "app", // run per app, recorded in each app db, app db is provided as an argument
 }
 
 exports.getMigrationsDoc = async db => {
@@ -28,40 +20,84 @@ exports.getMigrationsDoc = async db => {
   }
 }
 
-exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
-  const tenantId = getTenantId()
-  try {
-    let db
-    if (migrationDb === exports.MIGRATION_DBS.GLOBAL_DB) {
-      db = getGlobalDB()
-    } else {
-      throw new Error(`Unrecognised migration db [${migrationDb}]`)
-    }
+const runMigration = async (tenantId, CouchDB, migration, options = {}) => {
+  const migrationType = migration.type
+  const migrationName = migration.name
 
-    if (!DB_LOOKUP[migrationDb].includes(migrationName)) {
-      throw new Error(
-        `Unrecognised migration name [${migrationName}] for db [${migrationDb}]`
-      )
-    }
-
-    const doc = await exports.getMigrationsDoc(db)
-    // exit if the migration has been performed
-    if (doc[migrationName]) {
-      return
-    }
-
-    console.log(`[Tenant: ${tenantId}] Performing migration: ${migrationName}`)
-    await migrateFn()
-    console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`)
-
-    // mark as complete
-    doc[migrationName] = Date.now()
-    await db.put(doc)
-  } catch (err) {
-    console.error(
-      `[Tenant: ${tenantId}] Error performing migration: ${migrationName}: `,
-      err
+  // get the db to store the migration in
+  let dbNames
+  if (migrationType === exports.MIGRATION_TYPES.GLOBAL) {
+    dbNames = [getGlobalDBName(tenantId)]
+  } else if (migrationType === exports.MIGRATION_TYPES.APP) {
+    dbNames = await getAllApps(CouchDB, { all: true })
+  } else {
+    throw new Error(
+      `[Tenant: ${tenantId}] Unrecognised migration type [${migrationType}]`
     )
-    throw err
+  }
+
+  // run the migration against each db
+  for (const dbName of dbNames) {
+    const db = new CouchDB(dbName)
+    try {
+      const doc = await exports.getMigrationsDoc(db)
+
+      // exit if the migration has been performed already
+      if (doc[migrationName]) {
+        if (
+          options.force &&
+          options.force[migrationType] &&
+          options.force[migrationType].includes(migrationName)
+        ) {
+          console.log(
+            `[Tenant: ${tenantId}] Forcing migration [${migrationName}]`
+          )
+        } else {
+          // the migration has already been performed
+          return
+        }
+      }
+
+      console.log(
+        `[Tenant: ${tenantId}] Performing migration: ${migrationName}`
+      )
+      // run the migration with tenant context
+      await doInTenant(tenantId, () => migration.fn(db))
+      console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`)
+
+      // mark as complete
+      doc[migrationName] = Date.now()
+      await db.put(doc)
+    } catch (err) {
+      console.error(
+        `[Tenant: ${tenantId}] Error performing migration: ${migrationName} on db: ${db.name}: `,
+        err
+      )
+      throw err
+    }
   }
 }
+
+exports.runMigrations = async (CouchDB, migrations, options = {}) => {
+  console.log("Running migrations")
+  let tenantIds
+  if (environment.MULTI_TENANCY) {
+    if (!options.tenantIds || !options.tenantIds.length) {
+      // run for all tenants
+      tenantIds = await getTenantIds()
+    }
+  } else {
+    // single tenancy
+    tenantIds = [DEFAULT_TENANT_ID]
+  }
+
+  // for all tenants
+  for (const tenantId of tenantIds) {
+    // for all migrations
+    for (const migration of migrations) {
+      // run the migration
+      await runMigration(tenantId, CouchDB, migration, options)
+    }
+  }
+  console.log("Migrations complete")
+}
diff --git a/packages/backend-core/src/tenancy/tenancy.js b/packages/backend-core/src/tenancy/tenancy.js
index 2cd05ea925..de597eac01 100644
--- a/packages/backend-core/src/tenancy/tenancy.js
+++ b/packages/backend-core/src/tenancy/tenancy.js
@@ -148,3 +148,15 @@ exports.isUserInAppTenant = (appId, user = null) => {
   const tenantId = exports.getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID
   return tenantId === userTenantId
 }
+
+exports.getTenantIds = async () => {
+  const db = getDB(PLATFORM_INFO_DB)
+  let tenants
+  try {
+    tenants = await db.get(TENANT_DOC)
+  } catch (err) {
+    // if theres an error the doc doesn't exist, no tenants exist
+    return []
+  }
+  return (tenants && tenants.tenantIds) || []
+}
diff --git a/packages/backend-core/src/utils.js b/packages/backend-core/src/utils.js
index 85dd32946f..a11e5d42b7 100644
--- a/packages/backend-core/src/utils.js
+++ b/packages/backend-core/src/utils.js
@@ -20,9 +20,6 @@ const { hash } = require("./hashing")
 const userCache = require("./cache/user")
 const env = require("./environment")
 const { getUserSessions, invalidateSessions } = require("./security/sessions")
-const { migrateIfRequired } = require("./migrations")
-const { USER_EMAIL_VIEW_CASING } = require("./migrations").MIGRATIONS
-const { GLOBAL_DB } = require("./migrations").MIGRATION_DBS
 
 const APP_PREFIX = DocumentTypes.APP + SEPARATOR
 
@@ -149,11 +146,6 @@ exports.getGlobalUserByEmail = async email => {
   }
   const db = getGlobalDB()
 
-  await migrateIfRequired(GLOBAL_DB, USER_EMAIL_VIEW_CASING, async () => {
-    // re-create the view with latest changes
-    await createUserEmailView(db)
-  })
-
   try {
     let users = (
       await db.query(`database/${ViewNames.USER_BY_EMAIL}`, {
diff --git a/packages/server/src/api/controllers/migrations.js b/packages/server/src/api/controllers/migrations.js
new file mode 100644
index 0000000000..08892d4034
--- /dev/null
+++ b/packages/server/src/api/controllers/migrations.js
@@ -0,0 +1,8 @@
+const { migrate } = require("../../migrations")
+
+exports.migrate = async ctx => {
+  const options = ctx.request.body
+  // don't await as can take a while, just return
+  migrate(options)
+  ctx.status = 200
+}
diff --git a/packages/server/src/api/routes/index.js b/packages/server/src/api/routes/index.js
index 44c5fbb7b0..8ded7104b0 100644
--- a/packages/server/src/api/routes/index.js
+++ b/packages/server/src/api/routes/index.js
@@ -24,6 +24,7 @@ const backupRoutes = require("./backup")
 const metadataRoutes = require("./metadata")
 const devRoutes = require("./dev")
 const cloudRoutes = require("./cloud")
+const migrationRoutes = require("./migrations")
 
 exports.mainRoutes = [
   authRoutes,
@@ -53,6 +54,7 @@ exports.mainRoutes = [
   // this could be breaking as koa may recognise other routes as this
   tableRoutes,
   rowRoutes,
+  migrationRoutes,
 ]
 
 exports.staticRoutes = staticRoutes
diff --git a/packages/server/src/api/routes/migrations.js b/packages/server/src/api/routes/migrations.js
new file mode 100644
index 0000000000..7cdcdc5935
--- /dev/null
+++ b/packages/server/src/api/routes/migrations.js
@@ -0,0 +1,8 @@
+const Router = require("@koa/router")
+const migrationsController = require("../controllers/migrations")
+const router = Router()
+const { internalApi } = require("@budibase/backend-core/auth")
+
+router.post("/api/migrations/run", internalApi, migrationsController.migrate)
+
+module.exports = router
diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts
index 060169a777..0c0ef68ad9 100644
--- a/packages/server/src/app.ts
+++ b/packages/server/src/app.ts
@@ -1,7 +1,7 @@
 // need to load environment first
 import { ExtendableContext } from "koa"
 
-const env = require("./environment")
+import * as env from "./environment"
 const CouchDB = require("./db")
 require("@budibase/backend-core").init(CouchDB)
 const Koa = require("koa")
@@ -16,6 +16,7 @@ const Sentry = require("@sentry/node")
 const fileSystem = require("./utilities/fileSystem")
 const bullboard = require("./automations/bullboard")
 const redis = require("./utilities/redis")
+import * as migrations from "./migrations"
 
 const app = new Koa()
 
@@ -84,13 +85,25 @@ module.exports = server.listen(env.PORT || 0, async () => {
   await automations.init()
 })
 
-process.on("uncaughtException", err => {
-  console.error(err)
+const shutdown = () => {
   server.close()
   server.destroy()
+}
+
+process.on("uncaughtException", err => {
+  console.error(err)
+  shutdown()
 })
 
 process.on("SIGTERM", () => {
-  server.close()
-  server.destroy()
+  shutdown()
 })
+
+// run migrations on startup if not done via http
+// not recommended in a clustered environment
+if (!env.HTTP_MIGRATIONS) {
+  migrations.migrate().catch(err => {
+    console.error("Error performing migrations. Exiting.\n", err)
+    shutdown()
+  })
+}
diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js
index 614f41a29f..99343937d9 100644
--- a/packages/server/src/environment.js
+++ b/packages/server/src/environment.js
@@ -43,6 +43,7 @@ module.exports = {
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
+  HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
   // environment
   NODE_ENV: process.env.NODE_ENV,
   JEST_WORKER_ID: process.env.JEST_WORKER_ID,
diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js
index 4bafa75132..2cd0836113 100644
--- a/packages/server/src/middleware/usageQuota.js
+++ b/packages/server/src/middleware/usageQuota.js
@@ -5,7 +5,6 @@ const {
   isExternalTable,
   isRowId: isExternalRowId,
 } = require("../integrations/utils")
-const migration = require("../migrations/usageQuotas")
 
 // currently only counting new writes and deletes
 const METHOD_MAP = {
@@ -74,7 +73,6 @@ module.exports = async (ctx, next) => {
     usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
   try {
-    await migration.run()
     await performRequest(ctx, next, property, usage)
   } catch (err) {
     ctx.throw(400, err)
diff --git a/packages/server/src/migrations/functions/quotas1.ts b/packages/server/src/migrations/functions/quotas1.ts
new file mode 100644
index 0000000000..500aa68f51
--- /dev/null
+++ b/packages/server/src/migrations/functions/quotas1.ts
@@ -0,0 +1,20 @@
+import { runQuotaMigration } from "./usageQuotas"
+import * as syncApps from "./usageQuotas/syncApps"
+import * as syncRows from "./usageQuotas/syncRows"
+
+/**
+ * Date:
+ * January 2022
+ *
+ * Description:
+ * Synchronise the app and row quotas to the state of the db after it was
+ * discovered that the quota resets were still in place and the row quotas
+ * weren't being decremented correctly.
+ */
+
+export const run = async () => {
+  await runQuotaMigration(async () => {
+    await syncApps.run()
+    await syncRows.run()
+  })
+}
diff --git a/packages/server/src/migrations/functions/usageQuotas/index.ts b/packages/server/src/migrations/functions/usageQuotas/index.ts
new file mode 100644
index 0000000000..16c4bf1d89
--- /dev/null
+++ b/packages/server/src/migrations/functions/usageQuotas/index.ts
@@ -0,0 +1,8 @@
+const { useQuotas } = require("../../../utilities/usageQuota")
+
+export const runQuotaMigration = async (migration: Function) => {
+  if (!useQuotas()) {
+    return
+  }
+  await migration()
+}
diff --git a/packages/server/src/migrations/usageQuotas/syncApps.js b/packages/server/src/migrations/functions/usageQuotas/syncApps.ts
similarity index 80%
rename from packages/server/src/migrations/usageQuotas/syncApps.js
rename to packages/server/src/migrations/functions/usageQuotas/syncApps.ts
index ee106129e6..0fba4f0f7f 100644
--- a/packages/server/src/migrations/usageQuotas/syncApps.js
+++ b/packages/server/src/migrations/functions/usageQuotas/syncApps.ts
@@ -1,9 +1,9 @@
 const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
 const { getAllApps } = require("@budibase/backend-core/db")
-const CouchDB = require("../../db")
-const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+import CouchDB from "../../../db"
+import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
 
-exports.run = async () => {
+export const run = async () => {
   const db = getGlobalDB()
   // get app count
   const devApps = await getAllApps(CouchDB, { dev: true })
diff --git a/packages/server/src/migrations/usageQuotas/syncRows.js b/packages/server/src/migrations/functions/usageQuotas/syncRows.ts
similarity index 67%
rename from packages/server/src/migrations/usageQuotas/syncRows.js
rename to packages/server/src/migrations/functions/usageQuotas/syncRows.ts
index 7990f405de..58767d0c0a 100644
--- a/packages/server/src/migrations/usageQuotas/syncRows.js
+++ b/packages/server/src/migrations/functions/usageQuotas/syncRows.ts
@@ -1,14 +1,14 @@
 const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
 const { getAllApps } = require("@budibase/backend-core/db")
-const CouchDB = require("../../db")
-const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
-const { getUniqueRows } = require("../../utilities/usageQuota/rows")
+import CouchDB from "../../../db"
+import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
+import { getUniqueRows } from "../../../utilities/usageQuota/rows"
 
-exports.run = async () => {
+export const run = async () => {
   const db = getGlobalDB()
   // get all rows in all apps
   const allApps = await getAllApps(CouchDB, { all: true })
-  const appIds = allApps ? allApps.map(app => app.appId) : []
+  const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : []
   const rows = await getUniqueRows(appIds)
   const rowCount = rows ? rows.length : 0
 
diff --git a/packages/server/src/migrations/tests/usageQuotas/index.spec.js b/packages/server/src/migrations/functions/usageQuotas/tests/index.spec.js
similarity index 76%
rename from packages/server/src/migrations/tests/usageQuotas/index.spec.js
rename to packages/server/src/migrations/functions/usageQuotas/tests/index.spec.js
index 0c5b982909..586b796305 100644
--- a/packages/server/src/migrations/tests/usageQuotas/index.spec.js
+++ b/packages/server/src/migrations/functions/usageQuotas/tests/index.spec.js
@@ -1,5 +1,5 @@
-const env = require("../../../environment")
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const env = require("../../../../environment")
+const TestConfig = require("../../../../tests/utilities/TestConfiguration")
 
 const syncApps = jest.fn()
 const syncRows = jest.fn()
@@ -7,7 +7,7 @@ const syncRows = jest.fn()
 jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) )
 jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) )
 
-const migrations = require("../../usageQuotas")
+const migrations = require("..")
 
 describe("run", () => {
   let config = new TestConfig(false)
diff --git a/packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js b/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js
similarity index 94%
rename from packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js
rename to packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js
index 160319a31b..8772900890 100644
--- a/packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js
+++ b/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js
@@ -1,7 +1,7 @@
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 const TestConfig = require("../../../tests/utilities/TestConfiguration")
 const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
-const syncApps = require("../../usageQuotas/syncApps")
+const syncApps = require("../syncApps")
 const env = require("../../../environment")
 
 describe("syncApps", () => {
diff --git a/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js
similarity index 95%
rename from packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js
rename to packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js
index a09bea60bd..e83238fa18 100644
--- a/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js
+++ b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js
@@ -1,7 +1,7 @@
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 const TestConfig = require("../../../tests/utilities/TestConfiguration")
 const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
-const syncRows = require("../../usageQuotas/syncRows")
+const syncRows = require("../syncRows")
 const env = require("../../../environment")
 
 describe("syncRows", () => {
diff --git a/packages/server/src/migrations/functions/userEmailViewCasing.ts b/packages/server/src/migrations/functions/userEmailViewCasing.ts
new file mode 100644
index 0000000000..16f55655ab
--- /dev/null
+++ b/packages/server/src/migrations/functions/userEmailViewCasing.ts
@@ -0,0 +1,13 @@
+const { createUserEmailView } = require("@budibase/backend-core/db")
+
+/**
+ * Date:
+ * October 2021
+ *
+ * Description:
+ * Recreate the user email view to include latest changes i.e. lower casing the email address
+ */
+
+export const run = async (db: any) => {
+  await createUserEmailView(db)
+}
diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts
new file mode 100644
index 0000000000..def8aac5f7
--- /dev/null
+++ b/packages/server/src/migrations/index.ts
@@ -0,0 +1,48 @@
+import CouchDB from "../db"
+const {
+  MIGRATION_TYPES,
+  runMigrations,
+} = require("@budibase/backend-core/migrations")
+
+// migration functions
+import * as userEmailViewCasing from "./functions/userEmailViewCasing"
+import * as quota1 from "./functions/quotas1"
+
+export interface Migration {
+  type: string
+  name: string
+  fn: Function
+}
+
+/**
+ * e.g.
+ * {
+ *   tenantIds: ['bb'],
+ *   force: {
+ *    global: ['quota_1']
+ *   }
+ * }
+ */
+export interface MigrationOptions {
+  tenantIds?: string[]
+  forced?: {
+    [type: string]: string[]
+  }
+}
+
+const MIGRATIONS: Migration[] = [
+  {
+    type: MIGRATION_TYPES.GLOBAL,
+    name: "user_email_view_casing",
+    fn: userEmailViewCasing.run,
+  },
+  {
+    type: MIGRATION_TYPES.GLOBAL,
+    name: "quotas_1",
+    fn: quota1.run,
+  },
+]
+
+export const migrate = async (options?: MigrationOptions) => {
+  await runMigrations(CouchDB, MIGRATIONS, options)
+}
diff --git a/packages/server/src/migrations/usageQuotas/index.js b/packages/server/src/migrations/usageQuotas/index.js
deleted file mode 100644
index 39744093c2..0000000000
--- a/packages/server/src/migrations/usageQuotas/index.js
+++ /dev/null
@@ -1,24 +0,0 @@
-const {
-  MIGRATIONS,
-  MIGRATION_DBS,
-  migrateIfRequired,
-} = require("@budibase/backend-core/migrations")
-const { useQuotas } = require("../../utilities/usageQuota")
-const syncApps = require("./syncApps")
-const syncRows = require("./syncRows")
-
-exports.run = async () => {
-  if (!useQuotas()) {
-    return
-  }
-
-  // Jan 2022
-  await migrateIfRequired(
-    MIGRATION_DBS.GLOBAL_DB,
-    MIGRATIONS.QUOTAS_1,
-    async () => {
-      await syncApps.run()
-      await syncRows.run()
-    }
-  )
-}

From 767354ad60578d975b83a32b53ec759b5ccd9f16 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Thu, 27 Jan 2022 10:40:31 +0000
Subject: [PATCH 2/7] Add app db support + app url migration

---
 packages/backend-core/src/migrations/index.js | 31 ++++++++-----
 .../tests/__snapshots__/index.spec.js.snap    |  4 +-
 .../src/migrations/tests/index.spec.js        | 44 +++++++++----------
 .../server/src/api/controllers/application.js |  6 +--
 .../server/src/api/controllers/migrations.js  |  7 ++-
 packages/server/src/api/routes/migrations.js  |  8 +++-
 packages/server/src/integrations/rest.ts      | 37 ++++++++++++----
 packages/server/src/integrations/s3.ts        |  2 +-
 .../src/migrations/functions/appUrls.ts       | 25 +++++++++++
 .../functions/tests/appUrls.spec.js           | 29 ++++++++++++
 .../functions/tests/quotas1.spec.js           | 27 ++++++++++++
 .../tests/userEmailViewCasing.spec.js         | 25 +++++++++++
 .../functions/usageQuotas/tests/index.spec.js | 27 ------------
 .../usageQuotas/tests/syncApps.spec.js        |  6 +--
 .../usageQuotas/tests/syncRows.spec.js        |  6 +--
 packages/server/src/migrations/index.ts       | 10 ++++-
 .../src/tests/utilities/TestConfiguration.js  |  4 ++
 17 files changed, 213 insertions(+), 85 deletions(-)
 create mode 100644 packages/server/src/migrations/functions/appUrls.ts
 create mode 100644 packages/server/src/migrations/functions/tests/appUrls.spec.js
 create mode 100644 packages/server/src/migrations/functions/tests/quotas1.spec.js
 create mode 100644 packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js
 delete mode 100644 packages/server/src/migrations/functions/usageQuotas/tests/index.spec.js

diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js
index eae8f150b0..0e66232669 100644
--- a/packages/backend-core/src/migrations/index.js
+++ b/packages/backend-core/src/migrations/index.js
@@ -2,7 +2,12 @@ const { DEFAULT_TENANT_ID } = require("../constants")
 const { DocumentTypes } = require("../db/constants")
 const { getAllApps } = require("../db/utils")
 const environment = require("../environment")
-const { doInTenant, getTenantIds, getGlobalDBName } = require("../tenancy")
+const {
+  doInTenant,
+  getTenantIds,
+  getGlobalDBName,
+  getTenantId,
+} = require("../tenancy")
 
 exports.MIGRATION_TYPES = {
   GLOBAL: "global", // run once, recorded in global db, global db is provided as an argument
@@ -20,16 +25,18 @@ exports.getMigrationsDoc = async db => {
   }
 }
 
-const runMigration = async (tenantId, CouchDB, migration, options = {}) => {
+const runMigration = async (CouchDB, migration, options = {}) => {
+  const tenantId = getTenantId()
   const migrationType = migration.type
   const migrationName = migration.name
 
   // get the db to store the migration in
   let dbNames
   if (migrationType === exports.MIGRATION_TYPES.GLOBAL) {
-    dbNames = [getGlobalDBName(tenantId)]
+    dbNames = [getGlobalDBName()]
   } else if (migrationType === exports.MIGRATION_TYPES.APP) {
-    dbNames = await getAllApps(CouchDB, { all: true })
+    const apps = await getAllApps(CouchDB, migration.opts)
+    dbNames = apps.map(app => app.appId)
   } else {
     throw new Error(
       `[Tenant: ${tenantId}] Unrecognised migration type [${migrationType}]`
@@ -50,7 +57,7 @@ const runMigration = async (tenantId, CouchDB, migration, options = {}) => {
           options.force[migrationType].includes(migrationName)
         ) {
           console.log(
-            `[Tenant: ${tenantId}] Forcing migration [${migrationName}]`
+            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
           )
         } else {
           // the migration has already been performed
@@ -59,18 +66,20 @@ const runMigration = async (tenantId, CouchDB, migration, options = {}) => {
       }
 
       console.log(
-        `[Tenant: ${tenantId}] Performing migration: ${migrationName}`
+        `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running`
       )
       // run the migration with tenant context
-      await doInTenant(tenantId, () => migration.fn(db))
-      console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`)
+      await migration.fn(db)
+      console.log(
+        `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
+      )
 
       // mark as complete
       doc[migrationName] = Date.now()
       await db.put(doc)
     } catch (err) {
       console.error(
-        `[Tenant: ${tenantId}] Error performing migration: ${migrationName} on db: ${db.name}: `,
+        `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
         err
       )
       throw err
@@ -96,7 +105,9 @@ exports.runMigrations = async (CouchDB, migrations, options = {}) => {
     // for all migrations
     for (const migration of migrations) {
       // run the migration
-      await runMigration(tenantId, CouchDB, migration, options)
+      await doInTenant(tenantId, () =>
+        runMigration(CouchDB, migration, options)
+      )
     }
   }
   console.log("Migrations complete")
diff --git a/packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap b/packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap
index e9a18eadde..222c3b1228 100644
--- a/packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap
+++ b/packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap
@@ -3,7 +3,7 @@
 exports[`migrations should match snapshot 1`] = `
 Object {
   "_id": "migrations",
-  "_rev": "1-af6c272fe081efafecd2ea49a8fcbb40",
-  "user_email_view_casing": 1487076708000,
+  "_rev": "1-6277abc4e3db950221768e5a2618a059",
+  "test": 1487076708000,
 }
 `;
diff --git a/packages/backend-core/src/migrations/tests/index.spec.js b/packages/backend-core/src/migrations/tests/index.spec.js
index 0ed16fc184..12a2e54cb3 100644
--- a/packages/backend-core/src/migrations/tests/index.spec.js
+++ b/packages/backend-core/src/migrations/tests/index.spec.js
@@ -1,7 +1,7 @@
 require("../../tests/utilities/dbConfig")
 
-const { migrateIfRequired, MIGRATION_DBS, MIGRATIONS, getMigrationsDoc } = require("../index")
-const database = require("../../db")
+const { runMigrations, getMigrationsDoc } = require("../index")
+const CouchDB = require("../../db").getCouch()
 const {
   StaticDatabases,
 } = require("../../db/utils")
@@ -13,8 +13,14 @@ describe("migrations", () => {
 
   const migrationFunction = jest.fn()
 
+  const MIGRATIONS = [{
+    type: "global",
+    name: "test",
+    fn: migrationFunction
+  }]
+
   beforeEach(() => {
-    db = database.getDB(StaticDatabases.GLOBAL.name)
+    db = new CouchDB(StaticDatabases.GLOBAL.name)
   })
 
   afterEach(async () => {
@@ -22,39 +28,29 @@ describe("migrations", () => {
     await db.destroy()
   })
 
-  const validMigration = () => {
-    return migrateIfRequired(MIGRATION_DBS.GLOBAL_DB, MIGRATIONS.USER_EMAIL_VIEW_CASING, migrationFunction)
+  const migrate = () => {
+    return runMigrations(CouchDB, MIGRATIONS)
   }
 
   it("should run a new migration", async () => {
-    await validMigration()
+    await migrate()
     expect(migrationFunction).toHaveBeenCalled()
+    const doc = await getMigrationsDoc(db)
+    expect(doc.test).toBeDefined()
   })
 
   it("should match snapshot", async () => {
-    await validMigration()
+    await migrate()
     const doc = await getMigrationsDoc(db)
     expect(doc).toMatchSnapshot()
   })
 
   it("should skip a previously run migration", async () => {
-    await validMigration()
-    await validMigration()
+    await migrate()
+    const previousMigrationTime = await getMigrationsDoc(db).test
+    await migrate()
+    const currentMigrationTime = await getMigrationsDoc(db).test
     expect(migrationFunction).toHaveBeenCalledTimes(1)
+    expect(currentMigrationTime).toBe(previousMigrationTime)
   })
-
-  it("should reject an unknown migration name", async () => {
-    expect(async () => { 
-      await migrateIfRequired(MIGRATION_DBS.GLOBAL_DB, "bogus_name", migrationFunction)
-    }).rejects.toThrow()
-    expect(migrationFunction).not.toHaveBeenCalled()
-  })
-
-  it("should reject an unknown database name", async () => {
-    expect(async () => { 
-      await migrateIfRequired("bogus_db", MIGRATIONS.USER_EMAIL_VIEW_CASING, migrationFunction)
-    }).rejects.toThrow()
-    expect(migrationFunction).not.toHaveBeenCalled()
-  })
-
 })
\ No newline at end of file
diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js
index 7aaeebc025..0f88129794 100644
--- a/packages/server/src/api/controllers/application.js
+++ b/packages/server/src/api/controllers/application.js
@@ -75,7 +75,7 @@ function getUserRoleId(ctx) {
     : ctx.user.role._id
 }
 
-async function getAppUrl(ctx) {
+exports.getAppUrl = ctx => {
   // construct the url
   let url
   if (ctx.request.body.url) {
@@ -218,7 +218,7 @@ exports.create = async ctx => {
   const apps = await getAllApps(CouchDB, { dev: true })
   const name = ctx.request.body.name
   checkAppName(ctx, apps, name)
-  const url = await getAppUrl(ctx)
+  const url = exports.getAppUrl(ctx)
   checkAppUrl(ctx, apps, url)
 
   const { useTemplate, templateKey, templateString } = ctx.request.body
@@ -281,7 +281,7 @@ exports.update = async ctx => {
   // validation
   const name = ctx.request.body.name
   checkAppName(ctx, apps, name, ctx.params.appId)
-  const url = await getAppUrl(ctx)
+  const url = exports.getAppUrl(ctx)
   checkAppUrl(ctx, apps, url, ctx.params.appId)
 
   const appPackageUpdates = { name, url }
diff --git a/packages/server/src/api/controllers/migrations.js b/packages/server/src/api/controllers/migrations.js
index 08892d4034..6a890349c3 100644
--- a/packages/server/src/api/controllers/migrations.js
+++ b/packages/server/src/api/controllers/migrations.js
@@ -1,4 +1,4 @@
-const { migrate } = require("../../migrations")
+const { migrate, MIGRATIONS } = require("../../migrations")
 
 exports.migrate = async ctx => {
   const options = ctx.request.body
@@ -6,3 +6,8 @@ exports.migrate = async ctx => {
   migrate(options)
   ctx.status = 200
 }
+
+exports.fetchDefinitions = async ctx => {
+  ctx.body = MIGRATIONS
+  ctx.status = 200
+}
diff --git a/packages/server/src/api/routes/migrations.js b/packages/server/src/api/routes/migrations.js
index 7cdcdc5935..01e573edb3 100644
--- a/packages/server/src/api/routes/migrations.js
+++ b/packages/server/src/api/routes/migrations.js
@@ -3,6 +3,12 @@ const migrationsController = require("../controllers/migrations")
 const router = Router()
 const { internalApi } = require("@budibase/backend-core/auth")
 
-router.post("/api/migrations/run", internalApi, migrationsController.migrate)
+router
+  .post("/api/migrations/run", internalApi, migrationsController.migrate)
+  .get(
+    "/api/migrations/definitions",
+    internalApi,
+    migrationsController.fetchDefinitions
+  )
 
 module.exports = router
diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts
index 3199ce3bde..ea40dfb609 100644
--- a/packages/server/src/integrations/rest.ts
+++ b/packages/server/src/integrations/rest.ts
@@ -43,8 +43,8 @@ const coreFields = {
     enum: Object.values(BodyTypes),
   },
   pagination: {
-    type: DatasourceFieldTypes.OBJECT
-  }
+    type: DatasourceFieldTypes.OBJECT,
+  },
 }
 
 module RestModule {
@@ -178,12 +178,17 @@ module RestModule {
           headers,
         },
         pagination: {
-          cursor: nextCursor
-        }
+          cursor: nextCursor,
+        },
       }
     }
 
-    getUrl(path: string, queryString: string, pagination: PaginationConfig | null, paginationValues: PaginationValues | null): string {
+    getUrl(
+      path: string,
+      queryString: string,
+      pagination: PaginationConfig | null,
+      paginationValues: PaginationValues | null
+    ): string {
       // Add pagination params to query string if required
       if (pagination?.location === "query" && paginationValues) {
         const { pageParam, sizeParam } = pagination
@@ -217,14 +222,22 @@ module RestModule {
       return complete
     }
 
-    addBody(bodyType: string, body: string | any, input: any, pagination: PaginationConfig | null, paginationValues: PaginationValues | null) {
+    addBody(
+      bodyType: string,
+      body: string | any,
+      input: any,
+      pagination: PaginationConfig | null,
+      paginationValues: PaginationValues | null
+    ) {
       if (!input.headers) {
         input.headers = {}
       }
       if (bodyType === BodyTypes.NONE) {
         return input
       }
-      let error, object: any = {}, string = ""
+      let error,
+        object: any = {},
+        string = ""
       try {
         if (body) {
           string = typeof body !== "string" ? JSON.stringify(body) : body
@@ -333,7 +346,7 @@ module RestModule {
         requestBody,
         authConfigId,
         pagination,
-        paginationValues
+        paginationValues,
       } = query
       const authHeaders = this.getAuthHeaders(authConfigId)
 
@@ -352,7 +365,13 @@ module RestModule {
       }
 
       let input: any = { method, headers: this.headers }
-      input = this.addBody(bodyType, requestBody, input, pagination, paginationValues)
+      input = this.addBody(
+        bodyType,
+        requestBody,
+        input,
+        pagination,
+        paginationValues
+      )
 
       this.startTimeMs = performance.now()
       const url = this.getUrl(path, queryString, pagination, paginationValues)
diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts
index 25b439fd58..273f221575 100644
--- a/packages/server/src/integrations/s3.ts
+++ b/packages/server/src/integrations/s3.ts
@@ -38,7 +38,7 @@ module S3Module {
       signatureVersion: {
         type: "string",
         required: false,
-        default: "v4"
+        default: "v4",
       },
     },
     query: {
diff --git a/packages/server/src/migrations/functions/appUrls.ts b/packages/server/src/migrations/functions/appUrls.ts
new file mode 100644
index 0000000000..8852c27822
--- /dev/null
+++ b/packages/server/src/migrations/functions/appUrls.ts
@@ -0,0 +1,25 @@
+const { DocumentTypes } = require("@budibase/backend-core/db")
+import { getAppUrl } from "../../api/controllers/application"
+
+/**
+ * Date:
+ * January 2022
+ *
+ * Description:
+ * Add the url to the app metadata if it doesn't exist
+ */
+export const run = async (appDb: any) => {
+  const metadata = await appDb.get(DocumentTypes.APP_METADATA)
+  if (!metadata.url) {
+    const context = {
+      request: {
+        body: {
+          name: metadata.name,
+        },
+      },
+    }
+    metadata.url = getAppUrl(context)
+    console.log(`Adding url to app: ${metadata.url}`)
+  }
+  await appDb.put(metadata)
+}
diff --git a/packages/server/src/migrations/functions/tests/appUrls.spec.js b/packages/server/src/migrations/functions/tests/appUrls.spec.js
new file mode 100644
index 0000000000..d3f080dfd4
--- /dev/null
+++ b/packages/server/src/migrations/functions/tests/appUrls.spec.js
@@ -0,0 +1,29 @@
+const { DocumentTypes } = require("@budibase/backend-core/db")
+const env = require("../../../environment")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+
+const migration = require("../appUrls")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+  const CouchDB = config.getCouch()
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    const app = await config.createApp("testApp")
+    const appDb = new CouchDB(app.appId)
+    let metadata = await appDb.get(DocumentTypes.APP_METADATA)
+    delete metadata.url
+    await appDb.put(metadata)
+    
+    await migration.run(appDb)
+
+    metadata = await appDb.get(DocumentTypes.APP_METADATA)
+    expect(metadata.url).toEqual("/testapp")
+  })
+})
diff --git a/packages/server/src/migrations/functions/tests/quotas1.spec.js b/packages/server/src/migrations/functions/tests/quotas1.spec.js
new file mode 100644
index 0000000000..df8703e9a0
--- /dev/null
+++ b/packages/server/src/migrations/functions/tests/quotas1.spec.js
@@ -0,0 +1,27 @@
+const env = require("../../../environment")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+
+const syncApps = jest.fn()
+const syncRows = jest.fn()
+
+jest.mock("../usageQuotas/syncApps", () => ({ run: syncApps }) )
+jest.mock("../usageQuotas/syncRows", () => ({ run: syncRows }) )
+
+const migration = require("../quotas1")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs ", async () => {
+    await migration.run()
+    expect(syncApps).toHaveBeenCalledTimes(1)
+    expect(syncRows).toHaveBeenCalledTimes(1)
+  })
+})
diff --git a/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js b/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js
new file mode 100644
index 0000000000..c0d7823cbf
--- /dev/null
+++ b/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js
@@ -0,0 +1,25 @@
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+
+// mock email view creation
+const coreDb = require("@budibase/backend-core/db")
+const createUserEmailView = jest.fn()
+coreDb.createUserEmailView = createUserEmailView
+
+const migration = require("../userEmailViewCasing")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+  const globalDb = getGlobalDB()
+
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    await migration.run(globalDb)
+    expect(createUserEmailView).toHaveBeenCalledTimes(1)
+  })
+})
diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/index.spec.js b/packages/server/src/migrations/functions/usageQuotas/tests/index.spec.js
deleted file mode 100644
index 586b796305..0000000000
--- a/packages/server/src/migrations/functions/usageQuotas/tests/index.spec.js
+++ /dev/null
@@ -1,27 +0,0 @@
-const env = require("../../../../environment")
-const TestConfig = require("../../../../tests/utilities/TestConfiguration")
-
-const syncApps = jest.fn()
-const syncRows = jest.fn()
-
-jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) )
-jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) )
-
-const migrations = require("..")
-
-describe("run", () => {
-  let config = new TestConfig(false)
-
-  beforeEach(async () => {
-    await config.init()
-    env._set("USE_QUOTAS", 1)
-  })
-
-  afterAll(config.end)  
-
-  it("runs the required migrations", async () => {
-    await migrations.run()
-    expect(syncApps).toHaveBeenCalledTimes(1)
-    expect(syncRows).toHaveBeenCalledTimes(1)
-  })
-})
diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js b/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js
index 8772900890..7c74cbfe9a 100644
--- a/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js
+++ b/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js
@@ -1,8 +1,8 @@
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
-const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
+const TestConfig = require("../../../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../../../utilities/usageQuota")
 const syncApps = require("../syncApps")
-const env = require("../../../environment")
+const env = require("../../../../environment")
 
 describe("syncApps", () => {
   let config = new TestConfig(false)
diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js
index 9bccf7fe9d..034d0eb067 100644
--- a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js
+++ b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js
@@ -1,8 +1,8 @@
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const TestConfig = require("../../../tests/utilities/TestConfiguration")
-const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
+const TestConfig = require("../../../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../../../utilities/usageQuota")
 const syncRows = require("../syncRows")
-const env = require("../../../environment")
+const env = require("../../../../environment")
 
 describe("syncRows", () => {
   let config = new TestConfig(false)
diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts
index def8aac5f7..966041e0c9 100644
--- a/packages/server/src/migrations/index.ts
+++ b/packages/server/src/migrations/index.ts
@@ -7,10 +7,12 @@ const {
 // migration functions
 import * as userEmailViewCasing from "./functions/userEmailViewCasing"
 import * as quota1 from "./functions/quotas1"
+import * as appUrls from "./functions/appUrls"
 
 export interface Migration {
   type: string
   name: string
+  opts?: object
   fn: Function
 }
 
@@ -30,7 +32,7 @@ export interface MigrationOptions {
   }
 }
 
-const MIGRATIONS: Migration[] = [
+export const MIGRATIONS: Migration[] = [
   {
     type: MIGRATION_TYPES.GLOBAL,
     name: "user_email_view_casing",
@@ -41,6 +43,12 @@ const MIGRATIONS: Migration[] = [
     name: "quotas_1",
     fn: quota1.run,
   },
+  {
+    type: MIGRATION_TYPES.APP,
+    name: "app_urls",
+    opts: { all: true },
+    fn: appUrls.run,
+  },
 ]
 
 export const migrate = async (options?: MigrationOptions) => {
diff --git a/packages/server/src/tests/utilities/TestConfiguration.js b/packages/server/src/tests/utilities/TestConfiguration.js
index 7aefe4fb78..75b65f2dd3 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.js
+++ b/packages/server/src/tests/utilities/TestConfiguration.js
@@ -49,6 +49,10 @@ class TestConfiguration {
     return this.appId
   }
 
+  getCouch() {
+    return CouchDB
+  }
+
   async _req(config, params, controlFunc) {
     const request = {}
     // fake cookies, we don't need them

From 76075e4295301f21b48a429f523526e3edf5986b Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Thu, 27 Jan 2022 10:53:26 +0000
Subject: [PATCH 3/7] Add HTTP_MIGRATIONS to kubernetes environment

---
 charts/budibase/templates/app-service-deployment.yaml | 2 ++
 charts/budibase/values.yaml                           | 1 +
 2 files changed, 3 insertions(+)

diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml
index cd43631992..d9def8c641 100644
--- a/charts/budibase/templates/app-service-deployment.yaml
+++ b/charts/budibase/templates/app-service-deployment.yaml
@@ -108,6 +108,8 @@ spec:
           value: {{ .Values.globals.accountPortalApiKey | quote }}
         - name: COOKIE_DOMAIN
           value: {{ .Values.globals.cookieDomain | quote }}
+        - name: HTTP_MIGRATIONS
+          value: {{ .Values.globals.httpMigrations | quote }}
         image: budibase/apps:{{ .Values.globals.appVersion }}
         imagePullPolicy: Always
         name: bbapps
diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml
index 9ea055c6c0..bb582f69c4 100644
--- a/charts/budibase/values.yaml
+++ b/charts/budibase/values.yaml
@@ -99,6 +99,7 @@ globals:
   accountPortalApiKey: ""
   cookieDomain: ""
   platformUrl: ""
+  httpMigrations: "0"
 
   createSecrets: true # creates an internal API key, JWT secrets and redis password for you
 

From e001376e6d661d877c92a6fa7d8781c4dc8bb7b9 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Fri, 28 Jan 2022 11:32:28 +0000
Subject: [PATCH 4/7] Fix loop early exit

---
 packages/backend-core/src/migrations/index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js
index 0e66232669..e2ed75d407 100644
--- a/packages/backend-core/src/migrations/index.js
+++ b/packages/backend-core/src/migrations/index.js
@@ -61,7 +61,7 @@ const runMigration = async (CouchDB, migration, options = {}) => {
           )
         } else {
           // the migration has already been performed
-          return
+          continue
         }
       }
 

From 24b2fc65a07728c77a74b85d4d14bbe814861099 Mon Sep 17 00:00:00 2001
From: Maurits Lourens <maurits.lourens@nedap.com>
Date: Fri, 28 Jan 2022 16:06:46 +0100
Subject: [PATCH 5/7] Adds a url to the templates

---
 .../pages/builder/portal/apps/index.svelte    | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/packages/builder/src/pages/builder/portal/apps/index.svelte b/packages/builder/src/pages/builder/portal/apps/index.svelte
index faa57e5df3..bf783fdb86 100644
--- a/packages/builder/src/pages/builder/portal/apps/index.svelte
+++ b/packages/builder/src/pages/builder/portal/apps/index.svelte
@@ -337,6 +337,14 @@
             }}
             class="template-card"
           >
+            <a
+              href={item.url}
+              target="_blank"
+              class="external-link"
+              on:click|stopPropagation
+            >
+              <Icon name="LinkOut" size="S" />
+            </a>
             <div class="card-body">
               <div style="color: {item.background}" class="iconAlign">
                 <svg
@@ -518,6 +526,7 @@
     border: 1px solid var(--spectrum-global-color-gray-300);
     cursor: pointer;
     display: flex;
+    position: relative;
   }
 
   .template-card:hover {
@@ -528,6 +537,18 @@
     align-items: center;
     padding: 12px;
   }
+
+  .external-link {
+    position: absolute;
+    top: 5px;
+    right: 5px;
+    color: var(--spectrum-global-color-gray-300);
+    z-index: 99;
+  }
+  .external-link:hover {
+    color: var(--spectrum-global-color-gray-500);
+  }
+
   .iconAlign {
     padding: 0 0 0 var(--spacing-m);
     display: inline-block;

From bdc37b6ff70c18b8fcef5e1edb1bc2b42c2f99fd Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Mon, 31 Jan 2022 11:07:54 +0000
Subject: [PATCH 6/7] Fix CSRF token not present using local auth

---
 .../builder/src/pages/builder/_layout.svelte  |  2 +-
 .../src/pages/builder/auth/reset.svelte       |  2 +-
 packages/builder/src/stores/portal/auth.js    | 21 +++++++++++--------
 .../worker/src/api/controllers/global/auth.js |  5 +----
 4 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/packages/builder/src/pages/builder/_layout.svelte b/packages/builder/src/pages/builder/_layout.svelte
index 12a544096a..1d41af15e7 100644
--- a/packages/builder/src/pages/builder/_layout.svelte
+++ b/packages/builder/src/pages/builder/_layout.svelte
@@ -61,7 +61,7 @@
       await auth.setInitInfo({ init_template: $params["?template"] })
     }
 
-    await auth.checkAuth()
+    await auth.getSelf()
     await admin.init()
 
     if (useAccountPortal && multiTenancyEnabled) {
diff --git a/packages/builder/src/pages/builder/auth/reset.svelte b/packages/builder/src/pages/builder/auth/reset.svelte
index f78dd19eb9..5e5b615d73 100644
--- a/packages/builder/src/pages/builder/auth/reset.svelte
+++ b/packages/builder/src/pages/builder/auth/reset.svelte
@@ -31,7 +31,7 @@
   }
 
   onMount(async () => {
-    await auth.checkAuth()
+    await auth.getSelf()
     await organisation.init()
   })
 </script>
diff --git a/packages/builder/src/stores/portal/auth.js b/packages/builder/src/stores/portal/auth.js
index bdd4d95915..c4197a89c0 100644
--- a/packages/builder/src/stores/portal/auth.js
+++ b/packages/builder/src/stores/portal/auth.js
@@ -108,11 +108,7 @@ export function createAuthStore() {
     return json
   }
 
-  return {
-    subscribe: store.subscribe,
-    setOrganisation,
-    getInitInfo,
-    setInitInfo,
+  const actions = {
     checkQueryString: async () => {
       const urlParams = new URLSearchParams(window.location.search)
       if (urlParams.has("tenantId")) {
@@ -123,7 +119,7 @@ export function createAuthStore() {
     setOrg: async tenantId => {
       await setOrganisation(tenantId)
     },
-    checkAuth: async () => {
+    getSelf: async () => {
       const response = await api.get("/api/global/users/self")
       if (response.status !== 200) {
         setUser(null)
@@ -138,13 +134,12 @@ export function createAuthStore() {
         `/api/global/auth/${tenantId}/login`,
         creds
       )
-      const json = await response.json()
       if (response.status === 200) {
-        setUser(json.user)
+        await actions.getSelf()
       } else {
+        const json = await response.json()
         throw new Error(json.message ? json.message : "Invalid credentials")
       }
-      return json
     },
     logout: async () => {
       const response = await api.post(`/api/global/auth/logout`)
@@ -197,6 +192,14 @@ export function createAuthStore() {
       await response.json()
     },
   }
+
+  return {
+    subscribe: store.subscribe,
+    setOrganisation,
+    getInitInfo,
+    setInitInfo,
+    ...actions,
+  }
 }
 
 export const auth = createAuthStore()
diff --git a/packages/worker/src/api/controllers/global/auth.js b/packages/worker/src/api/controllers/global/auth.js
index b39e8745e9..7b0e50c099 100644
--- a/packages/worker/src/api/controllers/global/auth.js
+++ b/packages/worker/src/api/controllers/global/auth.js
@@ -74,10 +74,7 @@ async function authInternal(ctx, user, err = null, info = null) {
 exports.authenticate = async (ctx, next) => {
   return passport.authenticate("local", async (err, user, info) => {
     await authInternal(ctx, user, err, info)
-
-    delete user.token
-
-    ctx.body = { user }
+    ctx.status = 200
   })(ctx, next)
 }
 

From a063274133089232f0b0a1e17ce2190ed5fc7027 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Mon, 31 Jan 2022 16:54:48 +0000
Subject: [PATCH 7/7] v1.0.49-alpha.2

---
 lerna.json                             | 2 +-
 packages/backend-core/package.json     | 2 +-
 packages/bbui/package.json             | 2 +-
 packages/builder/package.json          | 8 ++++----
 packages/cli/package.json              | 2 +-
 packages/client/package.json           | 6 +++---
 packages/server/package.json           | 8 ++++----
 packages/string-templates/package.json | 2 +-
 packages/worker/package.json           | 6 +++---
 9 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/lerna.json b/lerna.json
index 6dde2921dd..b008fa64a2 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*"
diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index 637ac47a3a..727951c8ee 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "src/index.js",
   "author": "Budibase",
diff --git a/packages/bbui/package.json b/packages/bbui/package.json
index aab6f8f3a3..cb1ad164d9 100644
--- a/packages/bbui/package.json
+++ b/packages/bbui/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
diff --git a/packages/builder/package.json b/packages/builder/package.json
index 3adb4c23d7..9738bfcbca 100644
--- a/packages/builder/package.json
+++ b/packages/builder/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.49-alpha.1",
-    "@budibase/client": "^1.0.49-alpha.1",
+    "@budibase/bbui": "^1.0.49-alpha.2",
+    "@budibase/client": "^1.0.49-alpha.2",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^1.0.49-alpha.1",
+    "@budibase/string-templates": "^1.0.49-alpha.2",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
diff --git a/packages/cli/package.json b/packages/cli/package.json
index b8091c46ed..84fbaad998 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
diff --git a/packages/client/package.json b/packages/client/package.json
index 094d5f1d3d..78c2b7bd54 100644
--- a/packages/client/package.json
+++ b/packages/client/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.49-alpha.1",
+    "@budibase/bbui": "^1.0.49-alpha.2",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^1.0.49-alpha.1",
+    "@budibase/string-templates": "^1.0.49-alpha.2",
     "regexparam": "^1.3.0",
     "rollup-plugin-polyfill-node": "^0.8.0",
     "shortid": "^2.2.15",
diff --git a/packages/server/package.json b/packages/server/package.json
index deba95643c..bfc68f5eb2 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.49-alpha.1",
-    "@budibase/client": "^1.0.49-alpha.1",
-    "@budibase/string-templates": "^1.0.49-alpha.1",
+    "@budibase/backend-core": "^1.0.49-alpha.2",
+    "@budibase/client": "^1.0.49-alpha.2",
+    "@budibase/string-templates": "^1.0.49-alpha.2",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json
index de452d9151..a1b3359ba0 100644
--- a/packages/string-templates/package.json
+++ b/packages/string-templates/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",
diff --git a/packages/worker/package.json b/packages/worker/package.json
index 521479defd..ee84ec586f 100644
--- a/packages/worker/package.json
+++ b/packages/worker/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.0.49-alpha.1",
+  "version": "1.0.49-alpha.2",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -29,8 +29,8 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "^1.0.49-alpha.1",
-    "@budibase/string-templates": "^1.0.49-alpha.1",
+    "@budibase/backend-core": "^1.0.49-alpha.2",
+    "@budibase/string-templates": "^1.0.49-alpha.2",
     "@koa/router": "^8.0.0",
     "@sentry/node": "^6.0.0",
     "@techpass/passport-openidconnect": "^0.3.0",