From eb0214a23105fbb6ee61677a1bf6a8d2dc0ae1af Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Mon, 13 Jun 2022 10:51:29 +0100
Subject: [PATCH] Back populate no-op migrations on new app and tenant create

---
 packages/backend-core/src/events/analytics.ts |   2 +-
 packages/backend-core/src/index.ts            |   5 +-
 .../src/migrations/definitions.ts             |  40 ++++
 packages/backend-core/src/migrations/index.js | 163 ---------------
 packages/backend-core/src/migrations/index.ts |   2 +
 .../backend-core/src/migrations/migrations.ts | 189 ++++++++++++++++++
 packages/backend-core/src/tenancy/index.js    |   4 -
 packages/backend-core/src/tenancy/index.ts    |   9 +
 .../src/tenancy/{tenancy.js => tenancy.ts}    |  50 +++--
 .../server/src/api/controllers/application.ts |  10 +-
 packages/server/src/migrations/index.ts       | 149 +++++++-------
 .../server/src/migrations/tests/index.spec.ts |  20 +-
 packages/types/src/core/index.ts              |   3 -
 .../types/src/documents/account/account.ts    |   2 +-
 packages/types/src/index.ts                   |   6 +-
 packages/types/src/{core => sdk}/context.ts   |   2 +-
 .../types/src/{ => sdk}/events/account.ts     |   0
 packages/types/src/{ => sdk}/events/app.ts    |   0
 packages/types/src/{ => sdk}/events/auth.ts   |   0
 .../types/src/{ => sdk}/events/automation.ts  |   0
 .../types/src/{ => sdk}/events/backfill.ts    |   0
 .../types/src/{ => sdk}/events/datasource.ts  |   0
 packages/types/src/{ => sdk}/events/email.ts  |   0
 packages/types/src/{ => sdk}/events/event.ts  |   0
 .../{core => sdk/events}/identification.ts    |   2 +-
 packages/types/src/{ => sdk}/events/index.ts  |   1 +
 packages/types/src/{ => sdk}/events/layout.ts |   0
 .../types/src/{ => sdk}/events/license.ts     |   0
 packages/types/src/{ => sdk}/events/query.ts  |   0
 packages/types/src/{ => sdk}/events/role.ts   |   0
 packages/types/src/{ => sdk}/events/rows.ts   |   0
 packages/types/src/{ => sdk}/events/screen.ts |   0
 packages/types/src/{ => sdk}/events/serve.ts  |   0
 packages/types/src/{ => sdk}/events/table.ts  |   0
 packages/types/src/{ => sdk}/events/user.ts   |   0
 .../types/src/{ => sdk}/events/version.ts     |   0
 packages/types/src/{ => sdk}/events/view.ts   |   2 +-
 packages/types/src/{core => sdk}/hosting.ts   |   0
 packages/types/src/sdk/index.ts               |   5 +
 .../types/src/{ => sdk}/licensing/index.ts    |   0
 .../types/src/{ => sdk}/licensing/license.ts  |   0
 packages/types/src/sdk/migrations.ts          |  54 +++++
 packages/worker/src/sdk/users/users.ts        |   9 +-
 scripts/link-dependencies.sh                  |   8 +-
 44 files changed, 459 insertions(+), 278 deletions(-)
 create mode 100644 packages/backend-core/src/migrations/definitions.ts
 delete mode 100644 packages/backend-core/src/migrations/index.js
 create mode 100644 packages/backend-core/src/migrations/index.ts
 create mode 100644 packages/backend-core/src/migrations/migrations.ts
 delete mode 100644 packages/backend-core/src/tenancy/index.js
 create mode 100644 packages/backend-core/src/tenancy/index.ts
 rename packages/backend-core/src/tenancy/{tenancy.js => tenancy.ts} (68%)
 delete mode 100644 packages/types/src/core/index.ts
 rename packages/types/src/{core => sdk}/context.ts (87%)
 rename packages/types/src/{ => sdk}/events/account.ts (100%)
 rename packages/types/src/{ => sdk}/events/app.ts (100%)
 rename packages/types/src/{ => sdk}/events/auth.ts (100%)
 rename packages/types/src/{ => sdk}/events/automation.ts (100%)
 rename packages/types/src/{ => sdk}/events/backfill.ts (100%)
 rename packages/types/src/{ => sdk}/events/datasource.ts (100%)
 rename packages/types/src/{ => sdk}/events/email.ts (100%)
 rename packages/types/src/{ => sdk}/events/event.ts (100%)
 rename packages/types/src/{core => sdk/events}/identification.ts (97%)
 rename packages/types/src/{ => sdk}/events/index.ts (93%)
 rename packages/types/src/{ => sdk}/events/layout.ts (100%)
 rename packages/types/src/{ => sdk}/events/license.ts (100%)
 rename packages/types/src/{ => sdk}/events/query.ts (100%)
 rename packages/types/src/{ => sdk}/events/role.ts (100%)
 rename packages/types/src/{ => sdk}/events/rows.ts (100%)
 rename packages/types/src/{ => sdk}/events/screen.ts (100%)
 rename packages/types/src/{ => sdk}/events/serve.ts (100%)
 rename packages/types/src/{ => sdk}/events/table.ts (100%)
 rename packages/types/src/{ => sdk}/events/user.ts (100%)
 rename packages/types/src/{ => sdk}/events/version.ts (100%)
 rename packages/types/src/{ => sdk}/events/view.ts (95%)
 rename packages/types/src/{core => sdk}/hosting.ts (100%)
 create mode 100644 packages/types/src/sdk/index.ts
 rename packages/types/src/{ => sdk}/licensing/index.ts (100%)
 rename packages/types/src/{ => sdk}/licensing/license.ts (100%)
 create mode 100644 packages/types/src/sdk/migrations.ts

diff --git a/packages/backend-core/src/events/analytics.ts b/packages/backend-core/src/events/analytics.ts
index a7e2067fd6..802b6d6314 100644
--- a/packages/backend-core/src/events/analytics.ts
+++ b/packages/backend-core/src/events/analytics.ts
@@ -1,5 +1,5 @@
 import env from "../environment"
-import * as tenancy from "../tenancy"
+import tenancy from "../tenancy"
 import * as dbUtils from "../db/utils"
 import { Configs } from "../constants"
 import { withCache, TTL, CacheKeys } from "../cache/generic"
diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts
index 5166494883..6eb6b14bc4 100644
--- a/packages/backend-core/src/index.ts
+++ b/packages/backend-core/src/index.ts
@@ -12,6 +12,7 @@ import sessions from "./security/sessions"
 import deprovisioning from "./context/deprovision"
 import auth from "./auth"
 import constants from "./constants"
+import * as dbConstants from "./db/constants"
 
 // mimic the outer package exports
 import * as db from "./pkg/db"
@@ -20,7 +21,6 @@ import * as utils from "./pkg/utils"
 import redis from "./pkg/redis"
 import cache from "./pkg/cache"
 import context from "./pkg/context"
-const StaticDatabases = db.StaticDatabases
 
 const init = (opts: any = {}) => {
   db.init(opts.db)
@@ -28,8 +28,8 @@ const init = (opts: any = {}) => {
 
 const core = {
   init,
-  StaticDatabases,
   db,
+  ...dbConstants,
   redis,
   objectStore,
   utils,
@@ -37,6 +37,7 @@ const core = {
   cache,
   auth,
   constants,
+  ...constants,
   migrations,
   env,
   accounts,
diff --git a/packages/backend-core/src/migrations/definitions.ts b/packages/backend-core/src/migrations/definitions.ts
new file mode 100644
index 0000000000..745c8718c9
--- /dev/null
+++ b/packages/backend-core/src/migrations/definitions.ts
@@ -0,0 +1,40 @@
+import {
+  MigrationType,
+  MigrationName,
+  MigrationDefinition,
+} from "@budibase/types"
+
+export const DEFINITIONS: MigrationDefinition[] = [
+  {
+    type: MigrationType.GLOBAL,
+    name: MigrationName.USER_EMAIL_VIEW_CASING,
+  },
+  {
+    type: MigrationType.GLOBAL,
+    name: MigrationName.QUOTAS_1,
+  },
+  {
+    type: MigrationType.APP,
+    name: MigrationName.APP_URLS,
+  },
+  {
+    type: MigrationType.GLOBAL,
+    name: MigrationName.DEVELOPER_QUOTA,
+  },
+  {
+    type: MigrationType.GLOBAL,
+    name: MigrationName.PUBLISHED_APP_QUOTA,
+  },
+  {
+    type: MigrationType.APP,
+    name: MigrationName.EVENT_APP_BACKFILL,
+  },
+  {
+    type: MigrationType.GLOBAL,
+    name: MigrationName.EVENT_GLOBAL_BACKFILL,
+  },
+  {
+    type: MigrationType.INSTALLATION,
+    name: MigrationName.EVENT_INSTALLATION_BACKFILL,
+  },
+]
diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js
deleted file mode 100644
index 5c3ab4f4a5..0000000000
--- a/packages/backend-core/src/migrations/index.js
+++ /dev/null
@@ -1,163 +0,0 @@
-const { DEFAULT_TENANT_ID } = require("../constants")
-const { doWithDB } = require("../db")
-const { DocumentTypes, StaticDatabases } = require("../db/constants")
-const { getAllApps } = require("../db/utils")
-const environment = require("../environment")
-const {
-  doInTenant,
-  getTenantIds,
-  getGlobalDBName,
-  getTenantId,
-} = require("../tenancy")
-const context = require("../context")
-
-exports.MIGRATION_TYPES = {
-  GLOBAL: "global", // run once per tenant, recorded in global db, global db is provided as an argument
-  APP: "app", // run per app, recorded in each app db, app db is provided as an argument
-  INSTALLATION: "installation", // run once, recorded in global info db, global info db is provided as an argument
-}
-
-exports.getMigrationsDoc = async db => {
-  // get the migrations doc
-  try {
-    return await db.get(DocumentTypes.MIGRATIONS)
-  } catch (err) {
-    if (err.status && err.status === 404) {
-      return { _id: DocumentTypes.MIGRATIONS }
-    } else {
-      console.error(err)
-      throw err
-    }
-  }
-}
-
-exports.runMigration = async (migration, options = {}) => {
-  const migrationType = migration.type
-  let tenantId
-  if (migrationType !== exports.MIGRATION_TYPES.INSTALLATION) {
-    tenantId = getTenantId()
-  }
-  const migrationName = migration.name
-  const silent = migration.silent
-
-  const log = message => {
-    if (!silent) {
-      console.log(message)
-    }
-  }
-
-  // get the db to store the migration in
-  let dbNames
-  if (migrationType === exports.MIGRATION_TYPES.GLOBAL) {
-    dbNames = [getGlobalDBName()]
-  } else if (migrationType === exports.MIGRATION_TYPES.APP) {
-    const apps = await getAllApps(migration.opts)
-    dbNames = apps.map(app => app.appId)
-  } else if (migrationType === exports.MIGRATION_TYPES.INSTALLATION) {
-    dbNames = [StaticDatabases.PLATFORM_INFO.name]
-  } else {
-    throw new Error(`Unrecognised migration type [${migrationType}]`)
-  }
-
-  const length = dbNames.length
-  let count = 0
-
-  // run the migration against each db
-  for (const dbName of dbNames) {
-    count++
-    const lengthStatement = length > 1 ? `[${count}/${length}]` : ""
-
-    await doWithDB(dbName, async db => {
-      try {
-        const doc = await exports.getMigrationsDoc(db)
-
-        // exit if the migration has been performed already
-        if (doc[migrationName]) {
-          if (
-            options.force &&
-            options.force[migrationType] &&
-            options.force[migrationType].includes(migrationName)
-          ) {
-            log(
-              `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
-            )
-          } else {
-            // the migration has already been performed
-            return
-          }
-        }
-
-        log(
-          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
-        )
-
-        if (migration.preventRetry) {
-          // eagerly set the completion date
-          // so that we never run this migration twice even upon failure
-          doc[migrationName] = Date.now()
-          const response = await db.put(doc)
-          doc._rev = response.rev
-        }
-
-        // run the migration with tenant context
-        if (migrationType === exports.MIGRATION_TYPES.APP) {
-          await context.doInAppContext(db.name, async () => {
-            await migration.fn(db)
-          })
-        } else {
-          await migration.fn(db)
-        }
-
-        log(
-          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
-        )
-
-        // mark as complete
-        doc[migrationName] = Date.now()
-        await db.put(doc)
-      } catch (err) {
-        console.error(
-          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
-          err
-        )
-        throw err
-      }
-    })
-  }
-}
-
-exports.runMigrations = async (migrations, options = {}) => {
-  let tenantIds
-  if (environment.MULTI_TENANCY) {
-    if (!options.tenantIds || !options.tenantIds.length) {
-      // run for all tenants
-      tenantIds = await getTenantIds()
-    } else {
-      tenantIds = options.tenantIds
-    }
-  } else {
-    // single tenancy
-    tenantIds = [DEFAULT_TENANT_ID]
-  }
-
-  if (tenantIds.length > 1) {
-    console.log(`Checking migrations for ${tenantIds.length} tenants`)
-  } else {
-    console.log("Checking migrations")
-  }
-
-  let count = 0
-  // for all tenants
-  for (const tenantId of tenantIds) {
-    count++
-    if (tenantIds.length > 1) {
-      console.log(`Progress [${count}/${tenantIds.length}]`)
-    }
-    // for all migrations
-    for (const migration of migrations) {
-      // run the migration
-      await doInTenant(tenantId, () => exports.runMigration(migration, options))
-    }
-  }
-  console.log("Migrations complete")
-}
diff --git a/packages/backend-core/src/migrations/index.ts b/packages/backend-core/src/migrations/index.ts
new file mode 100644
index 0000000000..bce0cfc75c
--- /dev/null
+++ b/packages/backend-core/src/migrations/index.ts
@@ -0,0 +1,2 @@
+export * from "./migrations"
+export * from "./definitions"
diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts
new file mode 100644
index 0000000000..b926334317
--- /dev/null
+++ b/packages/backend-core/src/migrations/migrations.ts
@@ -0,0 +1,189 @@
+import { DEFAULT_TENANT_ID } from "../constants"
+import { doWithDB } from "../db"
+import { DocumentTypes, StaticDatabases } from "../db/constants"
+import { getAllApps } from "../db/utils"
+import environment from "../environment"
+import {
+  doInTenant,
+  getTenantIds,
+  getGlobalDBName,
+  getTenantId,
+} from "../tenancy"
+import context from "../context"
+import { DEFINITIONS } from "."
+import {
+  Migration,
+  MigrationOptions,
+  MigrationType,
+  MigrationNoOpOptions,
+} from "@budibase/types"
+
+export const getMigrationsDoc = async (db: any) => {
+  // get the migrations doc
+  try {
+    return await db.get(DocumentTypes.MIGRATIONS)
+  } catch (err: any) {
+    if (err.status && err.status === 404) {
+      return { _id: DocumentTypes.MIGRATIONS }
+    } else {
+      console.error(err)
+      throw err
+    }
+  }
+}
+
+export const backPopulateMigrations = async (opts: MigrationNoOpOptions) => {
+  // filter migrations to the type and populate a no-op migration
+  const migrations: Migration[] = DEFINITIONS.filter(
+    def => def.type === opts.type
+  ).map(d => ({ ...d, fn: () => {} }))
+  await runMigrations(migrations, { noOp: opts })
+}
+
+export const runMigration = async (
+  migration: Migration,
+  options: MigrationOptions = {}
+) => {
+  const migrationType = migration.type
+  let tenantId: string
+  if (migrationType !== MigrationType.INSTALLATION) {
+    tenantId = getTenantId()
+  }
+  const migrationName = migration.name
+  const silent = migration.silent
+
+  const log = (message: string) => {
+    if (!silent) {
+      console.log(message)
+    }
+  }
+
+  // get the db to store the migration in
+  let dbNames
+  if (migrationType === MigrationType.GLOBAL) {
+    dbNames = [getGlobalDBName()]
+  } else if (migrationType === MigrationType.APP) {
+    if (options.noOp) {
+      dbNames = [options.noOp.appId]
+    } else {
+      const apps = await getAllApps(migration.appOpts)
+      dbNames = apps.map(app => app.appId)
+    }
+  } else if (migrationType === MigrationType.INSTALLATION) {
+    dbNames = [StaticDatabases.PLATFORM_INFO.name]
+  } else {
+    throw new Error(`Unrecognised migration type [${migrationType}]`)
+  }
+
+  const length = dbNames.length
+  let count = 0
+
+  // run the migration against each db
+  for (const dbName of dbNames) {
+    count++
+    const lengthStatement = length > 1 ? `[${count}/${length}]` : ""
+
+    await doWithDB(dbName, async (db: any) => {
+      try {
+        const doc = await exports.getMigrationsDoc(db)
+
+        // the migration has already been run
+        if (doc[migrationName]) {
+          // check for force
+          if (
+            options.force &&
+            options.force[migrationType] &&
+            options.force[migrationType].includes(migrationName)
+          ) {
+            log(
+              `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
+            )
+          } else {
+            // no force, exit
+            return
+          }
+        }
+
+        // check if the migration is not a no-op
+        if (!options.noOp) {
+          log(
+            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
+          )
+
+          if (migration.preventRetry) {
+            // eagerly set the completion date
+            // so that we never run this migration twice even upon failure
+            doc[migrationName] = Date.now()
+            const response = await db.put(doc)
+            doc._rev = response.rev
+          }
+
+          // run the migration
+          if (migrationType === MigrationType.APP) {
+            await context.doInAppContext(db.name, async () => {
+              await migration.fn(db)
+            })
+          } else {
+            await migration.fn(db)
+          }
+
+          log(
+            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
+          )
+        }
+
+        // mark as complete
+        doc[migrationName] = Date.now()
+        await db.put(doc)
+      } catch (err) {
+        console.error(
+          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
+          err
+        )
+        throw err
+      }
+    })
+  }
+}
+
+export const runMigrations = async (
+  migrations: Migration[],
+  options: MigrationOptions = {}
+) => {
+  let tenantIds
+
+  if (environment.MULTI_TENANCY) {
+    if (options.noOp) {
+      tenantIds = [options.noOp.tenantId]
+    } else if (!options.tenantIds || !options.tenantIds.length) {
+      // run for all tenants
+      tenantIds = await getTenantIds()
+    } else {
+      tenantIds = options.tenantIds
+    }
+  } else {
+    // single tenancy
+    tenantIds = [DEFAULT_TENANT_ID]
+  }
+
+  if (tenantIds.length > 1) {
+    console.log(`Checking migrations for ${tenantIds.length} tenants`)
+  } else {
+    console.log("Checking migrations")
+  }
+
+  let count = 0
+  // for all tenants
+  for (const tenantId of tenantIds) {
+    count++
+    if (tenantIds.length > 1) {
+      console.log(`Progress [${count}/${tenantIds.length}]`)
+    }
+    // for all migrations
+    for (const migration of migrations) {
+      // run the migration
+      await doInTenant(tenantId, () => runMigration(migration, options))
+    }
+  }
+  console.log("Migrations complete")
+}
diff --git a/packages/backend-core/src/tenancy/index.js b/packages/backend-core/src/tenancy/index.js
deleted file mode 100644
index c847033a12..0000000000
--- a/packages/backend-core/src/tenancy/index.js
+++ /dev/null
@@ -1,4 +0,0 @@
-module.exports = {
-  ...require("../context"),
-  ...require("./tenancy"),
-}
diff --git a/packages/backend-core/src/tenancy/index.ts b/packages/backend-core/src/tenancy/index.ts
new file mode 100644
index 0000000000..e0006abab2
--- /dev/null
+++ b/packages/backend-core/src/tenancy/index.ts
@@ -0,0 +1,9 @@
+import * as context from "../context"
+import * as tenancy from "./tenancy"
+
+const pkg = {
+  ...context,
+  ...tenancy,
+}
+
+export = pkg
diff --git a/packages/backend-core/src/tenancy/tenancy.js b/packages/backend-core/src/tenancy/tenancy.ts
similarity index 68%
rename from packages/backend-core/src/tenancy/tenancy.js
rename to packages/backend-core/src/tenancy/tenancy.ts
index b9d5ad7fbe..041f694d34 100644
--- a/packages/backend-core/src/tenancy/tenancy.js
+++ b/packages/backend-core/src/tenancy/tenancy.ts
@@ -1,18 +1,18 @@
-const { doWithDB } = require("../db")
-const { StaticDatabases } = require("../db/constants")
-const { baseGlobalDBName } = require("./utils")
-const {
+import { doWithDB } from "../db"
+import { StaticDatabases } from "../db/constants"
+import { baseGlobalDBName } from "./utils"
+import {
   getTenantId,
   DEFAULT_TENANT_ID,
   isMultiTenant,
   getTenantIDFromAppID,
-} = require("../context")
-const env = require("../environment")
+} from "../context"
+import env from "../environment"
 
 const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
 const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
 
-exports.addTenantToUrl = url => {
+export const addTenantToUrl = (url: string) => {
   const tenantId = getTenantId()
 
   if (isMultiTenant()) {
@@ -23,8 +23,8 @@ exports.addTenantToUrl = url => {
   return url
 }
 
-exports.doesTenantExist = async tenantId => {
-  return doWithDB(PLATFORM_INFO_DB, async db => {
+export const doesTenantExist = async (tenantId: string) => {
+  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
     let tenants
     try {
       tenants = await db.get(TENANT_DOC)
@@ -40,9 +40,14 @@ exports.doesTenantExist = async tenantId => {
   })
 }
 
-exports.tryAddTenant = async (tenantId, userId, email) => {
-  return doWithDB(PLATFORM_INFO_DB, async db => {
-    const getDoc = async id => {
+export const tryAddTenant = async (
+  tenantId: string,
+  userId: string,
+  email: string,
+  afterCreateTenant: () => Promise<void>
+) => {
+  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
+    const getDoc = async (id: string) => {
       if (!id) {
         return null
       }
@@ -76,12 +81,13 @@ exports.tryAddTenant = async (tenantId, userId, email) => {
     if (tenants.tenantIds.indexOf(tenantId) === -1) {
       tenants.tenantIds.push(tenantId)
       promises.push(db.put(tenants))
+      await afterCreateTenant()
     }
     await Promise.all(promises)
   })
 }
 
-exports.getGlobalDBName = (tenantId = null) => {
+export const getGlobalDBName = (tenantId?: string) => {
   // tenant ID can be set externally, for example user API where
   // new tenants are being created, this may be the case
   if (!tenantId) {
@@ -90,12 +96,12 @@ exports.getGlobalDBName = (tenantId = null) => {
   return baseGlobalDBName(tenantId)
 }
 
-exports.doWithGlobalDB = (tenantId, cb) => {
-  return doWithDB(exports.getGlobalDBName(tenantId), cb)
+export const doWithGlobalDB = (tenantId: string, cb: any) => {
+  return doWithDB(getGlobalDBName(tenantId), cb)
 }
 
-exports.lookupTenantId = async userId => {
-  return doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => {
+export const lookupTenantId = async (userId: string) => {
+  return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: any) => {
     let tenantId = env.MULTI_TENANCY ? DEFAULT_TENANT_ID : null
     try {
       const doc = await db.get(userId)
@@ -110,8 +116,8 @@ exports.lookupTenantId = async userId => {
 }
 
 // lookup, could be email or userId, either will return a doc
-exports.getTenantUser = async identifier => {
-  return doWithDB(PLATFORM_INFO_DB, async db => {
+export const getTenantUser = async (identifier: string) => {
+  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
     try {
       return await db.get(identifier)
     } catch (err) {
@@ -120,7 +126,7 @@ exports.getTenantUser = async identifier => {
   })
 }
 
-exports.isUserInAppTenant = (appId, user = null) => {
+export const isUserInAppTenant = (appId: string, user: any) => {
   let userTenantId
   if (user) {
     userTenantId = user.tenantId || DEFAULT_TENANT_ID
@@ -131,8 +137,8 @@ exports.isUserInAppTenant = (appId, user = null) => {
   return tenantId === userTenantId
 }
 
-exports.getTenantIds = async () => {
-  return doWithDB(PLATFORM_INFO_DB, async db => {
+export const getTenantIds = async () => {
+  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
     let tenants
     try {
       tenants = await db.get(TENANT_DOC)
diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 67c810765a..7d78e5dded 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -52,8 +52,8 @@ const {
 } = require("@budibase/backend-core/context")
 import { getUniqueRows } from "../../utilities/usageQuota/rows"
 import { quotas } from "@budibase/pro"
-import { errors, events } from "@budibase/backend-core"
-import { App } from "@budibase/types"
+import { errors, events, migrations } from "@budibase/backend-core"
+import { App, MigrationType } from "@budibase/types"
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -319,6 +319,12 @@ const creationEvents = async (request: any, app: App) => {
 }
 
 const appPostCreate = async (ctx: any, app: App) => {
+  const tenantId = getTenantId()
+  await migrations.backPopulateMigrations({
+    type: MigrationType.APP,
+    tenantId,
+    appId: app.appId,
+  })
   await creationEvents(ctx.request, app)
   // app import & template creation
   if (ctx.request.body.useTemplate === "true") {
diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts
index c93f628c71..ebba4f420d 100644
--- a/packages/server/src/migrations/index.ts
+++ b/packages/server/src/migrations/index.ts
@@ -1,4 +1,6 @@
 import { migrations, redis } from "@budibase/backend-core"
+import { Migration, MigrationOptions, MigrationName } from "@budibase/types"
+import env from "../environment"
 
 // migration functions
 import * as userEmailViewCasing from "./functions/userEmailViewCasing"
@@ -7,83 +9,88 @@ import * as appUrls from "./functions/appUrls"
 import * as developerQuota from "./functions/developerQuota"
 import * as publishedAppsQuota from "./functions/publishedAppsQuota"
 import * as backfill from "./functions/backfill"
-import env from "../environment"
-
-export interface Migration {
-  type: string
-  name: string
-  opts?: object
-  fn: Function
-  silent?: boolean
-  preventRetry?: boolean
-}
 
 /**
- * e.g.
- * {
- *   tenantIds: ['bb'],
- *   force: {
- *    global: ['quota_1']
- *   }
- * }
+ * Populate the migration function and additional configuration from
+ * the static migration definitions.
  */
-export interface MigrationOptions {
-  tenantIds?: string[]
-  force?: {
-    [type: string]: string[]
+export const buildMigrations = () => {
+  const definitions = migrations.DEFINITIONS
+  const serverMigrations: Migration[] = []
+
+  for (const definition of definitions) {
+    switch (definition.name) {
+      case MigrationName.USER_EMAIL_VIEW_CASING: {
+        serverMigrations.push({
+          ...definition,
+          fn: userEmailViewCasing.run,
+        })
+        break
+      }
+      case MigrationName.QUOTAS_1: {
+        serverMigrations.push({
+          ...definition,
+          fn: quota1.run,
+        })
+        break
+      }
+      case MigrationName.APP_URLS: {
+        serverMigrations.push({
+          ...definition,
+          appOpts: { all: true },
+          fn: appUrls.run,
+        })
+        break
+      }
+      case MigrationName.DEVELOPER_QUOTA: {
+        serverMigrations.push({
+          ...definition,
+          fn: developerQuota.run,
+        })
+        break
+      }
+      case MigrationName.PUBLISHED_APP_QUOTA: {
+        serverMigrations.push({
+          ...definition,
+          fn: publishedAppsQuota.run,
+        })
+        break
+      }
+      case MigrationName.EVENT_APP_BACKFILL: {
+        serverMigrations.push({
+          ...definition,
+          appOpts: { all: true },
+          fn: backfill.app.run,
+          silent: !!env.SELF_HOSTED, // reduce noisy logging
+          preventRetry: !!env.SELF_HOSTED, // only ever run once
+        })
+        break
+      }
+      case MigrationName.EVENT_GLOBAL_BACKFILL: {
+        serverMigrations.push({
+          ...definition,
+          fn: backfill.global.run,
+          silent: !!env.SELF_HOSTED, // reduce noisy logging
+          preventRetry: !!env.SELF_HOSTED, // only ever run once
+        })
+        break
+      }
+      case MigrationName.EVENT_INSTALLATION_BACKFILL: {
+        serverMigrations.push({
+          ...definition,
+          fn: backfill.installation.run,
+          silent: !!env.SELF_HOSTED, // reduce noisy logging
+          preventRetry: !!env.SELF_HOSTED, // only ever run once
+        })
+        break
+      }
+    }
   }
+
+  return serverMigrations
 }
 
-export const MIGRATIONS: Migration[] = [
-  {
-    type: migrations.MIGRATION_TYPES.GLOBAL,
-    name: "user_email_view_casing",
-    fn: userEmailViewCasing.run,
-  },
-  {
-    type: migrations.MIGRATION_TYPES.GLOBAL,
-    name: "quotas_1",
-    fn: quota1.run,
-  },
-  {
-    type: migrations.MIGRATION_TYPES.APP,
-    name: "app_urls",
-    opts: { all: true },
-    fn: appUrls.run,
-  },
-  {
-    type: migrations.MIGRATION_TYPES.GLOBAL,
-    name: "developer_quota",
-    fn: developerQuota.run,
-  },
-  {
-    type: migrations.MIGRATION_TYPES.GLOBAL,
-    name: "published_apps_quota",
-    fn: publishedAppsQuota.run,
-  },
-  {
-    type: migrations.MIGRATION_TYPES.APP,
-    name: "event_app_backfill",
-    opts: { all: true },
-    fn: backfill.app.run,
-    silent: !!env.SELF_HOSTED, // reduce noisy logging
-    preventRetry: !!env.SELF_HOSTED, // only ever run once
-  },
-  {
-    type: migrations.MIGRATION_TYPES.GLOBAL,
-    name: "event_global_backfill",
-    fn: backfill.global.run,
-    silent: !!env.SELF_HOSTED, // reduce noisy logging
-    preventRetry: !!env.SELF_HOSTED, // only ever run once
-  },
-  {
-    type: migrations.MIGRATION_TYPES.INSTALLATION,
-    name: "event_installation_backfill",
-    fn: backfill.installation.run,
-    silent: !!env.SELF_HOSTED, // reduce noisy logging
-    preventRetry: !!env.SELF_HOSTED, // only ever run once
-  },
-]
+export const MIGRATIONS = buildMigrations()
 
 export const migrate = async (options?: MigrationOptions) => {
   if (env.SELF_HOSTED) {
diff --git a/packages/server/src/migrations/tests/index.spec.ts b/packages/server/src/migrations/tests/index.spec.ts
index 40dbd7ef73..ca30fbca06 100644
--- a/packages/server/src/migrations/tests/index.spec.ts
+++ b/packages/server/src/migrations/tests/index.spec.ts
@@ -1,4 +1,11 @@
-import { events, migrations, tenancy } from "@budibase/backend-core"
+import {
+  events,
+  migrations,
+  tenancy,
+  DocumentTypes,
+  context,
+  db,
+} from "@budibase/backend-core"
 import TestConfig from "../../tests/utilities/TestConfiguration"
 import structures from "../../tests/utilities/structures"
 import { MIGRATIONS } from "../"
@@ -7,6 +14,15 @@ import * as helpers from "./helpers"
 const { mocks } = require("@budibase/backend-core/tests")
 const timestamp = mocks.date.MOCK_DATE.toISOString()
 
+const clearMigrations = async () => {
+  const dbs = [context.getDevAppDB(), context.getProdAppDB()]
+  for (const db of dbs) {
+    const doc = await db.get(DocumentTypes.MIGRATIONS)
+    const newDoc = { _id: doc._id, _rev: doc._rev }
+    await db.put(newDoc)
+  }
+}
+
 describe("migrations", () => {
   const config = new TestConfig()
 
@@ -21,6 +37,7 @@ describe("migrations", () => {
   describe("backfill", () => {
     it("runs app db migration", async () => {
       await config.doInContext(null, async () => {
+        await clearMigrations()
         await config.createAutomation()
         await config.createAutomation(structures.newAutomation())
         await config.createDatasource()
@@ -71,6 +88,7 @@ describe("migrations", () => {
 
   it("runs global db migration", async () => {
     await config.doInContext(null, async () => {
+      await clearMigrations()
       const appId = config.prodAppId
       const roles = { [appId]: "role_12345" }
       await config.createUser(undefined, undefined, false, true, roles) // admin only
diff --git a/packages/types/src/core/index.ts b/packages/types/src/core/index.ts
deleted file mode 100644
index bed86d4f9f..0000000000
--- a/packages/types/src/core/index.ts
+++ /dev/null
@@ -1,3 +0,0 @@
-export * from "./hosting"
-export * from "./context"
-export * from "./identification"
diff --git a/packages/types/src/documents/account/account.ts b/packages/types/src/documents/account/account.ts
index a31ccda59f..344b699137 100644
--- a/packages/types/src/documents/account/account.ts
+++ b/packages/types/src/documents/account/account.ts
@@ -1,4 +1,4 @@
-import { Hosting } from "../../core"
+import { Hosting } from "../../sdk"
 
 export interface CreateAccount {
   email: string
diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts
index b0c3e10d88..b7453c7349 100644
--- a/packages/types/src/index.ts
+++ b/packages/types/src/index.ts
@@ -1,4 +1,4 @@
 export * from "./documents"
-export * from "./events"
-export * from "./licensing"
-export * from "./core"
+export * from "./sdk/events"
+export * from "./sdk/licensing"
+export * from "./sdk"
diff --git a/packages/types/src/core/context.ts b/packages/types/src/sdk/context.ts
similarity index 87%
rename from packages/types/src/core/context.ts
rename to packages/types/src/sdk/context.ts
index bf433d5e5f..0500b31faa 100644
--- a/packages/types/src/core/context.ts
+++ b/packages/types/src/sdk/context.ts
@@ -1,5 +1,5 @@
 import { User, Account } from "../documents"
-import { IdentityType } from "./identification"
+import { IdentityType } from "./events/identification"
 
 export interface BaseContext {
   _id: string
diff --git a/packages/types/src/events/account.ts b/packages/types/src/sdk/events/account.ts
similarity index 100%
rename from packages/types/src/events/account.ts
rename to packages/types/src/sdk/events/account.ts
diff --git a/packages/types/src/events/app.ts b/packages/types/src/sdk/events/app.ts
similarity index 100%
rename from packages/types/src/events/app.ts
rename to packages/types/src/sdk/events/app.ts
diff --git a/packages/types/src/events/auth.ts b/packages/types/src/sdk/events/auth.ts
similarity index 100%
rename from packages/types/src/events/auth.ts
rename to packages/types/src/sdk/events/auth.ts
diff --git a/packages/types/src/events/automation.ts b/packages/types/src/sdk/events/automation.ts
similarity index 100%
rename from packages/types/src/events/automation.ts
rename to packages/types/src/sdk/events/automation.ts
diff --git a/packages/types/src/events/backfill.ts b/packages/types/src/sdk/events/backfill.ts
similarity index 100%
rename from packages/types/src/events/backfill.ts
rename to packages/types/src/sdk/events/backfill.ts
diff --git a/packages/types/src/events/datasource.ts b/packages/types/src/sdk/events/datasource.ts
similarity index 100%
rename from packages/types/src/events/datasource.ts
rename to packages/types/src/sdk/events/datasource.ts
diff --git a/packages/types/src/events/email.ts b/packages/types/src/sdk/events/email.ts
similarity index 100%
rename from packages/types/src/events/email.ts
rename to packages/types/src/sdk/events/email.ts
diff --git a/packages/types/src/events/event.ts b/packages/types/src/sdk/events/event.ts
similarity index 100%
rename from packages/types/src/events/event.ts
rename to packages/types/src/sdk/events/event.ts
diff --git a/packages/types/src/core/identification.ts b/packages/types/src/sdk/events/identification.ts
similarity index 97%
rename from packages/types/src/core/identification.ts
rename to packages/types/src/sdk/events/identification.ts
index b7ce0fbaf4..1b9c43d2e7 100644
--- a/packages/types/src/core/identification.ts
+++ b/packages/types/src/sdk/events/identification.ts
@@ -1,4 +1,4 @@
-import { Hosting } from "."
+import { Hosting } from ".."
 
 // GROUPS
 
diff --git a/packages/types/src/events/index.ts b/packages/types/src/sdk/events/index.ts
similarity index 93%
rename from packages/types/src/events/index.ts
rename to packages/types/src/sdk/events/index.ts
index b88a078087..5822f66597 100644
--- a/packages/types/src/events/index.ts
+++ b/packages/types/src/sdk/events/index.ts
@@ -17,3 +17,4 @@ export * from "./user"
 export * from "./view"
 export * from "./account"
 export * from "./backfill"
+export * from "./identification"
diff --git a/packages/types/src/events/layout.ts b/packages/types/src/sdk/events/layout.ts
similarity index 100%
rename from packages/types/src/events/layout.ts
rename to packages/types/src/sdk/events/layout.ts
diff --git a/packages/types/src/events/license.ts b/packages/types/src/sdk/events/license.ts
similarity index 100%
rename from packages/types/src/events/license.ts
rename to packages/types/src/sdk/events/license.ts
diff --git a/packages/types/src/events/query.ts b/packages/types/src/sdk/events/query.ts
similarity index 100%
rename from packages/types/src/events/query.ts
rename to packages/types/src/sdk/events/query.ts
diff --git a/packages/types/src/events/role.ts b/packages/types/src/sdk/events/role.ts
similarity index 100%
rename from packages/types/src/events/role.ts
rename to packages/types/src/sdk/events/role.ts
diff --git a/packages/types/src/events/rows.ts b/packages/types/src/sdk/events/rows.ts
similarity index 100%
rename from packages/types/src/events/rows.ts
rename to packages/types/src/sdk/events/rows.ts
diff --git a/packages/types/src/events/screen.ts b/packages/types/src/sdk/events/screen.ts
similarity index 100%
rename from packages/types/src/events/screen.ts
rename to packages/types/src/sdk/events/screen.ts
diff --git a/packages/types/src/events/serve.ts b/packages/types/src/sdk/events/serve.ts
similarity index 100%
rename from packages/types/src/events/serve.ts
rename to packages/types/src/sdk/events/serve.ts
diff --git a/packages/types/src/events/table.ts b/packages/types/src/sdk/events/table.ts
similarity index 100%
rename from packages/types/src/events/table.ts
rename to packages/types/src/sdk/events/table.ts
diff --git a/packages/types/src/events/user.ts b/packages/types/src/sdk/events/user.ts
similarity index 100%
rename from packages/types/src/events/user.ts
rename to packages/types/src/sdk/events/user.ts
diff --git a/packages/types/src/events/version.ts b/packages/types/src/sdk/events/version.ts
similarity index 100%
rename from packages/types/src/events/version.ts
rename to packages/types/src/sdk/events/version.ts
diff --git a/packages/types/src/events/view.ts b/packages/types/src/sdk/events/view.ts
similarity index 95%
rename from packages/types/src/events/view.ts
rename to packages/types/src/sdk/events/view.ts
index f75bc4263f..452094d2f4 100644
--- a/packages/types/src/events/view.ts
+++ b/packages/types/src/sdk/events/view.ts
@@ -1,4 +1,4 @@
-import { ViewCalculation } from "../documents"
+import { ViewCalculation } from "../../documents"
 import { BaseEvent, TableExportFormat } from "./event"
 
 export interface ViewCreatedEvent extends BaseEvent {
diff --git a/packages/types/src/core/hosting.ts b/packages/types/src/sdk/hosting.ts
similarity index 100%
rename from packages/types/src/core/hosting.ts
rename to packages/types/src/sdk/hosting.ts
diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts
new file mode 100644
index 0000000000..0f2eee6e13
--- /dev/null
+++ b/packages/types/src/sdk/index.ts
@@ -0,0 +1,5 @@
+export * from "./hosting"
+export * from "./context"
+export * from "./events"
+export * from "./licensing"
+export * from "./migrations"
diff --git a/packages/types/src/licensing/index.ts b/packages/types/src/sdk/licensing/index.ts
similarity index 100%
rename from packages/types/src/licensing/index.ts
rename to packages/types/src/sdk/licensing/index.ts
diff --git a/packages/types/src/licensing/license.ts b/packages/types/src/sdk/licensing/license.ts
similarity index 100%
rename from packages/types/src/licensing/license.ts
rename to packages/types/src/sdk/licensing/license.ts
diff --git a/packages/types/src/sdk/migrations.ts b/packages/types/src/sdk/migrations.ts
new file mode 100644
index 0000000000..bb32d2e045
--- /dev/null
+++ b/packages/types/src/sdk/migrations.ts
@@ -0,0 +1,54 @@
+export interface Migration extends MigrationDefinition {
+  appOpts?: object
+  fn: Function
+  silent?: boolean
+  preventRetry?: boolean
+}
+
+export enum MigrationType {
+  // run once per tenant, recorded in global db, global db is provided as an argument
+  GLOBAL = "global",
+  // run per app, recorded in each app db, app db is provided as an argument
+  APP = "app",
+  // run once, recorded in global info db, global info db is provided as an argument
+  INSTALLATION = "installation",
+}
+
+export interface MigrationNoOpOptions {
+  type: MigrationType
+  tenantId: string
+  appId?: string
+}
+
+/**
+ * e.g.
+ * {
+ *   tenantIds: ['bb'],
+ *   force: {
+ *    global: ['quota_1']
+ *   }
+ * }
+ */
+export interface MigrationOptions {
+  tenantIds?: string[]
+  force?: {
+    [type: string]: string[]
+  }
+  noOp?: MigrationNoOpOptions
+}
+
+export enum MigrationName {
+  USER_EMAIL_VIEW_CASING = "user_email_view_casing",
+  QUOTAS_1 = "quotas_1",
+  APP_URLS = "app_urls",
+  DEVELOPER_QUOTA = "developer_quota",
+  PUBLISHED_APP_QUOTA = "published_apps_quota",
+  EVENT_APP_BACKFILL = "event_app_backfill",
+  EVENT_GLOBAL_BACKFILL = "event_global_backfill",
+  EVENT_INSTALLATION_BACKFILL = "event_installation_backfill",
+}
+
+export interface MigrationDefinition {
+  type: MigrationType
+  name: MigrationName
+}
diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts
index a35a44007e..cb2b439c7b 100644
--- a/packages/worker/src/sdk/users/users.ts
+++ b/packages/worker/src/sdk/users/users.ts
@@ -13,7 +13,9 @@ import {
   sessions,
   HTTPError,
   accounts,
+  migrations,
 } from "@budibase/backend-core"
+import { MigrationType } from "@budibase/types"
 
 /**
  * Retrieves all users from the current tenancy.
@@ -146,7 +148,12 @@ export const save = async (
     await eventHelpers.handleSaveEvents(user, dbUser)
 
     if (env.MULTI_TENANCY) {
-      await tenancy.tryAddTenant(tenantId, _id, email)
+      const afterCreateTenant = () =>
+        migrations.backPopulateMigrations({
+          type: MigrationType.GLOBAL,
+          tenantId,
+        })
+      await tenancy.tryAddTenant(tenantId, _id, email, afterCreateTenant)
     }
     await cache.user.invalidateUser(response.id)
     // let server know to sync user
diff --git a/scripts/link-dependencies.sh b/scripts/link-dependencies.sh
index cf9c536b1d..448a48feb0 100755
--- a/scripts/link-dependencies.sh
+++ b/scripts/link-dependencies.sh
@@ -13,6 +13,10 @@ cd packages/types
 yarn link
 cd -
 
+echo "Linking bbui"
+cd packages/bbui 
+yarn link
+cd -
 
 if [ -d "../budibase-pro" ]; then
   cd ../budibase-pro
@@ -52,5 +56,7 @@ if [ -d "../account-portal" ]; then
     yarn link "@budibase/pro"
   fi
 
-  cd -
+  cd ../ui
+  echo "Linking bbui to account-portal"
+  yarn link "@budibase/bbui"
 fi