diff --git a/packages/backend-core/src/events/analytics.ts b/packages/backend-core/src/events/analytics.ts index a7e2067fd6..802b6d6314 100644 --- a/packages/backend-core/src/events/analytics.ts +++ b/packages/backend-core/src/events/analytics.ts @@ -1,5 +1,5 @@ import env from "../environment" -import * as tenancy from "../tenancy" +import tenancy from "../tenancy" import * as dbUtils from "../db/utils" import { Configs } from "../constants" import { withCache, TTL, CacheKeys } from "../cache/generic" diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 5166494883..6eb6b14bc4 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -12,6 +12,7 @@ import sessions from "./security/sessions" import deprovisioning from "./context/deprovision" import auth from "./auth" import constants from "./constants" +import * as dbConstants from "./db/constants" // mimic the outer package exports import * as db from "./pkg/db" @@ -20,7 +21,6 @@ import * as utils from "./pkg/utils" import redis from "./pkg/redis" import cache from "./pkg/cache" import context from "./pkg/context" -const StaticDatabases = db.StaticDatabases const init = (opts: any = {}) => { db.init(opts.db) @@ -28,8 +28,8 @@ const init = (opts: any = {}) => { const core = { init, - StaticDatabases, db, + ...dbConstants, redis, objectStore, utils, @@ -37,6 +37,7 @@ const core = { cache, auth, constants, + ...constants, migrations, env, accounts, diff --git a/packages/backend-core/src/migrations/definitions.ts b/packages/backend-core/src/migrations/definitions.ts new file mode 100644 index 0000000000..745c8718c9 --- /dev/null +++ b/packages/backend-core/src/migrations/definitions.ts @@ -0,0 +1,40 @@ +import { + MigrationType, + MigrationName, + MigrationDefinition, +} from "@budibase/types" + +export const DEFINITIONS: MigrationDefinition[] = [ + { + type: MigrationType.GLOBAL, + name: MigrationName.USER_EMAIL_VIEW_CASING, + }, + { + type: MigrationType.GLOBAL, + name: MigrationName.QUOTAS_1, + }, + { + type: MigrationType.APP, + name: MigrationName.APP_URLS, + }, + { + type: MigrationType.GLOBAL, + name: MigrationName.DEVELOPER_QUOTA, + }, + { + type: MigrationType.GLOBAL, + name: MigrationName.PUBLISHED_APP_QUOTA, + }, + { + type: MigrationType.APP, + name: MigrationName.EVENT_APP_BACKFILL, + }, + { + type: MigrationType.GLOBAL, + name: MigrationName.EVENT_GLOBAL_BACKFILL, + }, + { + type: MigrationType.INSTALLATION, + name: MigrationName.EVENT_INSTALLATION_BACKFILL, + }, +] diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js deleted file mode 100644 index 5c3ab4f4a5..0000000000 --- a/packages/backend-core/src/migrations/index.js +++ /dev/null @@ -1,163 +0,0 @@ -const { DEFAULT_TENANT_ID } = require("../constants") -const { doWithDB } = require("../db") -const { DocumentTypes, StaticDatabases } = require("../db/constants") -const { getAllApps } = require("../db/utils") -const environment = require("../environment") -const { - doInTenant, - getTenantIds, - getGlobalDBName, - getTenantId, -} = require("../tenancy") -const context = require("../context") - -exports.MIGRATION_TYPES = { - GLOBAL: "global", // run once per tenant, recorded in global db, global db is provided as an argument - APP: "app", // run per app, recorded in each app db, app db is provided as an argument - INSTALLATION: "installation", // run once, recorded in global info db, global info db is provided as an argument -} - -exports.getMigrationsDoc = async db => { - // get the migrations doc - try { - return await db.get(DocumentTypes.MIGRATIONS) - } catch (err) { - if (err.status && err.status === 404) { - return { _id: DocumentTypes.MIGRATIONS } - } else { - console.error(err) - throw err - } - } -} - -exports.runMigration = async (migration, options = {}) => { - const migrationType = migration.type - let tenantId - if (migrationType !== exports.MIGRATION_TYPES.INSTALLATION) { - tenantId = getTenantId() - } - const migrationName = migration.name - const silent = migration.silent - - const log = message => { - if (!silent) { - console.log(message) - } - } - - // get the db to store the migration in - let dbNames - if (migrationType === exports.MIGRATION_TYPES.GLOBAL) { - dbNames = [getGlobalDBName()] - } else if (migrationType === exports.MIGRATION_TYPES.APP) { - const apps = await getAllApps(migration.opts) - dbNames = apps.map(app => app.appId) - } else if (migrationType === exports.MIGRATION_TYPES.INSTALLATION) { - dbNames = [StaticDatabases.PLATFORM_INFO.name] - } else { - throw new Error(`Unrecognised migration type [${migrationType}]`) - } - - const length = dbNames.length - let count = 0 - - // run the migration against each db - for (const dbName of dbNames) { - count++ - const lengthStatement = length > 1 ? `[${count}/${length}]` : "" - - await doWithDB(dbName, async db => { - try { - const doc = await exports.getMigrationsDoc(db) - - // exit if the migration has been performed already - if (doc[migrationName]) { - if ( - options.force && - options.force[migrationType] && - options.force[migrationType].includes(migrationName) - ) { - log( - `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing` - ) - } else { - // the migration has already been performed - return - } - } - - log( - `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}` - ) - - if (migration.preventRetry) { - // eagerly set the completion date - // so that we never run this migration twice even upon failure - doc[migrationName] = Date.now() - const response = await db.put(doc) - doc._rev = response.rev - } - - // run the migration with tenant context - if (migrationType === exports.MIGRATION_TYPES.APP) { - await context.doInAppContext(db.name, async () => { - await migration.fn(db) - }) - } else { - await migration.fn(db) - } - - log( - `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete` - ) - - // mark as complete - doc[migrationName] = Date.now() - await db.put(doc) - } catch (err) { - console.error( - `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `, - err - ) - throw err - } - }) - } -} - -exports.runMigrations = async (migrations, options = {}) => { - let tenantIds - if (environment.MULTI_TENANCY) { - if (!options.tenantIds || !options.tenantIds.length) { - // run for all tenants - tenantIds = await getTenantIds() - } else { - tenantIds = options.tenantIds - } - } else { - // single tenancy - tenantIds = [DEFAULT_TENANT_ID] - } - - if (tenantIds.length > 1) { - console.log(`Checking migrations for ${tenantIds.length} tenants`) - } else { - console.log("Checking migrations") - } - - let count = 0 - // for all tenants - for (const tenantId of tenantIds) { - count++ - if (tenantIds.length > 1) { - console.log(`Progress [${count}/${tenantIds.length}]`) - } - // for all migrations - for (const migration of migrations) { - // run the migration - await doInTenant(tenantId, () => exports.runMigration(migration, options)) - } - } - console.log("Migrations complete") -} diff --git a/packages/backend-core/src/migrations/index.ts b/packages/backend-core/src/migrations/index.ts new file mode 100644 index 0000000000..bce0cfc75c --- /dev/null +++ b/packages/backend-core/src/migrations/index.ts @@ -0,0 +1,2 @@ +export * from "./migrations" +export * from "./definitions" diff --git a/packages/backend-core/src/migrations/migrations.ts b/packages/backend-core/src/migrations/migrations.ts new file mode 100644 index 0000000000..b926334317 --- /dev/null +++ b/packages/backend-core/src/migrations/migrations.ts @@ -0,0 +1,189 @@ +import { DEFAULT_TENANT_ID } from "../constants" +import { doWithDB } from "../db" +import { DocumentTypes, StaticDatabases } from "../db/constants" +import { getAllApps } from "../db/utils" +import environment from "../environment" +import { + doInTenant, + getTenantIds, + getGlobalDBName, + getTenantId, +} from "../tenancy" +import context from "../context" +import { DEFINITIONS } from "." +import { + Migration, + MigrationOptions, + MigrationType, + MigrationNoOpOptions, +} from "@budibase/types" + +export const getMigrationsDoc = async (db: any) => { + // get the migrations doc + try { + return await db.get(DocumentTypes.MIGRATIONS) + } catch (err: any) { + if (err.status && err.status === 404) { + return { _id: DocumentTypes.MIGRATIONS } + } else { + console.error(err) + throw err + } + } +} + +export const backPopulateMigrations = async (opts: MigrationNoOpOptions) => { + // filter migrations to the type and populate a no-op migration + const migrations: Migration[] = DEFINITIONS.filter( + def => def.type === opts.type + ).map(d => ({ ...d, fn: () => {} })) + await runMigrations(migrations, { noOp: opts }) +} + +export const runMigration = async ( + migration: Migration, + options: MigrationOptions = {} +) => { + const migrationType = migration.type + let tenantId: string + if (migrationType !== MigrationType.INSTALLATION) { + tenantId = getTenantId() + } + const migrationName = migration.name + const silent = migration.silent + + const log = (message: string) => { + if (!silent) { + console.log(message) + } + } + + // get the db to store the migration in + let dbNames + if (migrationType === MigrationType.GLOBAL) { + dbNames = [getGlobalDBName()] + } else if (migrationType === MigrationType.APP) { + if (options.noOp) { + dbNames = [options.noOp.appId] + } else { + const apps = await getAllApps(migration.appOpts) + dbNames = apps.map(app => app.appId) + } + } else if (migrationType === MigrationType.INSTALLATION) { + dbNames = [StaticDatabases.PLATFORM_INFO.name] + } else { + throw new Error(`Unrecognised migration type [${migrationType}]`) + } + + const length = dbNames.length + let count = 0 + + // run the migration against each db + for (const dbName of dbNames) { + count++ + const lengthStatement = length > 1 ? `[${count}/${length}]` : "" + + await doWithDB(dbName, async (db: any) => { + try { + const doc = await exports.getMigrationsDoc(db) + + // the migration has already been run + if (doc[migrationName]) { + // check for force + if ( + options.force && + options.force[migrationType] && + options.force[migrationType].includes(migrationName) + ) { + log( + `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing` + ) + } else { + // no force, exit + return + } + } + + // check if the migration is not a no-op + if (!options.noOp) { + log( + `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}` + ) + + if (migration.preventRetry) { + // eagerly set the completion date + // so that we never run this migration twice even upon failure + doc[migrationName] = Date.now() + const response = await db.put(doc) + doc._rev = response.rev + } + + // run the migration + if (migrationType === MigrationType.APP) { + await context.doInAppContext(db.name, async () => { + await migration.fn(db) + }) + } else { + await migration.fn(db) + } + + log( + `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete` + ) + } + + // mark as complete + doc[migrationName] = Date.now() + await db.put(doc) + } catch (err) { + console.error( + `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `, + err + ) + throw err + } + }) + } +} + +export const runMigrations = async ( + migrations: Migration[], + options: MigrationOptions = {} +) => { + let tenantIds + + if (environment.MULTI_TENANCY) { + if (options.noOp) { + tenantIds = [options.noOp.tenantId] + } else if (!options.tenantIds || !options.tenantIds.length) { + // run for all tenants + tenantIds = await getTenantIds() + } else { + tenantIds = options.tenantIds + } + } else { + // single tenancy + tenantIds = [DEFAULT_TENANT_ID] + } + + if (tenantIds.length > 1) { + console.log(`Checking migrations for ${tenantIds.length} tenants`) + } else { + console.log("Checking migrations") + } + + let count = 0 + // for all tenants + for (const tenantId of tenantIds) { + count++ + if (tenantIds.length > 1) { + console.log(`Progress [${count}/${tenantIds.length}]`) + } + // for all migrations + for (const migration of migrations) { + // run the migration + await doInTenant(tenantId, () => runMigration(migration, options)) + } + } + console.log("Migrations complete") +} diff --git a/packages/backend-core/src/tenancy/index.js b/packages/backend-core/src/tenancy/index.js deleted file mode 100644 index c847033a12..0000000000 --- a/packages/backend-core/src/tenancy/index.js +++ /dev/null @@ -1,4 +0,0 @@ -module.exports = { - ...require("../context"), - ...require("./tenancy"), -} diff --git a/packages/backend-core/src/tenancy/index.ts b/packages/backend-core/src/tenancy/index.ts new file mode 100644 index 0000000000..e0006abab2 --- /dev/null +++ b/packages/backend-core/src/tenancy/index.ts @@ -0,0 +1,9 @@ +import * as context from "../context" +import * as tenancy from "./tenancy" + +const pkg = { + ...context, + ...tenancy, +} + +export = pkg diff --git a/packages/backend-core/src/tenancy/tenancy.js b/packages/backend-core/src/tenancy/tenancy.ts similarity index 68% rename from packages/backend-core/src/tenancy/tenancy.js rename to packages/backend-core/src/tenancy/tenancy.ts index b9d5ad7fbe..041f694d34 100644 --- a/packages/backend-core/src/tenancy/tenancy.js +++ b/packages/backend-core/src/tenancy/tenancy.ts @@ -1,18 +1,18 @@ -const { doWithDB } = require("../db") -const { StaticDatabases } = require("../db/constants") -const { baseGlobalDBName } = require("./utils") -const { +import { doWithDB } from "../db" +import { StaticDatabases } from "../db/constants" +import { baseGlobalDBName } from "./utils" +import { getTenantId, DEFAULT_TENANT_ID, isMultiTenant, getTenantIDFromAppID, -} = require("../context") -const env = require("../environment") +} from "../context" +import env from "../environment" const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name -exports.addTenantToUrl = url => { +export const addTenantToUrl = (url: string) => { const tenantId = getTenantId() if (isMultiTenant()) { @@ -23,8 +23,8 @@ exports.addTenantToUrl = url => { return url } -exports.doesTenantExist = async tenantId => { - return doWithDB(PLATFORM_INFO_DB, async db => { +export const doesTenantExist = async (tenantId: string) => { + return doWithDB(PLATFORM_INFO_DB, async (db: any) => { let tenants try { tenants = await db.get(TENANT_DOC) @@ -40,9 +40,14 @@ exports.doesTenantExist = async tenantId => { }) } -exports.tryAddTenant = async (tenantId, userId, email) => { - return doWithDB(PLATFORM_INFO_DB, async db => { - const getDoc = async id => { +export const tryAddTenant = async ( + tenantId: string, + userId: string, + email: string, + afterCreateTenant: () => Promise +) => { + return doWithDB(PLATFORM_INFO_DB, async (db: any) => { + const getDoc = async (id: string) => { if (!id) { return null } @@ -76,12 +81,13 @@ exports.tryAddTenant = async (tenantId, userId, email) => { if (tenants.tenantIds.indexOf(tenantId) === -1) { tenants.tenantIds.push(tenantId) promises.push(db.put(tenants)) + await afterCreateTenant() } await Promise.all(promises) }) } -exports.getGlobalDBName = (tenantId = null) => { +export const getGlobalDBName = (tenantId?: string) => { // tenant ID can be set externally, for example user API where // new tenants are being created, this may be the case if (!tenantId) { @@ -90,12 +96,12 @@ exports.getGlobalDBName = (tenantId = null) => { return baseGlobalDBName(tenantId) } -exports.doWithGlobalDB = (tenantId, cb) => { - return doWithDB(exports.getGlobalDBName(tenantId), cb) +export const doWithGlobalDB = (tenantId: string, cb: any) => { + return doWithDB(getGlobalDBName(tenantId), cb) } -exports.lookupTenantId = async userId => { - return doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => { +export const lookupTenantId = async (userId: string) => { + return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: any) => { let tenantId = env.MULTI_TENANCY ? DEFAULT_TENANT_ID : null try { const doc = await db.get(userId) @@ -110,8 +116,8 @@ exports.lookupTenantId = async userId => { } // lookup, could be email or userId, either will return a doc -exports.getTenantUser = async identifier => { - return doWithDB(PLATFORM_INFO_DB, async db => { +export const getTenantUser = async (identifier: string) => { + return doWithDB(PLATFORM_INFO_DB, async (db: any) => { try { return await db.get(identifier) } catch (err) { @@ -120,7 +126,7 @@ exports.getTenantUser = async identifier => { }) } -exports.isUserInAppTenant = (appId, user = null) => { +export const isUserInAppTenant = (appId: string, user: any) => { let userTenantId if (user) { userTenantId = user.tenantId || DEFAULT_TENANT_ID @@ -131,8 +137,8 @@ exports.isUserInAppTenant = (appId, user = null) => { return tenantId === userTenantId } -exports.getTenantIds = async () => { - return doWithDB(PLATFORM_INFO_DB, async db => { +export const getTenantIds = async () => { + return doWithDB(PLATFORM_INFO_DB, async (db: any) => { let tenants try { tenants = await db.get(TENANT_DOC) diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 67c810765a..7d78e5dded 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -52,8 +52,8 @@ const { } = require("@budibase/backend-core/context") import { getUniqueRows } from "../../utilities/usageQuota/rows" import { quotas } from "@budibase/pro" -import { errors, events } from "@budibase/backend-core" -import { App } from "@budibase/types" +import { errors, events, migrations } from "@budibase/backend-core" +import { App, MigrationType } from "@budibase/types" const URL_REGEX_SLASH = /\/|\\/g @@ -319,6 +319,12 @@ const creationEvents = async (request: any, app: App) => { } const appPostCreate = async (ctx: any, app: App) => { + const tenantId = getTenantId() + await migrations.backPopulateMigrations({ + type: MigrationType.APP, + tenantId, + appId: app.appId, + }) await creationEvents(ctx.request, app) // app import & template creation if (ctx.request.body.useTemplate === "true") { diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts index c93f628c71..ebba4f420d 100644 --- a/packages/server/src/migrations/index.ts +++ b/packages/server/src/migrations/index.ts @@ -1,4 +1,6 @@ import { migrations, redis } from "@budibase/backend-core" +import { Migration, MigrationOptions, MigrationName } from "@budibase/types" +import env from "../environment" // migration functions import * as userEmailViewCasing from "./functions/userEmailViewCasing" @@ -7,83 +9,88 @@ import * as appUrls from "./functions/appUrls" import * as developerQuota from "./functions/developerQuota" import * as publishedAppsQuota from "./functions/publishedAppsQuota" import * as backfill from "./functions/backfill" -import env from "../environment" - -export interface Migration { - type: string - name: string - opts?: object - fn: Function - silent?: boolean - preventRetry?: boolean -} /** - * e.g. - * { - * tenantIds: ['bb'], - * force: { - * global: ['quota_1'] - * } - * } + * Populate the migration function and additional configuration from + * the static migration definitions. */ -export interface MigrationOptions { - tenantIds?: string[] - force?: { - [type: string]: string[] +export const buildMigrations = () => { + const definitions = migrations.DEFINITIONS + const serverMigrations: Migration[] = [] + + for (const definition of definitions) { + switch (definition.name) { + case MigrationName.USER_EMAIL_VIEW_CASING: { + serverMigrations.push({ + ...definition, + fn: userEmailViewCasing.run, + }) + break + } + case MigrationName.QUOTAS_1: { + serverMigrations.push({ + ...definition, + fn: quota1.run, + }) + break + } + case MigrationName.APP_URLS: { + serverMigrations.push({ + ...definition, + appOpts: { all: true }, + fn: appUrls.run, + }) + break + } + case MigrationName.DEVELOPER_QUOTA: { + serverMigrations.push({ + ...definition, + fn: developerQuota.run, + }) + break + } + case MigrationName.PUBLISHED_APP_QUOTA: { + serverMigrations.push({ + ...definition, + fn: publishedAppsQuota.run, + }) + break + } + case MigrationName.EVENT_APP_BACKFILL: { + serverMigrations.push({ + ...definition, + appOpts: { all: true }, + fn: backfill.app.run, + silent: !!env.SELF_HOSTED, // reduce noisy logging + preventRetry: !!env.SELF_HOSTED, // only ever run once + }) + break + } + case MigrationName.EVENT_GLOBAL_BACKFILL: { + serverMigrations.push({ + ...definition, + fn: backfill.global.run, + silent: !!env.SELF_HOSTED, // reduce noisy logging + preventRetry: !!env.SELF_HOSTED, // only ever run once + }) + break + } + case MigrationName.EVENT_INSTALLATION_BACKFILL: { + serverMigrations.push({ + ...definition, + fn: backfill.installation.run, + silent: !!env.SELF_HOSTED, // reduce noisy logging + preventRetry: !!env.SELF_HOSTED, // only ever run once + }) + break + } + } } + + return serverMigrations } -export const MIGRATIONS: Migration[] = [ - { - type: migrations.MIGRATION_TYPES.GLOBAL, - name: "user_email_view_casing", - fn: userEmailViewCasing.run, - }, - { - type: migrations.MIGRATION_TYPES.GLOBAL, - name: "quotas_1", - fn: quota1.run, - }, - { - type: migrations.MIGRATION_TYPES.APP, - name: "app_urls", - opts: { all: true }, - fn: appUrls.run, - }, - { - type: migrations.MIGRATION_TYPES.GLOBAL, - name: "developer_quota", - fn: developerQuota.run, - }, - { - type: migrations.MIGRATION_TYPES.GLOBAL, - name: "published_apps_quota", - fn: publishedAppsQuota.run, - }, - { - type: migrations.MIGRATION_TYPES.APP, - name: "event_app_backfill", - opts: { all: true }, - fn: backfill.app.run, - silent: !!env.SELF_HOSTED, // reduce noisy logging - preventRetry: !!env.SELF_HOSTED, // only ever run once - }, - { - type: migrations.MIGRATION_TYPES.GLOBAL, - name: "event_global_backfill", - fn: backfill.global.run, - silent: !!env.SELF_HOSTED, // reduce noisy logging - preventRetry: !!env.SELF_HOSTED, // only ever run once - }, - { - type: migrations.MIGRATION_TYPES.INSTALLATION, - name: "event_installation_backfill", - fn: backfill.installation.run, - silent: !!env.SELF_HOSTED, // reduce noisy logging - preventRetry: !!env.SELF_HOSTED, // only ever run once - }, -] +export const MIGRATIONS = buildMigrations() export const migrate = async (options?: MigrationOptions) => { if (env.SELF_HOSTED) { diff --git a/packages/server/src/migrations/tests/index.spec.ts b/packages/server/src/migrations/tests/index.spec.ts index 40dbd7ef73..ca30fbca06 100644 --- a/packages/server/src/migrations/tests/index.spec.ts +++ b/packages/server/src/migrations/tests/index.spec.ts @@ -1,4 +1,11 @@ -import { events, migrations, tenancy } from "@budibase/backend-core" +import { + events, + migrations, + tenancy, + DocumentTypes, + context, + db, +} from "@budibase/backend-core" import TestConfig from "../../tests/utilities/TestConfiguration" import structures from "../../tests/utilities/structures" import { MIGRATIONS } from "../" @@ -7,6 +14,15 @@ import * as helpers from "./helpers" const { mocks } = require("@budibase/backend-core/tests") const timestamp = mocks.date.MOCK_DATE.toISOString() +const clearMigrations = async () => { + const dbs = [context.getDevAppDB(), context.getProdAppDB()] + for (const db of dbs) { + const doc = await db.get(DocumentTypes.MIGRATIONS) + const newDoc = { _id: doc._id, _rev: doc._rev } + await db.put(newDoc) + } +} + describe("migrations", () => { const config = new TestConfig() @@ -21,6 +37,7 @@ describe("migrations", () => { describe("backfill", () => { it("runs app db migration", async () => { await config.doInContext(null, async () => { + await clearMigrations() await config.createAutomation() await config.createAutomation(structures.newAutomation()) await config.createDatasource() @@ -71,6 +88,7 @@ describe("migrations", () => { it("runs global db migration", async () => { await config.doInContext(null, async () => { + await clearMigrations() const appId = config.prodAppId const roles = { [appId]: "role_12345" } await config.createUser(undefined, undefined, false, true, roles) // admin only diff --git a/packages/types/src/core/index.ts b/packages/types/src/core/index.ts deleted file mode 100644 index bed86d4f9f..0000000000 --- a/packages/types/src/core/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from "./hosting" -export * from "./context" -export * from "./identification" diff --git a/packages/types/src/documents/account/account.ts b/packages/types/src/documents/account/account.ts index a31ccda59f..344b699137 100644 --- a/packages/types/src/documents/account/account.ts +++ b/packages/types/src/documents/account/account.ts @@ -1,4 +1,4 @@ -import { Hosting } from "../../core" +import { Hosting } from "../../sdk" export interface CreateAccount { email: string diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index b0c3e10d88..b7453c7349 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -1,4 +1,4 @@ export * from "./documents" -export * from "./events" -export * from "./licensing" -export * from "./core" +export * from "./sdk/events" +export * from "./sdk/licensing" +export * from "./sdk" diff --git a/packages/types/src/core/context.ts b/packages/types/src/sdk/context.ts similarity index 87% rename from packages/types/src/core/context.ts rename to packages/types/src/sdk/context.ts index bf433d5e5f..0500b31faa 100644 --- a/packages/types/src/core/context.ts +++ b/packages/types/src/sdk/context.ts @@ -1,5 +1,5 @@ import { User, Account } from "../documents" -import { IdentityType } from "./identification" +import { IdentityType } from "./events/identification" export interface BaseContext { _id: string diff --git a/packages/types/src/events/account.ts b/packages/types/src/sdk/events/account.ts similarity index 100% rename from packages/types/src/events/account.ts rename to packages/types/src/sdk/events/account.ts diff --git a/packages/types/src/events/app.ts b/packages/types/src/sdk/events/app.ts similarity index 100% rename from packages/types/src/events/app.ts rename to packages/types/src/sdk/events/app.ts diff --git a/packages/types/src/events/auth.ts b/packages/types/src/sdk/events/auth.ts similarity index 100% rename from packages/types/src/events/auth.ts rename to packages/types/src/sdk/events/auth.ts diff --git a/packages/types/src/events/automation.ts b/packages/types/src/sdk/events/automation.ts similarity index 100% rename from packages/types/src/events/automation.ts rename to packages/types/src/sdk/events/automation.ts diff --git a/packages/types/src/events/backfill.ts b/packages/types/src/sdk/events/backfill.ts similarity index 100% rename from packages/types/src/events/backfill.ts rename to packages/types/src/sdk/events/backfill.ts diff --git a/packages/types/src/events/datasource.ts b/packages/types/src/sdk/events/datasource.ts similarity index 100% rename from packages/types/src/events/datasource.ts rename to packages/types/src/sdk/events/datasource.ts diff --git a/packages/types/src/events/email.ts b/packages/types/src/sdk/events/email.ts similarity index 100% rename from packages/types/src/events/email.ts rename to packages/types/src/sdk/events/email.ts diff --git a/packages/types/src/events/event.ts b/packages/types/src/sdk/events/event.ts similarity index 100% rename from packages/types/src/events/event.ts rename to packages/types/src/sdk/events/event.ts diff --git a/packages/types/src/core/identification.ts b/packages/types/src/sdk/events/identification.ts similarity index 97% rename from packages/types/src/core/identification.ts rename to packages/types/src/sdk/events/identification.ts index b7ce0fbaf4..1b9c43d2e7 100644 --- a/packages/types/src/core/identification.ts +++ b/packages/types/src/sdk/events/identification.ts @@ -1,4 +1,4 @@ -import { Hosting } from "." +import { Hosting } from ".." // GROUPS diff --git a/packages/types/src/events/index.ts b/packages/types/src/sdk/events/index.ts similarity index 93% rename from packages/types/src/events/index.ts rename to packages/types/src/sdk/events/index.ts index b88a078087..5822f66597 100644 --- a/packages/types/src/events/index.ts +++ b/packages/types/src/sdk/events/index.ts @@ -17,3 +17,4 @@ export * from "./user" export * from "./view" export * from "./account" export * from "./backfill" +export * from "./identification" diff --git a/packages/types/src/events/layout.ts b/packages/types/src/sdk/events/layout.ts similarity index 100% rename from packages/types/src/events/layout.ts rename to packages/types/src/sdk/events/layout.ts diff --git a/packages/types/src/events/license.ts b/packages/types/src/sdk/events/license.ts similarity index 100% rename from packages/types/src/events/license.ts rename to packages/types/src/sdk/events/license.ts diff --git a/packages/types/src/events/query.ts b/packages/types/src/sdk/events/query.ts similarity index 100% rename from packages/types/src/events/query.ts rename to packages/types/src/sdk/events/query.ts diff --git a/packages/types/src/events/role.ts b/packages/types/src/sdk/events/role.ts similarity index 100% rename from packages/types/src/events/role.ts rename to packages/types/src/sdk/events/role.ts diff --git a/packages/types/src/events/rows.ts b/packages/types/src/sdk/events/rows.ts similarity index 100% rename from packages/types/src/events/rows.ts rename to packages/types/src/sdk/events/rows.ts diff --git a/packages/types/src/events/screen.ts b/packages/types/src/sdk/events/screen.ts similarity index 100% rename from packages/types/src/events/screen.ts rename to packages/types/src/sdk/events/screen.ts diff --git a/packages/types/src/events/serve.ts b/packages/types/src/sdk/events/serve.ts similarity index 100% rename from packages/types/src/events/serve.ts rename to packages/types/src/sdk/events/serve.ts diff --git a/packages/types/src/events/table.ts b/packages/types/src/sdk/events/table.ts similarity index 100% rename from packages/types/src/events/table.ts rename to packages/types/src/sdk/events/table.ts diff --git a/packages/types/src/events/user.ts b/packages/types/src/sdk/events/user.ts similarity index 100% rename from packages/types/src/events/user.ts rename to packages/types/src/sdk/events/user.ts diff --git a/packages/types/src/events/version.ts b/packages/types/src/sdk/events/version.ts similarity index 100% rename from packages/types/src/events/version.ts rename to packages/types/src/sdk/events/version.ts diff --git a/packages/types/src/events/view.ts b/packages/types/src/sdk/events/view.ts similarity index 95% rename from packages/types/src/events/view.ts rename to packages/types/src/sdk/events/view.ts index f75bc4263f..452094d2f4 100644 --- a/packages/types/src/events/view.ts +++ b/packages/types/src/sdk/events/view.ts @@ -1,4 +1,4 @@ -import { ViewCalculation } from "../documents" +import { ViewCalculation } from "../../documents" import { BaseEvent, TableExportFormat } from "./event" export interface ViewCreatedEvent extends BaseEvent { diff --git a/packages/types/src/core/hosting.ts b/packages/types/src/sdk/hosting.ts similarity index 100% rename from packages/types/src/core/hosting.ts rename to packages/types/src/sdk/hosting.ts diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts new file mode 100644 index 0000000000..0f2eee6e13 --- /dev/null +++ b/packages/types/src/sdk/index.ts @@ -0,0 +1,5 @@ +export * from "./hosting" +export * from "./context" +export * from "./events" +export * from "./licensing" +export * from "./migrations" diff --git a/packages/types/src/licensing/index.ts b/packages/types/src/sdk/licensing/index.ts similarity index 100% rename from packages/types/src/licensing/index.ts rename to packages/types/src/sdk/licensing/index.ts diff --git a/packages/types/src/licensing/license.ts b/packages/types/src/sdk/licensing/license.ts similarity index 100% rename from packages/types/src/licensing/license.ts rename to packages/types/src/sdk/licensing/license.ts diff --git a/packages/types/src/sdk/migrations.ts b/packages/types/src/sdk/migrations.ts new file mode 100644 index 0000000000..bb32d2e045 --- /dev/null +++ b/packages/types/src/sdk/migrations.ts @@ -0,0 +1,54 @@ +export interface Migration extends MigrationDefinition { + appOpts?: object + fn: Function + silent?: boolean + preventRetry?: boolean +} + +export enum MigrationType { + // run once per tenant, recorded in global db, global db is provided as an argument + GLOBAL = "global", + // run per app, recorded in each app db, app db is provided as an argument + APP = "app", + // run once, recorded in global info db, global info db is provided as an argument + INSTALLATION = "installation", +} + +export interface MigrationNoOpOptions { + type: MigrationType + tenantId: string + appId?: string +} + +/** + * e.g. + * { + * tenantIds: ['bb'], + * force: { + * global: ['quota_1'] + * } + * } + */ +export interface MigrationOptions { + tenantIds?: string[] + force?: { + [type: string]: string[] + } + noOp?: MigrationNoOpOptions +} + +export enum MigrationName { + USER_EMAIL_VIEW_CASING = "user_email_view_casing", + QUOTAS_1 = "quotas_1", + APP_URLS = "app_urls", + DEVELOPER_QUOTA = "developer_quota", + PUBLISHED_APP_QUOTA = "published_apps_quota", + EVENT_APP_BACKFILL = "event_app_backfill", + EVENT_GLOBAL_BACKFILL = "event_global_backfill", + EVENT_INSTALLATION_BACKFILL = "event_installation_backfill", +} + +export interface MigrationDefinition { + type: MigrationType + name: MigrationName +} diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts index a35a44007e..cb2b439c7b 100644 --- a/packages/worker/src/sdk/users/users.ts +++ b/packages/worker/src/sdk/users/users.ts @@ -13,7 +13,9 @@ import { sessions, HTTPError, accounts, + migrations, } from "@budibase/backend-core" +import { MigrationType } from "@budibase/types" /** * Retrieves all users from the current tenancy. @@ -146,7 +148,12 @@ export const save = async ( await eventHelpers.handleSaveEvents(user, dbUser) if (env.MULTI_TENANCY) { - await tenancy.tryAddTenant(tenantId, _id, email) + const afterCreateTenant = () => + migrations.backPopulateMigrations({ + type: MigrationType.GLOBAL, + tenantId, + }) + await tenancy.tryAddTenant(tenantId, _id, email, afterCreateTenant) } await cache.user.invalidateUser(response.id) // let server know to sync user diff --git a/scripts/link-dependencies.sh b/scripts/link-dependencies.sh index cf9c536b1d..448a48feb0 100755 --- a/scripts/link-dependencies.sh +++ b/scripts/link-dependencies.sh @@ -13,6 +13,10 @@ cd packages/types yarn link cd - +echo "Linking bbui" +cd packages/bbui +yarn link +cd - if [ -d "../budibase-pro" ]; then cd ../budibase-pro @@ -52,5 +56,7 @@ if [ -d "../account-portal" ]; then yarn link "@budibase/pro" fi - cd - + cd ../ui + echo "Linking bbui to account-portal" + yarn link "@budibase/bbui" fi