diff --git a/packages/backend-core/db.js b/packages/backend-core/db.js index a7b38821a7..47854ca9c7 100644 --- a/packages/backend-core/db.js +++ b/packages/backend-core/db.js @@ -1,4 +1,5 @@ module.exports = { ...require("./src/db/utils"), ...require("./src/db/constants"), + ...require("./src/db/views"), } diff --git a/packages/backend-core/src/auth.js b/packages/backend-core/src/auth.js index 7f66d887ae..068dd99a95 100644 --- a/packages/backend-core/src/auth.js +++ b/packages/backend-core/src/auth.js @@ -12,6 +12,7 @@ const { tenancy, appTenancy, authError, + internalApi, } = require("./middleware") // Strategies @@ -42,4 +43,5 @@ module.exports = { buildAppTenancyMiddleware: appTenancy, auditLog, authError, + internalApi, } diff --git a/packages/backend-core/src/middleware/index.js b/packages/backend-core/src/middleware/index.js index cf8676a2bc..794aa707a7 100644 --- a/packages/backend-core/src/middleware/index.js +++ b/packages/backend-core/src/middleware/index.js @@ -7,6 +7,7 @@ const authenticated = require("./authenticated") const auditLog = require("./auditLog") const tenancy = require("./tenancy") const appTenancy = require("./appTenancy") +const internalApi = require("./internalApi") module.exports = { google, @@ -18,4 +19,5 @@ module.exports = { tenancy, appTenancy, authError, + internalApi, } diff --git a/packages/backend-core/src/middleware/internalApi.js b/packages/backend-core/src/middleware/internalApi.js new file mode 100644 index 0000000000..275d559a9e --- /dev/null +++ b/packages/backend-core/src/middleware/internalApi.js @@ -0,0 +1,14 @@ +const env = require("../environment") +const { Headers } = require("../constants") + +/** + * API Key only endpoint. + */ +module.exports = async (ctx, next) => { + const apiKey = ctx.request.headers[Headers.API_KEY] + if (apiKey !== env.INTERNAL_API_KEY) { + ctx.throw(403, "Unauthorized") + } + + return next() +} diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js index 6b8eb3a95c..eae8f150b0 100644 --- a/packages/backend-core/src/migrations/index.js +++ b/packages/backend-core/src/migrations/index.js @@ -1,20 +1,12 @@ +const { DEFAULT_TENANT_ID } = require("../constants") const { DocumentTypes } = require("../db/constants") -const { getGlobalDB, getTenantId } = require("../tenancy") +const { getAllApps } = require("../db/utils") +const environment = require("../environment") +const { doInTenant, getTenantIds, getGlobalDBName } = require("../tenancy") -exports.MIGRATION_DBS = { - GLOBAL_DB: "GLOBAL_DB", -} - -exports.MIGRATIONS = { - USER_EMAIL_VIEW_CASING: "user_email_view_casing", - QUOTAS_1: "quotas_1", -} - -const DB_LOOKUP = { - [exports.MIGRATION_DBS.GLOBAL_DB]: [ - exports.MIGRATIONS.USER_EMAIL_VIEW_CASING, - exports.MIGRATIONS.QUOTAS_1, - ], +exports.MIGRATION_TYPES = { + GLOBAL: "global", // run once, recorded in global db, global db is provided as an argument + APP: "app", // run per app, recorded in each app db, app db is provided as an argument } exports.getMigrationsDoc = async db => { @@ -28,40 +20,84 @@ exports.getMigrationsDoc = async db => { } } -exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => { - const tenantId = getTenantId() - try { - let db - if (migrationDb === exports.MIGRATION_DBS.GLOBAL_DB) { - db = getGlobalDB() - } else { - throw new Error(`Unrecognised migration db [${migrationDb}]`) - } +const runMigration = async (tenantId, CouchDB, migration, options = {}) => { + const migrationType = migration.type + const migrationName = migration.name - if (!DB_LOOKUP[migrationDb].includes(migrationName)) { - throw new Error( - `Unrecognised migration name [${migrationName}] for db [${migrationDb}]` - ) - } - - const doc = await exports.getMigrationsDoc(db) - // exit if the migration has been performed - if (doc[migrationName]) { - return - } - - console.log(`[Tenant: ${tenantId}] Performing migration: ${migrationName}`) - await migrateFn() - console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`) - - // mark as complete - doc[migrationName] = Date.now() - await db.put(doc) - } catch (err) { - console.error( - `[Tenant: ${tenantId}] Error performing migration: ${migrationName}: `, - err + // get the db to store the migration in + let dbNames + if (migrationType === exports.MIGRATION_TYPES.GLOBAL) { + dbNames = [getGlobalDBName(tenantId)] + } else if (migrationType === exports.MIGRATION_TYPES.APP) { + dbNames = await getAllApps(CouchDB, { all: true }) + } else { + throw new Error( + `[Tenant: ${tenantId}] Unrecognised migration type [${migrationType}]` ) - throw err + } + + // run the migration against each db + for (const dbName of dbNames) { + const db = new CouchDB(dbName) + try { + const doc = await exports.getMigrationsDoc(db) + + // exit if the migration has been performed already + if (doc[migrationName]) { + if ( + options.force && + options.force[migrationType] && + options.force[migrationType].includes(migrationName) + ) { + console.log( + `[Tenant: ${tenantId}] Forcing migration [${migrationName}]` + ) + } else { + // the migration has already been performed + return + } + } + + console.log( + `[Tenant: ${tenantId}] Performing migration: ${migrationName}` + ) + // run the migration with tenant context + await doInTenant(tenantId, () => migration.fn(db)) + console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`) + + // mark as complete + doc[migrationName] = Date.now() + await db.put(doc) + } catch (err) { + console.error( + `[Tenant: ${tenantId}] Error performing migration: ${migrationName} on db: ${db.name}: `, + err + ) + throw err + } } } + +exports.runMigrations = async (CouchDB, migrations, options = {}) => { + console.log("Running migrations") + let tenantIds + if (environment.MULTI_TENANCY) { + if (!options.tenantIds || !options.tenantIds.length) { + // run for all tenants + tenantIds = await getTenantIds() + } + } else { + // single tenancy + tenantIds = [DEFAULT_TENANT_ID] + } + + // for all tenants + for (const tenantId of tenantIds) { + // for all migrations + for (const migration of migrations) { + // run the migration + await runMigration(tenantId, CouchDB, migration, options) + } + } + console.log("Migrations complete") +} diff --git a/packages/backend-core/src/tenancy/tenancy.js b/packages/backend-core/src/tenancy/tenancy.js index 2cd05ea925..de597eac01 100644 --- a/packages/backend-core/src/tenancy/tenancy.js +++ b/packages/backend-core/src/tenancy/tenancy.js @@ -148,3 +148,15 @@ exports.isUserInAppTenant = (appId, user = null) => { const tenantId = exports.getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID return tenantId === userTenantId } + +exports.getTenantIds = async () => { + const db = getDB(PLATFORM_INFO_DB) + let tenants + try { + tenants = await db.get(TENANT_DOC) + } catch (err) { + // if theres an error the doc doesn't exist, no tenants exist + return [] + } + return (tenants && tenants.tenantIds) || [] +} diff --git a/packages/backend-core/src/utils.js b/packages/backend-core/src/utils.js index 85dd32946f..a11e5d42b7 100644 --- a/packages/backend-core/src/utils.js +++ b/packages/backend-core/src/utils.js @@ -20,9 +20,6 @@ const { hash } = require("./hashing") const userCache = require("./cache/user") const env = require("./environment") const { getUserSessions, invalidateSessions } = require("./security/sessions") -const { migrateIfRequired } = require("./migrations") -const { USER_EMAIL_VIEW_CASING } = require("./migrations").MIGRATIONS -const { GLOBAL_DB } = require("./migrations").MIGRATION_DBS const APP_PREFIX = DocumentTypes.APP + SEPARATOR @@ -149,11 +146,6 @@ exports.getGlobalUserByEmail = async email => { } const db = getGlobalDB() - await migrateIfRequired(GLOBAL_DB, USER_EMAIL_VIEW_CASING, async () => { - // re-create the view with latest changes - await createUserEmailView(db) - }) - try { let users = ( await db.query(`database/${ViewNames.USER_BY_EMAIL}`, { diff --git a/packages/server/src/api/controllers/migrations.js b/packages/server/src/api/controllers/migrations.js new file mode 100644 index 0000000000..08892d4034 --- /dev/null +++ b/packages/server/src/api/controllers/migrations.js @@ -0,0 +1,8 @@ +const { migrate } = require("../../migrations") + +exports.migrate = async ctx => { + const options = ctx.request.body + // don't await as can take a while, just return + migrate(options) + ctx.status = 200 +} diff --git a/packages/server/src/api/routes/index.js b/packages/server/src/api/routes/index.js index 44c5fbb7b0..8ded7104b0 100644 --- a/packages/server/src/api/routes/index.js +++ b/packages/server/src/api/routes/index.js @@ -24,6 +24,7 @@ const backupRoutes = require("./backup") const metadataRoutes = require("./metadata") const devRoutes = require("./dev") const cloudRoutes = require("./cloud") +const migrationRoutes = require("./migrations") exports.mainRoutes = [ authRoutes, @@ -53,6 +54,7 @@ exports.mainRoutes = [ // this could be breaking as koa may recognise other routes as this tableRoutes, rowRoutes, + migrationRoutes, ] exports.staticRoutes = staticRoutes diff --git a/packages/server/src/api/routes/migrations.js b/packages/server/src/api/routes/migrations.js new file mode 100644 index 0000000000..7cdcdc5935 --- /dev/null +++ b/packages/server/src/api/routes/migrations.js @@ -0,0 +1,8 @@ +const Router = require("@koa/router") +const migrationsController = require("../controllers/migrations") +const router = Router() +const { internalApi } = require("@budibase/backend-core/auth") + +router.post("/api/migrations/run", internalApi, migrationsController.migrate) + +module.exports = router diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 060169a777..0c0ef68ad9 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -1,7 +1,7 @@ // need to load environment first import { ExtendableContext } from "koa" -const env = require("./environment") +import * as env from "./environment" const CouchDB = require("./db") require("@budibase/backend-core").init(CouchDB) const Koa = require("koa") @@ -16,6 +16,7 @@ const Sentry = require("@sentry/node") const fileSystem = require("./utilities/fileSystem") const bullboard = require("./automations/bullboard") const redis = require("./utilities/redis") +import * as migrations from "./migrations" const app = new Koa() @@ -84,13 +85,25 @@ module.exports = server.listen(env.PORT || 0, async () => { await automations.init() }) -process.on("uncaughtException", err => { - console.error(err) +const shutdown = () => { server.close() server.destroy() +} + +process.on("uncaughtException", err => { + console.error(err) + shutdown() }) process.on("SIGTERM", () => { - server.close() - server.destroy() + shutdown() }) + +// run migrations on startup if not done via http +// not recommended in a clustered environment +if (!env.HTTP_MIGRATIONS) { + migrations.migrate().catch(err => { + console.error("Error performing migrations. Exiting.\n", err) + shutdown() + }) +} diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index 614f41a29f..99343937d9 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -43,6 +43,7 @@ module.exports = { REDIS_PASSWORD: process.env.REDIS_PASSWORD, INTERNAL_API_KEY: process.env.INTERNAL_API_KEY, MULTI_TENANCY: process.env.MULTI_TENANCY, + HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS, // environment NODE_ENV: process.env.NODE_ENV, JEST_WORKER_ID: process.env.JEST_WORKER_ID, diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index 4bafa75132..2cd0836113 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -5,7 +5,6 @@ const { isExternalTable, isRowId: isExternalRowId, } = require("../integrations/utils") -const migration = require("../migrations/usageQuotas") // currently only counting new writes and deletes const METHOD_MAP = { @@ -74,7 +73,6 @@ module.exports = async (ctx, next) => { usage = files.map(file => file.size).reduce((total, size) => total + size) } try { - await migration.run() await performRequest(ctx, next, property, usage) } catch (err) { ctx.throw(400, err) diff --git a/packages/server/src/migrations/functions/quotas1.ts b/packages/server/src/migrations/functions/quotas1.ts new file mode 100644 index 0000000000..500aa68f51 --- /dev/null +++ b/packages/server/src/migrations/functions/quotas1.ts @@ -0,0 +1,20 @@ +import { runQuotaMigration } from "./usageQuotas" +import * as syncApps from "./usageQuotas/syncApps" +import * as syncRows from "./usageQuotas/syncRows" + +/** + * Date: + * January 2022 + * + * Description: + * Synchronise the app and row quotas to the state of the db after it was + * discovered that the quota resets were still in place and the row quotas + * weren't being decremented correctly. + */ + +export const run = async () => { + await runQuotaMigration(async () => { + await syncApps.run() + await syncRows.run() + }) +} diff --git a/packages/server/src/migrations/functions/usageQuotas/index.ts b/packages/server/src/migrations/functions/usageQuotas/index.ts new file mode 100644 index 0000000000..16c4bf1d89 --- /dev/null +++ b/packages/server/src/migrations/functions/usageQuotas/index.ts @@ -0,0 +1,8 @@ +const { useQuotas } = require("../../../utilities/usageQuota") + +export const runQuotaMigration = async (migration: Function) => { + if (!useQuotas()) { + return + } + await migration() +} diff --git a/packages/server/src/migrations/usageQuotas/syncApps.js b/packages/server/src/migrations/functions/usageQuotas/syncApps.ts similarity index 80% rename from packages/server/src/migrations/usageQuotas/syncApps.js rename to packages/server/src/migrations/functions/usageQuotas/syncApps.ts index ee106129e6..0fba4f0f7f 100644 --- a/packages/server/src/migrations/usageQuotas/syncApps.js +++ b/packages/server/src/migrations/functions/usageQuotas/syncApps.ts @@ -1,9 +1,9 @@ const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") const { getAllApps } = require("@budibase/backend-core/db") -const CouchDB = require("../../db") -const { getUsageQuotaDoc } = require("../../utilities/usageQuota") +import CouchDB from "../../../db" +import { getUsageQuotaDoc } from "../../../utilities/usageQuota" -exports.run = async () => { +export const run = async () => { const db = getGlobalDB() // get app count const devApps = await getAllApps(CouchDB, { dev: true }) diff --git a/packages/server/src/migrations/usageQuotas/syncRows.js b/packages/server/src/migrations/functions/usageQuotas/syncRows.ts similarity index 67% rename from packages/server/src/migrations/usageQuotas/syncRows.js rename to packages/server/src/migrations/functions/usageQuotas/syncRows.ts index 7990f405de..58767d0c0a 100644 --- a/packages/server/src/migrations/usageQuotas/syncRows.js +++ b/packages/server/src/migrations/functions/usageQuotas/syncRows.ts @@ -1,14 +1,14 @@ const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy") const { getAllApps } = require("@budibase/backend-core/db") -const CouchDB = require("../../db") -const { getUsageQuotaDoc } = require("../../utilities/usageQuota") -const { getUniqueRows } = require("../../utilities/usageQuota/rows") +import CouchDB from "../../../db" +import { getUsageQuotaDoc } from "../../../utilities/usageQuota" +import { getUniqueRows } from "../../../utilities/usageQuota/rows" -exports.run = async () => { +export const run = async () => { const db = getGlobalDB() // get all rows in all apps const allApps = await getAllApps(CouchDB, { all: true }) - const appIds = allApps ? allApps.map(app => app.appId) : [] + const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : [] const rows = await getUniqueRows(appIds) const rowCount = rows ? rows.length : 0 diff --git a/packages/server/src/migrations/tests/usageQuotas/index.spec.js b/packages/server/src/migrations/functions/usageQuotas/tests/index.spec.js similarity index 76% rename from packages/server/src/migrations/tests/usageQuotas/index.spec.js rename to packages/server/src/migrations/functions/usageQuotas/tests/index.spec.js index 0c5b982909..586b796305 100644 --- a/packages/server/src/migrations/tests/usageQuotas/index.spec.js +++ b/packages/server/src/migrations/functions/usageQuotas/tests/index.spec.js @@ -1,5 +1,5 @@ -const env = require("../../../environment") -const TestConfig = require("../../../tests/utilities/TestConfiguration") +const env = require("../../../../environment") +const TestConfig = require("../../../../tests/utilities/TestConfiguration") const syncApps = jest.fn() const syncRows = jest.fn() @@ -7,7 +7,7 @@ const syncRows = jest.fn() jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) ) jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) ) -const migrations = require("../../usageQuotas") +const migrations = require("..") describe("run", () => { let config = new TestConfig(false) diff --git a/packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js b/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js similarity index 94% rename from packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js rename to packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js index 160319a31b..8772900890 100644 --- a/packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js +++ b/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js @@ -1,7 +1,7 @@ const { getGlobalDB } = require("@budibase/backend-core/tenancy") const TestConfig = require("../../../tests/utilities/TestConfiguration") const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota") -const syncApps = require("../../usageQuotas/syncApps") +const syncApps = require("../syncApps") const env = require("../../../environment") describe("syncApps", () => { diff --git a/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js similarity index 95% rename from packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js rename to packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js index a09bea60bd..e83238fa18 100644 --- a/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js +++ b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js @@ -1,7 +1,7 @@ const { getGlobalDB } = require("@budibase/backend-core/tenancy") const TestConfig = require("../../../tests/utilities/TestConfiguration") const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota") -const syncRows = require("../../usageQuotas/syncRows") +const syncRows = require("../syncRows") const env = require("../../../environment") describe("syncRows", () => { diff --git a/packages/server/src/migrations/functions/userEmailViewCasing.ts b/packages/server/src/migrations/functions/userEmailViewCasing.ts new file mode 100644 index 0000000000..16f55655ab --- /dev/null +++ b/packages/server/src/migrations/functions/userEmailViewCasing.ts @@ -0,0 +1,13 @@ +const { createUserEmailView } = require("@budibase/backend-core/db") + +/** + * Date: + * October 2021 + * + * Description: + * Recreate the user email view to include latest changes i.e. lower casing the email address + */ + +export const run = async (db: any) => { + await createUserEmailView(db) +} diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts new file mode 100644 index 0000000000..def8aac5f7 --- /dev/null +++ b/packages/server/src/migrations/index.ts @@ -0,0 +1,48 @@ +import CouchDB from "../db" +const { + MIGRATION_TYPES, + runMigrations, +} = require("@budibase/backend-core/migrations") + +// migration functions +import * as userEmailViewCasing from "./functions/userEmailViewCasing" +import * as quota1 from "./functions/quotas1" + +export interface Migration { + type: string + name: string + fn: Function +} + +/** + * e.g. + * { + * tenantIds: ['bb'], + * force: { + * global: ['quota_1'] + * } + * } + */ +export interface MigrationOptions { + tenantIds?: string[] + forced?: { + [type: string]: string[] + } +} + +const MIGRATIONS: Migration[] = [ + { + type: MIGRATION_TYPES.GLOBAL, + name: "user_email_view_casing", + fn: userEmailViewCasing.run, + }, + { + type: MIGRATION_TYPES.GLOBAL, + name: "quotas_1", + fn: quota1.run, + }, +] + +export const migrate = async (options?: MigrationOptions) => { + await runMigrations(CouchDB, MIGRATIONS, options) +} diff --git a/packages/server/src/migrations/usageQuotas/index.js b/packages/server/src/migrations/usageQuotas/index.js deleted file mode 100644 index 39744093c2..0000000000 --- a/packages/server/src/migrations/usageQuotas/index.js +++ /dev/null @@ -1,24 +0,0 @@ -const { - MIGRATIONS, - MIGRATION_DBS, - migrateIfRequired, -} = require("@budibase/backend-core/migrations") -const { useQuotas } = require("../../utilities/usageQuota") -const syncApps = require("./syncApps") -const syncRows = require("./syncRows") - -exports.run = async () => { - if (!useQuotas()) { - return - } - - // Jan 2022 - await migrateIfRequired( - MIGRATION_DBS.GLOBAL_DB, - MIGRATIONS.QUOTAS_1, - async () => { - await syncApps.run() - await syncRows.run() - } - ) -}