diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js index eae8f150b0..0e66232669 100644 --- a/packages/backend-core/src/migrations/index.js +++ b/packages/backend-core/src/migrations/index.js @@ -2,7 +2,12 @@ const { DEFAULT_TENANT_ID } = require("../constants") const { DocumentTypes } = require("../db/constants") const { getAllApps } = require("../db/utils") const environment = require("../environment") -const { doInTenant, getTenantIds, getGlobalDBName } = require("../tenancy") +const { + doInTenant, + getTenantIds, + getGlobalDBName, + getTenantId, +} = require("../tenancy") exports.MIGRATION_TYPES = { GLOBAL: "global", // run once, recorded in global db, global db is provided as an argument @@ -20,16 +25,18 @@ exports.getMigrationsDoc = async db => { } } -const runMigration = async (tenantId, CouchDB, migration, options = {}) => { +const runMigration = async (CouchDB, migration, options = {}) => { + const tenantId = getTenantId() const migrationType = migration.type const migrationName = migration.name // get the db to store the migration in let dbNames if (migrationType === exports.MIGRATION_TYPES.GLOBAL) { - dbNames = [getGlobalDBName(tenantId)] + dbNames = [getGlobalDBName()] } else if (migrationType === exports.MIGRATION_TYPES.APP) { - dbNames = await getAllApps(CouchDB, { all: true }) + const apps = await getAllApps(CouchDB, migration.opts) + dbNames = apps.map(app => app.appId) } else { throw new Error( `[Tenant: ${tenantId}] Unrecognised migration type [${migrationType}]` @@ -50,7 +57,7 @@ const runMigration = async (tenantId, CouchDB, migration, options = {}) => { options.force[migrationType].includes(migrationName) ) { console.log( - `[Tenant: ${tenantId}] Forcing migration [${migrationName}]` + `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing` ) } else { // the migration has already been performed @@ -59,18 +66,20 @@ const runMigration = async (tenantId, CouchDB, migration, options = {}) => { } console.log( - `[Tenant: ${tenantId}] Performing migration: ${migrationName}` + `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running` ) // run the migration with tenant context - await doInTenant(tenantId, () => migration.fn(db)) - console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`) + await migration.fn(db) + console.log( + `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete` + ) // mark as complete doc[migrationName] = Date.now() await db.put(doc) } catch (err) { console.error( - `[Tenant: ${tenantId}] Error performing migration: ${migrationName} on db: ${db.name}: `, + `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `, err ) throw err @@ -96,7 +105,9 @@ exports.runMigrations = async (CouchDB, migrations, options = {}) => { // for all migrations for (const migration of migrations) { // run the migration - await runMigration(tenantId, CouchDB, migration, options) + await doInTenant(tenantId, () => + runMigration(CouchDB, migration, options) + ) } } console.log("Migrations complete") diff --git a/packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap b/packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap index e9a18eadde..222c3b1228 100644 --- a/packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap +++ b/packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap @@ -3,7 +3,7 @@ exports[`migrations should match snapshot 1`] = ` Object { "_id": "migrations", - "_rev": "1-af6c272fe081efafecd2ea49a8fcbb40", - "user_email_view_casing": 1487076708000, + "_rev": "1-6277abc4e3db950221768e5a2618a059", + "test": 1487076708000, } `; diff --git a/packages/backend-core/src/migrations/tests/index.spec.js b/packages/backend-core/src/migrations/tests/index.spec.js index 0ed16fc184..12a2e54cb3 100644 --- a/packages/backend-core/src/migrations/tests/index.spec.js +++ b/packages/backend-core/src/migrations/tests/index.spec.js @@ -1,7 +1,7 @@ require("../../tests/utilities/dbConfig") -const { migrateIfRequired, MIGRATION_DBS, MIGRATIONS, getMigrationsDoc } = require("../index") -const database = require("../../db") +const { runMigrations, getMigrationsDoc } = require("../index") +const CouchDB = require("../../db").getCouch() const { StaticDatabases, } = require("../../db/utils") @@ -13,8 +13,14 @@ describe("migrations", () => { const migrationFunction = jest.fn() + const MIGRATIONS = [{ + type: "global", + name: "test", + fn: migrationFunction + }] + beforeEach(() => { - db = database.getDB(StaticDatabases.GLOBAL.name) + db = new CouchDB(StaticDatabases.GLOBAL.name) }) afterEach(async () => { @@ -22,39 +28,29 @@ describe("migrations", () => { await db.destroy() }) - const validMigration = () => { - return migrateIfRequired(MIGRATION_DBS.GLOBAL_DB, MIGRATIONS.USER_EMAIL_VIEW_CASING, migrationFunction) + const migrate = () => { + return runMigrations(CouchDB, MIGRATIONS) } it("should run a new migration", async () => { - await validMigration() + await migrate() expect(migrationFunction).toHaveBeenCalled() + const doc = await getMigrationsDoc(db) + expect(doc.test).toBeDefined() }) it("should match snapshot", async () => { - await validMigration() + await migrate() const doc = await getMigrationsDoc(db) expect(doc).toMatchSnapshot() }) it("should skip a previously run migration", async () => { - await validMigration() - await validMigration() + await migrate() + const previousMigrationTime = await getMigrationsDoc(db).test + await migrate() + const currentMigrationTime = await getMigrationsDoc(db).test expect(migrationFunction).toHaveBeenCalledTimes(1) + expect(currentMigrationTime).toBe(previousMigrationTime) }) - - it("should reject an unknown migration name", async () => { - expect(async () => { - await migrateIfRequired(MIGRATION_DBS.GLOBAL_DB, "bogus_name", migrationFunction) - }).rejects.toThrow() - expect(migrationFunction).not.toHaveBeenCalled() - }) - - it("should reject an unknown database name", async () => { - expect(async () => { - await migrateIfRequired("bogus_db", MIGRATIONS.USER_EMAIL_VIEW_CASING, migrationFunction) - }).rejects.toThrow() - expect(migrationFunction).not.toHaveBeenCalled() - }) - }) \ No newline at end of file diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index 7aaeebc025..0f88129794 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -75,7 +75,7 @@ function getUserRoleId(ctx) { : ctx.user.role._id } -async function getAppUrl(ctx) { +exports.getAppUrl = ctx => { // construct the url let url if (ctx.request.body.url) { @@ -218,7 +218,7 @@ exports.create = async ctx => { const apps = await getAllApps(CouchDB, { dev: true }) const name = ctx.request.body.name checkAppName(ctx, apps, name) - const url = await getAppUrl(ctx) + const url = exports.getAppUrl(ctx) checkAppUrl(ctx, apps, url) const { useTemplate, templateKey, templateString } = ctx.request.body @@ -281,7 +281,7 @@ exports.update = async ctx => { // validation const name = ctx.request.body.name checkAppName(ctx, apps, name, ctx.params.appId) - const url = await getAppUrl(ctx) + const url = exports.getAppUrl(ctx) checkAppUrl(ctx, apps, url, ctx.params.appId) const appPackageUpdates = { name, url } diff --git a/packages/server/src/api/controllers/migrations.js b/packages/server/src/api/controllers/migrations.js index 08892d4034..6a890349c3 100644 --- a/packages/server/src/api/controllers/migrations.js +++ b/packages/server/src/api/controllers/migrations.js @@ -1,4 +1,4 @@ -const { migrate } = require("../../migrations") +const { migrate, MIGRATIONS } = require("../../migrations") exports.migrate = async ctx => { const options = ctx.request.body @@ -6,3 +6,8 @@ exports.migrate = async ctx => { migrate(options) ctx.status = 200 } + +exports.fetchDefinitions = async ctx => { + ctx.body = MIGRATIONS + ctx.status = 200 +} diff --git a/packages/server/src/api/routes/migrations.js b/packages/server/src/api/routes/migrations.js index 7cdcdc5935..01e573edb3 100644 --- a/packages/server/src/api/routes/migrations.js +++ b/packages/server/src/api/routes/migrations.js @@ -3,6 +3,12 @@ const migrationsController = require("../controllers/migrations") const router = Router() const { internalApi } = require("@budibase/backend-core/auth") -router.post("/api/migrations/run", internalApi, migrationsController.migrate) +router + .post("/api/migrations/run", internalApi, migrationsController.migrate) + .get( + "/api/migrations/definitions", + internalApi, + migrationsController.fetchDefinitions + ) module.exports = router diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index 3199ce3bde..ea40dfb609 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -43,8 +43,8 @@ const coreFields = { enum: Object.values(BodyTypes), }, pagination: { - type: DatasourceFieldTypes.OBJECT - } + type: DatasourceFieldTypes.OBJECT, + }, } module RestModule { @@ -178,12 +178,17 @@ module RestModule { headers, }, pagination: { - cursor: nextCursor - } + cursor: nextCursor, + }, } } - getUrl(path: string, queryString: string, pagination: PaginationConfig | null, paginationValues: PaginationValues | null): string { + getUrl( + path: string, + queryString: string, + pagination: PaginationConfig | null, + paginationValues: PaginationValues | null + ): string { // Add pagination params to query string if required if (pagination?.location === "query" && paginationValues) { const { pageParam, sizeParam } = pagination @@ -217,14 +222,22 @@ module RestModule { return complete } - addBody(bodyType: string, body: string | any, input: any, pagination: PaginationConfig | null, paginationValues: PaginationValues | null) { + addBody( + bodyType: string, + body: string | any, + input: any, + pagination: PaginationConfig | null, + paginationValues: PaginationValues | null + ) { if (!input.headers) { input.headers = {} } if (bodyType === BodyTypes.NONE) { return input } - let error, object: any = {}, string = "" + let error, + object: any = {}, + string = "" try { if (body) { string = typeof body !== "string" ? JSON.stringify(body) : body @@ -333,7 +346,7 @@ module RestModule { requestBody, authConfigId, pagination, - paginationValues + paginationValues, } = query const authHeaders = this.getAuthHeaders(authConfigId) @@ -352,7 +365,13 @@ module RestModule { } let input: any = { method, headers: this.headers } - input = this.addBody(bodyType, requestBody, input, pagination, paginationValues) + input = this.addBody( + bodyType, + requestBody, + input, + pagination, + paginationValues + ) this.startTimeMs = performance.now() const url = this.getUrl(path, queryString, pagination, paginationValues) diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts index 25b439fd58..273f221575 100644 --- a/packages/server/src/integrations/s3.ts +++ b/packages/server/src/integrations/s3.ts @@ -38,7 +38,7 @@ module S3Module { signatureVersion: { type: "string", required: false, - default: "v4" + default: "v4", }, }, query: { diff --git a/packages/server/src/migrations/functions/appUrls.ts b/packages/server/src/migrations/functions/appUrls.ts new file mode 100644 index 0000000000..8852c27822 --- /dev/null +++ b/packages/server/src/migrations/functions/appUrls.ts @@ -0,0 +1,25 @@ +const { DocumentTypes } = require("@budibase/backend-core/db") +import { getAppUrl } from "../../api/controllers/application" + +/** + * Date: + * January 2022 + * + * Description: + * Add the url to the app metadata if it doesn't exist + */ +export const run = async (appDb: any) => { + const metadata = await appDb.get(DocumentTypes.APP_METADATA) + if (!metadata.url) { + const context = { + request: { + body: { + name: metadata.name, + }, + }, + } + metadata.url = getAppUrl(context) + console.log(`Adding url to app: ${metadata.url}`) + } + await appDb.put(metadata) +} diff --git a/packages/server/src/migrations/functions/tests/appUrls.spec.js b/packages/server/src/migrations/functions/tests/appUrls.spec.js new file mode 100644 index 0000000000..d3f080dfd4 --- /dev/null +++ b/packages/server/src/migrations/functions/tests/appUrls.spec.js @@ -0,0 +1,29 @@ +const { DocumentTypes } = require("@budibase/backend-core/db") +const env = require("../../../environment") +const TestConfig = require("../../../tests/utilities/TestConfiguration") + +const migration = require("../appUrls") + +describe("run", () => { + let config = new TestConfig(false) + const CouchDB = config.getCouch() + + beforeEach(async () => { + await config.init() + }) + + afterAll(config.end) + + it("runs successfully", async () => { + const app = await config.createApp("testApp") + const appDb = new CouchDB(app.appId) + let metadata = await appDb.get(DocumentTypes.APP_METADATA) + delete metadata.url + await appDb.put(metadata) + + await migration.run(appDb) + + metadata = await appDb.get(DocumentTypes.APP_METADATA) + expect(metadata.url).toEqual("/testapp") + }) +}) diff --git a/packages/server/src/migrations/functions/tests/quotas1.spec.js b/packages/server/src/migrations/functions/tests/quotas1.spec.js new file mode 100644 index 0000000000..df8703e9a0 --- /dev/null +++ b/packages/server/src/migrations/functions/tests/quotas1.spec.js @@ -0,0 +1,27 @@ +const env = require("../../../environment") +const TestConfig = require("../../../tests/utilities/TestConfiguration") + +const syncApps = jest.fn() +const syncRows = jest.fn() + +jest.mock("../usageQuotas/syncApps", () => ({ run: syncApps }) ) +jest.mock("../usageQuotas/syncRows", () => ({ run: syncRows }) ) + +const migration = require("../quotas1") + +describe("run", () => { + let config = new TestConfig(false) + + beforeEach(async () => { + await config.init() + env._set("USE_QUOTAS", 1) + }) + + afterAll(config.end) + + it("runs ", async () => { + await migration.run() + expect(syncApps).toHaveBeenCalledTimes(1) + expect(syncRows).toHaveBeenCalledTimes(1) + }) +}) diff --git a/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js b/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js new file mode 100644 index 0000000000..c0d7823cbf --- /dev/null +++ b/packages/server/src/migrations/functions/tests/userEmailViewCasing.spec.js @@ -0,0 +1,25 @@ +const TestConfig = require("../../../tests/utilities/TestConfiguration") +const { getGlobalDB } = require("@budibase/backend-core/tenancy") + +// mock email view creation +const coreDb = require("@budibase/backend-core/db") +const createUserEmailView = jest.fn() +coreDb.createUserEmailView = createUserEmailView + +const migration = require("../userEmailViewCasing") + +describe("run", () => { + let config = new TestConfig(false) + const globalDb = getGlobalDB() + + beforeEach(async () => { + await config.init() + }) + + afterAll(config.end) + + it("runs successfully", async () => { + await migration.run(globalDb) + expect(createUserEmailView).toHaveBeenCalledTimes(1) + }) +}) diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/index.spec.js b/packages/server/src/migrations/functions/usageQuotas/tests/index.spec.js deleted file mode 100644 index 586b796305..0000000000 --- a/packages/server/src/migrations/functions/usageQuotas/tests/index.spec.js +++ /dev/null @@ -1,27 +0,0 @@ -const env = require("../../../../environment") -const TestConfig = require("../../../../tests/utilities/TestConfiguration") - -const syncApps = jest.fn() -const syncRows = jest.fn() - -jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) ) -jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) ) - -const migrations = require("..") - -describe("run", () => { - let config = new TestConfig(false) - - beforeEach(async () => { - await config.init() - env._set("USE_QUOTAS", 1) - }) - - afterAll(config.end) - - it("runs the required migrations", async () => { - await migrations.run() - expect(syncApps).toHaveBeenCalledTimes(1) - expect(syncRows).toHaveBeenCalledTimes(1) - }) -}) diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js b/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js index 8772900890..7c74cbfe9a 100644 --- a/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js +++ b/packages/server/src/migrations/functions/usageQuotas/tests/syncApps.spec.js @@ -1,8 +1,8 @@ const { getGlobalDB } = require("@budibase/backend-core/tenancy") -const TestConfig = require("../../../tests/utilities/TestConfiguration") -const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota") +const TestConfig = require("../../../../tests/utilities/TestConfiguration") +const { getUsageQuotaDoc, update, Properties } = require("../../../../utilities/usageQuota") const syncApps = require("../syncApps") -const env = require("../../../environment") +const env = require("../../../../environment") describe("syncApps", () => { let config = new TestConfig(false) diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js index 9bccf7fe9d..034d0eb067 100644 --- a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js +++ b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.js @@ -1,8 +1,8 @@ const { getGlobalDB } = require("@budibase/backend-core/tenancy") -const TestConfig = require("../../../tests/utilities/TestConfiguration") -const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota") +const TestConfig = require("../../../../tests/utilities/TestConfiguration") +const { getUsageQuotaDoc, update, Properties } = require("../../../../utilities/usageQuota") const syncRows = require("../syncRows") -const env = require("../../../environment") +const env = require("../../../../environment") describe("syncRows", () => { let config = new TestConfig(false) diff --git a/packages/server/src/migrations/index.ts b/packages/server/src/migrations/index.ts index def8aac5f7..966041e0c9 100644 --- a/packages/server/src/migrations/index.ts +++ b/packages/server/src/migrations/index.ts @@ -7,10 +7,12 @@ const { // migration functions import * as userEmailViewCasing from "./functions/userEmailViewCasing" import * as quota1 from "./functions/quotas1" +import * as appUrls from "./functions/appUrls" export interface Migration { type: string name: string + opts?: object fn: Function } @@ -30,7 +32,7 @@ export interface MigrationOptions { } } -const MIGRATIONS: Migration[] = [ +export const MIGRATIONS: Migration[] = [ { type: MIGRATION_TYPES.GLOBAL, name: "user_email_view_casing", @@ -41,6 +43,12 @@ const MIGRATIONS: Migration[] = [ name: "quotas_1", fn: quota1.run, }, + { + type: MIGRATION_TYPES.APP, + name: "app_urls", + opts: { all: true }, + fn: appUrls.run, + }, ] export const migrate = async (options?: MigrationOptions) => { diff --git a/packages/server/src/tests/utilities/TestConfiguration.js b/packages/server/src/tests/utilities/TestConfiguration.js index 7aefe4fb78..75b65f2dd3 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.js +++ b/packages/server/src/tests/utilities/TestConfiguration.js @@ -49,6 +49,10 @@ class TestConfiguration { return this.appId } + getCouch() { + return CouchDB + } + async _req(config, params, controlFunc) { const request = {} // fake cookies, we don't need them