From 747db3daa2fc9f7504d46742522833befad583d4 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Tue, 18 Jan 2022 14:43:24 +0000 Subject: [PATCH] Refactor to allow for future quota migrations --- packages/backend-core/src/migrations/index.js | 4 +- packages/server/src/middleware/usageQuota.js | 15 ++++-- .../sync_app_and_reset_rows_quotas.js | 48 ------------------- .../tests/usageQuotas/index.spec.js | 27 +++++++++++ .../syncApps.spec.js} | 20 ++++---- .../tests/usageQuotas/syncRows.spec.js | 43 +++++++++++++++++ .../src/migrations/usageQuotas/index.js | 24 ++++++++++ .../src/migrations/usageQuotas/syncApps.js | 16 +++++++ .../src/migrations/usageQuotas/syncRows.js | 18 +++++++ 9 files changed, 151 insertions(+), 64 deletions(-) delete mode 100644 packages/server/src/migrations/sync_app_and_reset_rows_quotas.js create mode 100644 packages/server/src/migrations/tests/usageQuotas/index.spec.js rename packages/server/src/migrations/tests/{sync_app_and_reset_rows_quotas.spec.js => usageQuotas/syncApps.spec.js} (59%) create mode 100644 packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js create mode 100644 packages/server/src/migrations/usageQuotas/index.js create mode 100644 packages/server/src/migrations/usageQuotas/syncApps.js create mode 100644 packages/server/src/migrations/usageQuotas/syncRows.js diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js index 30e04e198b..4708776ec3 100644 --- a/packages/backend-core/src/migrations/index.js +++ b/packages/backend-core/src/migrations/index.js @@ -7,13 +7,13 @@ exports.MIGRATION_DBS = { exports.MIGRATIONS = { USER_EMAIL_VIEW_CASING: "user_email_view_casing", - SYNC_APP_AND_RESET_ROWS_QUOTAS: "sync_app_and_reset_rows_quotas", + QUOTAS_1: "quotas_1", } const DB_LOOKUP = { [exports.MIGRATION_DBS.GLOBAL_DB]: [ exports.MIGRATIONS.USER_EMAIL_VIEW_CASING, - exports.MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS, + exports.MIGRATIONS.QUOTAS_1, ], } diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index c42442affe..8e3daa6268 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -5,7 +5,7 @@ const { isExternalTable, isRowId: isExternalRowId, } = require("../integrations/utils") -const quotaMigration = require("../migrations/sync_app_and_reset_rows_quotas") +const migration = require("../migrations/usageQuotas") // currently only counting new writes and deletes const METHOD_MAP = { @@ -74,7 +74,7 @@ module.exports = async (ctx, next) => { usage = files.map(file => file.size).reduce((total, size) => total + size) } try { - await quotaMigration.runIfRequired() + await migration.run() await performRequest(ctx, next, property, usage) } catch (err) { ctx.throw(400, err) @@ -142,6 +142,13 @@ const appPostDelete = async (ctx, usageContext) => { } } +// const appPostCreate = async (ctx, usageContext) => { +// if (ctx.request) { +// const rowCount = await getUniqueRows([ctx.appId]).length +// await usageQuota.update(usageQuota.Properties.ROW, -rowCount) +// } +// } + const PRE_DELETE = { [usageQuota.Properties.APPS]: appPreDelete, } @@ -152,4 +159,6 @@ const POST_DELETE = { const PRE_CREATE = {} -const POST_CREATE = {} +const POST_CREATE = { + // [usageQuota.Properties.APPS]: appPostCreate, +} diff --git a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js deleted file mode 100644 index 4ab2e413f3..0000000000 --- a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js +++ /dev/null @@ -1,48 +0,0 @@ -const { - MIGRATIONS, - MIGRATION_DBS, - migrateIfRequired, -} = require("@budibase/backend-core/migrations") -const { getGlobalDB } = require("@budibase/backend-core/tenancy") -const { getAllApps } = require("@budibase/backend-core/db") -const CouchDB = require("../db") -const { getUsageQuotaDoc, useQuotas } = require("../utilities/usageQuota") -const { getUniqueRows } = require("../utilities/usageQuota/rows") - -const syncRowsQuota = async db => { - // get all rows in all apps - const allApps = await getAllApps(CouchDB, { all: true }) - const appIds = allApps ? allApps.map(app => app.appId) : [] - const rows = await getUniqueRows(appIds) - - // sync row count - const usageDoc = await getUsageQuotaDoc(db) - usageDoc.usageQuota.rows = rows.length - await db.put(usageDoc) -} - -const syncAppsQuota = async db => { - // get app count - const devApps = await getAllApps(CouchDB, { dev: true }) - const appCount = devApps ? devApps.length : 0 - - // sync app count - const usageDoc = await getUsageQuotaDoc(db) - usageDoc.usageQuota.apps = appCount - await db.put(usageDoc) -} - -exports.runIfRequired = async () => { - await migrateIfRequired( - MIGRATION_DBS.GLOBAL_DB, - MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS, - async () => { - if (!useQuotas()) { - return - } - const db = getGlobalDB() - await syncAppsQuota(db) - await syncRowsQuota(db) - } - ) -} diff --git a/packages/server/src/migrations/tests/usageQuotas/index.spec.js b/packages/server/src/migrations/tests/usageQuotas/index.spec.js new file mode 100644 index 0000000000..0c5b982909 --- /dev/null +++ b/packages/server/src/migrations/tests/usageQuotas/index.spec.js @@ -0,0 +1,27 @@ +const env = require("../../../environment") +const TestConfig = require("../../../tests/utilities/TestConfiguration") + +const syncApps = jest.fn() +const syncRows = jest.fn() + +jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) ) +jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) ) + +const migrations = require("../../usageQuotas") + +describe("run", () => { + let config = new TestConfig(false) + + beforeEach(async () => { + await config.init() + env._set("USE_QUOTAS", 1) + }) + + afterAll(config.end) + + it("runs the required migrations", async () => { + await migrations.run() + expect(syncApps).toHaveBeenCalledTimes(1) + expect(syncRows).toHaveBeenCalledTimes(1) + }) +}) diff --git a/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js b/packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js similarity index 59% rename from packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js rename to packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js index cd533e3f06..160319a31b 100644 --- a/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js +++ b/packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js @@ -1,10 +1,10 @@ const { getGlobalDB } = require("@budibase/backend-core/tenancy") -const TestConfig = require("../../tests/utilities/TestConfiguration") -const { getUsageQuotaDoc, update, Properties } = require("../../utilities/usageQuota") -const { runIfRequired } = require("../sync_app_and_reset_rows_quotas") -const env = require("../../environment") +const TestConfig = require("../../../tests/utilities/TestConfiguration") +const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota") +const syncApps = require("../../usageQuotas/syncApps") +const env = require("../../../environment") -describe("Sync App And Reset Rows Quotas Migration", () => { +describe("syncApps", () => { let config = new TestConfig(false) beforeEach(async () => { @@ -12,28 +12,26 @@ describe("Sync App And Reset Rows Quotas Migration", () => { env._set("USE_QUOTAS", 1) }) - afterAll(config.end) + afterAll(config.end) - it("migrates successfully", async () => { + it("runs successfully", async () => { // create the usage quota doc and mock usages const db = getGlobalDB() await getUsageQuotaDoc(db) await update(Properties.APPS, 3) - await update(Properties.ROW, 300) let usageDoc = await getUsageQuotaDoc(db) expect(usageDoc.usageQuota.apps).toEqual(3) - expect(usageDoc.usageQuota.rows).toEqual(300) // create an extra app to test the migration await config.createApp("quota-test") // migrate - await runIfRequired() + await syncApps.run() // assert the migration worked usageDoc = await getUsageQuotaDoc(db) expect(usageDoc.usageQuota.apps).toEqual(2) - expect(usageDoc.usageQuota.rows).toEqual(0) }) }) + diff --git a/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js b/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js new file mode 100644 index 0000000000..a09bea60bd --- /dev/null +++ b/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js @@ -0,0 +1,43 @@ +const { getGlobalDB } = require("@budibase/backend-core/tenancy") +const TestConfig = require("../../../tests/utilities/TestConfiguration") +const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota") +const syncRows = require("../../usageQuotas/syncRows") +const env = require("../../../environment") + +describe("syncRows", () => { + let config = new TestConfig(false) + + beforeEach(async () => { + await config.init() + env._set("USE_QUOTAS", 1) + }) + + afterAll(config.end) + + it("runs successfully", async () => { + // create the usage quota doc and mock usages + const db = getGlobalDB() + await getUsageQuotaDoc(db) + await update(Properties.ROW, 300) + + let usageDoc = await getUsageQuotaDoc(db) + expect(usageDoc.usageQuota.rows).toEqual(300) + + // app 1 + await config.createTable() + await config.createRow() + // app 2 + await config.createApp() + await config.createTable() + await config.createRow() + await config.createRow() + + // migrate + await syncRows.run() + + // assert the migration worked + usageDoc = await getUsageQuotaDoc(db) + expect(usageDoc.usageQuota.rows).toEqual(3) + }) +}) + diff --git a/packages/server/src/migrations/usageQuotas/index.js b/packages/server/src/migrations/usageQuotas/index.js new file mode 100644 index 0000000000..39744093c2 --- /dev/null +++ b/packages/server/src/migrations/usageQuotas/index.js @@ -0,0 +1,24 @@ +const { + MIGRATIONS, + MIGRATION_DBS, + migrateIfRequired, +} = require("@budibase/backend-core/migrations") +const { useQuotas } = require("../../utilities/usageQuota") +const syncApps = require("./syncApps") +const syncRows = require("./syncRows") + +exports.run = async () => { + if (!useQuotas()) { + return + } + + // Jan 2022 + await migrateIfRequired( + MIGRATION_DBS.GLOBAL_DB, + MIGRATIONS.QUOTAS_1, + async () => { + await syncApps.run() + await syncRows.run() + } + ) +} diff --git a/packages/server/src/migrations/usageQuotas/syncApps.js b/packages/server/src/migrations/usageQuotas/syncApps.js new file mode 100644 index 0000000000..c285bafe4e --- /dev/null +++ b/packages/server/src/migrations/usageQuotas/syncApps.js @@ -0,0 +1,16 @@ +const { getGlobalDB } = require("@budibase/backend-core/tenancy") +const { getAllApps } = require("@budibase/backend-core/db") +const CouchDB = require("../../db") +const { getUsageQuotaDoc } = require("../../utilities/usageQuota") + +exports.run = async () => { + const db = getGlobalDB() + // get app count + const devApps = await getAllApps(CouchDB, { dev: true }) + const appCount = devApps ? devApps.length : 0 + + // sync app count + const usageDoc = await getUsageQuotaDoc(db) + usageDoc.usageQuota.apps = appCount + await db.put(usageDoc) +} diff --git a/packages/server/src/migrations/usageQuotas/syncRows.js b/packages/server/src/migrations/usageQuotas/syncRows.js new file mode 100644 index 0000000000..73f6f1477d --- /dev/null +++ b/packages/server/src/migrations/usageQuotas/syncRows.js @@ -0,0 +1,18 @@ +const { getGlobalDB } = require("@budibase/backend-core/tenancy") +const { getAllApps } = require("@budibase/backend-core/db") +const CouchDB = require("../../db") +const { getUsageQuotaDoc } = require("../../utilities/usageQuota") +const { getUniqueRows } = require("../../utilities/usageQuota/rows") + +exports.run = async () => { + const db = getGlobalDB() + // get all rows in all apps + const allApps = await getAllApps(CouchDB, { all: true }) + const appIds = allApps ? allApps.map(app => app.appId) : [] + const rows = await getUniqueRows(appIds) + + // sync row count + const usageDoc = await getUsageQuotaDoc(db) + usageDoc.usageQuota.rows = rows.length + await db.put(usageDoc) +}