From 726315afde528d18a068f67ecb7048367045a11d Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Tue, 11 Jan 2022 17:49:42 +0000 Subject: [PATCH] Sync app quota to correct number, reset and disable rows quota --- packages/auth/src/db/constants.js | 2 +- packages/auth/src/db/utils.js | 2 +- packages/auth/src/index.js | 1 + packages/auth/src/migrations/index.js | 2 + packages/server/src/middleware/usageQuota.js | 10 ++--- .../sync_app_and_reset_rows_quotas.js | 27 +++++++++++++ .../sync_app_and_reset_rows_quotas.spec.js | 39 +++++++++++++++++++ packages/server/src/utilities/usageQuota.js | 32 +++++---------- .../server/src/utilities/usageQuoteReset.js | 18 +++++++++ .../src/api/controllers/global/users.js | 4 +- 10 files changed, 105 insertions(+), 32 deletions(-) create mode 100644 packages/server/src/migrations/sync_app_and_reset_rows_quotas.js create mode 100644 packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js create mode 100644 packages/server/src/utilities/usageQuoteReset.js diff --git a/packages/auth/src/db/constants.js b/packages/auth/src/db/constants.js index ecdaae5bad..2affb09c7c 100644 --- a/packages/auth/src/db/constants.js +++ b/packages/auth/src/db/constants.js @@ -21,6 +21,7 @@ exports.StaticDatabases = { name: "global-db", docs: { apiKeys: "apikeys", + usageQuota: "usage_quota", }, }, // contains information about tenancy and so on @@ -28,7 +29,6 @@ exports.StaticDatabases = { name: "global-info", docs: { tenants: "tenants", - usageQuota: "usage_quota", }, }, } diff --git a/packages/auth/src/db/utils.js b/packages/auth/src/db/utils.js index 5830de4721..2bc5462646 100644 --- a/packages/auth/src/db/utils.js +++ b/packages/auth/src/db/utils.js @@ -450,7 +450,7 @@ async function getScopedConfig(db, params) { function generateNewUsageQuotaDoc() { return { - _id: StaticDatabases.PLATFORM_INFO.docs.usageQuota, + _id: StaticDatabases.GLOBAL.docs.usageQuota, quotaReset: Date.now() + 2592000000, usageQuota: { automationRuns: 0, diff --git a/packages/auth/src/index.js b/packages/auth/src/index.js index 4aa2c8ab96..bd8d7c2118 100644 --- a/packages/auth/src/index.js +++ b/packages/auth/src/index.js @@ -68,4 +68,5 @@ module.exports = { }, StaticDatabases, constants: require("./constants"), + migrations: require("./migrations"), } diff --git a/packages/auth/src/migrations/index.js b/packages/auth/src/migrations/index.js index 7492e94511..30e04e198b 100644 --- a/packages/auth/src/migrations/index.js +++ b/packages/auth/src/migrations/index.js @@ -7,11 +7,13 @@ exports.MIGRATION_DBS = { exports.MIGRATIONS = { USER_EMAIL_VIEW_CASING: "user_email_view_casing", + SYNC_APP_AND_RESET_ROWS_QUOTAS: "sync_app_and_reset_rows_quotas", } const DB_LOOKUP = { [exports.MIGRATION_DBS.GLOBAL_DB]: [ exports.MIGRATIONS.USER_EMAIL_VIEW_CASING, + exports.MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS, ], } diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js index 2b189b8660..f9e5d4b6ee 100644 --- a/packages/server/src/middleware/usageQuota.js +++ b/packages/server/src/middleware/usageQuota.js @@ -17,14 +17,14 @@ const METHOD_MAP = { } const DOMAIN_MAP = { - rows: usageQuota.Properties.ROW, - upload: usageQuota.Properties.UPLOAD, - views: usageQuota.Properties.VIEW, - users: usageQuota.Properties.USER, + // rows: usageQuota.Properties.ROW, // works - disabled + // upload: usageQuota.Properties.UPLOAD, // doesn't work yet + // views: usageQuota.Properties.VIEW, // doesn't work yet + // users: usageQuota.Properties.USER, // doesn't work yet applications: usageQuota.Properties.APPS, // this will not be updated by endpoint calls // instead it will be updated by triggerInfo - automationRuns: usageQuota.Properties.AUTOMATION, + // automationRuns: usageQuota.Properties.AUTOMATION, // doesn't work yet } function getProperty(url) { diff --git a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js new file mode 100644 index 0000000000..39781471dd --- /dev/null +++ b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js @@ -0,0 +1,27 @@ +const { MIGRATIONS, MIGRATION_DBS, migrateIfRequired } = + require("@budibase/auth").migrations +const { getGlobalDB } = require("@budibase/auth/tenancy") +const { getUsageQuotaDoc } = require("../utilities/usageQuota") +const { getAllApps } = require("@budibase/auth/db") +const CouchDB = require("../db") + +exports.migrate = async () => { + await migrateIfRequired( + MIGRATION_DBS.GLOBAL_DB, + MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS, + async () => { + const globalDb = getGlobalDB() + const usageDoc = await getUsageQuotaDoc(globalDb) + + // reset the rows + usageDoc.usageQuota.rows = 0 + + // sync the apps + const apps = await getAllApps(CouchDB, { dev: true }) + const appCount = apps ? apps.length : 0 + usageDoc.usageQuota.apps = appCount + + await globalDb.put(usageDoc) + } + ) +} diff --git a/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js b/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js new file mode 100644 index 0000000000..a5b9f2d9c8 --- /dev/null +++ b/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js @@ -0,0 +1,39 @@ +const { getGlobalDB } = require("@budibase/auth/tenancy") +const TestConfig = require("../../tests/utilities/TestConfiguration") +const { getUsageQuotaDoc, update, Properties } = require("../../utilities/usageQuota") +const { migrate } = require("../sync_app_and_reset_rows_quotas") +const env = require("../../environment") + +describe("Sync App And Reset Rows Quotas Migration", () => { + let config = new TestConfig(false) + + beforeEach(async () => { + await config.init() + env._set("USE_QUOTAS", 1) + }) + + afterAll(config.end) + + it("migrates successfully", async () => { + // create the usage quota doc and mock usages + const db = getGlobalDB() + await getUsageQuotaDoc(db) + await update(Properties.APPS, 3) + await update(Properties.ROW, 300) + + let usageDoc = await getUsageQuotaDoc(db) + expect(usageDoc.usageQuota.apps).toEqual(3) + expect(usageDoc.usageQuota.rows).toEqual(300) + + // create an extra app to test the migration + await config.createApp("quota-test") + + // migrate + await migrate() + + // assert the migration worked + usageDoc = await getUsageQuotaDoc(db) + expect(usageDoc.usageQuota.apps).toEqual(2) + expect(usageDoc.usageQuota.rows).toEqual(0) + }) +}) diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js index 1980b13ef0..e669dfcefc 100644 --- a/packages/server/src/utilities/usageQuota.js +++ b/packages/server/src/utilities/usageQuota.js @@ -5,24 +5,20 @@ const { generateNewUsageQuotaDoc, } = require("@budibase/auth/db") -function getNewQuotaReset() { - return Date.now() + 2592000000 -} - exports.Properties = { - ROW: "rows", - UPLOAD: "storage", - VIEW: "views", - USER: "users", - AUTOMATION: "automationRuns", - APPS: "apps", - EMAILS: "emails", + ROW: "rows", // mostly works - disabled - app / table deletion not yet accounted for + UPLOAD: "storage", // doesn't work yet + VIEW: "views", // doesn't work yet + USER: "users", // doesn't work yet + AUTOMATION: "automationRuns", // doesn't work yet + APPS: "apps", // works + EMAILS: "emails", // doesn't work yet } -async function getUsageQuotaDoc(db) { +exports.getUsageQuotaDoc = async db => { let quota try { - quota = await db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota) + quota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota) } catch (err) { // doc doesn't exist. Create it quota = await db.post(generateNewUsageQuotaDoc()) @@ -45,15 +41,7 @@ exports.update = async (property, usage) => { try { const db = getGlobalDB() - const quota = await getUsageQuotaDoc(db) - - // Check if the quota needs reset - if (Date.now() >= quota.quotaReset) { - quota.quotaReset = getNewQuotaReset() - for (let prop of Object.keys(quota.usageQuota)) { - quota.usageQuota[prop] = 0 - } - } + const quota = await exports.getUsageQuotaDoc(db) // increment the quota quota.usageQuota[property] += usage diff --git a/packages/server/src/utilities/usageQuoteReset.js b/packages/server/src/utilities/usageQuoteReset.js new file mode 100644 index 0000000000..ff5a1aa00e --- /dev/null +++ b/packages/server/src/utilities/usageQuoteReset.js @@ -0,0 +1,18 @@ +// UNUSED CODE +// Preserved for future use + +/* eslint-disable no-unused-vars */ + +function getNewQuotaReset() { + return Date.now() + 2592000000 +} + +function resetQuotasIfRequired(quota) { + // Check if the quota needs reset + if (Date.now() >= quota.quotaReset) { + quota.quotaReset = getNewQuotaReset() + for (let prop of Object.keys(quota.usageQuota)) { + quota.usageQuota[prop] = 0 + } + } +} diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index 87194a7ceb..22f155b663 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -69,9 +69,7 @@ exports.adminUser = async ctx => { if (!env.SELF_HOSTED) { // could be a scenario where it exists, make sure its clean try { - const usageQuota = await db.get( - StaticDatabases.PLATFORM_INFO.docs.usageQuota - ) + const usageQuota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota) if (usageQuota) { await db.remove(usageQuota._id, usageQuota._rev) }