MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor to allow for future quota migrations

This commit is contained in:
Rory Powell 2022-01-18 14:43:24 +00:00
parent a09bbfb492
commit 747db3daa2
9 changed files with 151 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
packages/backend-core/src/migrations/index.js
View File

@ -7,13 +7,13 @@ exports.MIGRATION_DBS = {
exports.MIGRATIONS = {
USER_EMAIL_VIEW_CASING: "user_email_view_casing",
SYNC_APP_AND_RESET_ROWS_QUOTAS: "sync_app_and_reset_rows_quotas",
QUOTAS_1: "quotas_1",
}
const DB_LOOKUP = {
[exports.MIGRATION_DBS.GLOBAL_DB]: [
exports.MIGRATIONS.USER_EMAIL_VIEW_CASING,
exports.MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS,
exports.MIGRATIONS.QUOTAS_1,
],
}

15
packages/server/src/middleware/usageQuota.js
View File

@ -5,7 +5,7 @@ const {
isExternalTable,
isRowId: isExternalRowId,
} = require("../integrations/utils")
const quotaMigration = require("../migrations/sync_app_and_reset_rows_quotas")
const migration = require("../migrations/usageQuotas")
// currently only counting new writes and deletes
const METHOD_MAP = {
@ -74,7 +74,7 @@ module.exports = async (ctx, next) => {
usage = files.map(file => file.size).reduce((total, size) => total + size)
}
try {
await quotaMigration.runIfRequired()
await migration.run()
await performRequest(ctx, next, property, usage)
} catch (err) {
ctx.throw(400, err)
@ -142,6 +142,13 @@ const appPostDelete = async (ctx, usageContext) => {
}
}
// const appPostCreate = async (ctx, usageContext) => {
// if (ctx.request) {
// const rowCount = await getUniqueRows([ctx.appId]).length
// await usageQuota.update(usageQuota.Properties.ROW, -rowCount)
// }
// }
const PRE_DELETE = {
[usageQuota.Properties.APPS]: appPreDelete,
}
@ -152,4 +159,6 @@ const POST_DELETE = {
const PRE_CREATE = {}
const POST_CREATE = {}
const POST_CREATE = {
// [usageQuota.Properties.APPS]: appPostCreate,
}

48
packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
View File

@ -1,48 +0,0 @@
const {
MIGRATIONS,
MIGRATION_DBS,
migrateIfRequired,
} = require("@budibase/backend-core/migrations")
const { getGlobalDB } = require("@budibase/backend-core/tenancy")
const { getAllApps } = require("@budibase/backend-core/db")
const CouchDB = require("../db")
const { getUsageQuotaDoc, useQuotas } = require("../utilities/usageQuota")
const { getUniqueRows } = require("../utilities/usageQuota/rows")
const syncRowsQuota = async db => {
// get all rows in all apps
const allApps = await getAllApps(CouchDB, { all: true })
const appIds = allApps ? allApps.map(app => app.appId) : []
const rows = await getUniqueRows(appIds)
// sync row count
const usageDoc = await getUsageQuotaDoc(db)
usageDoc.usageQuota.rows = rows.length
await db.put(usageDoc)
}
const syncAppsQuota = async db => {
// get app count
const devApps = await getAllApps(CouchDB, { dev: true })
const appCount = devApps ? devApps.length : 0
// sync app count
const usageDoc = await getUsageQuotaDoc(db)
usageDoc.usageQuota.apps = appCount
await db.put(usageDoc)
}
exports.runIfRequired = async () => {
await migrateIfRequired(
MIGRATION_DBS.GLOBAL_DB,
MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS,
async () => {
if (!useQuotas()) {
return
}
const db = getGlobalDB()
await syncAppsQuota(db)
await syncRowsQuota(db)
}
)
}

27
packages/server/src/migrations/tests/usageQuotas/index.spec.js Normal file
View File

@ -0,0 +1,27 @@
const env = require("../../../environment")
const TestConfig = require("../../../tests/utilities/TestConfiguration")
const syncApps = jest.fn()
const syncRows = jest.fn()
jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) )
jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) )
const migrations = require("../../usageQuotas")
describe("run", () => {
let config = new TestConfig(false)
beforeEach(async () => {
await config.init()
env._set("USE_QUOTAS", 1)
})
afterAll(config.end)
it("runs the required migrations", async () => {
await migrations.run()
expect(syncApps).toHaveBeenCalledTimes(1)
expect(syncRows).toHaveBeenCalledTimes(1)
})
})

18
packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js → packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js
View File

@ -1,10 +1,10 @@
const { getGlobalDB } = require("@budibase/backend-core/tenancy")
const TestConfig = require("../../tests/utilities/TestConfiguration")
const { getUsageQuotaDoc, update, Properties } = require("../../utilities/usageQuota")
const { runIfRequired } = require("../sync_app_and_reset_rows_quotas")
const env = require("../../environment")
const TestConfig = require("../../../tests/utilities/TestConfiguration")
const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
const syncApps = require("../../usageQuotas/syncApps")
const env = require("../../../environment")
describe("Sync App And Reset Rows Quotas Migration", () => {
describe("syncApps", () => {
let config = new TestConfig(false)
beforeEach(async () => {
@ -14,26 +14,24 @@ describe("Sync App And Reset Rows Quotas Migration", () => {
afterAll(config.end)
it("migrates successfully", async () => {
it("runs successfully", async () => {
// create the usage quota doc and mock usages
const db = getGlobalDB()
await getUsageQuotaDoc(db)
await update(Properties.APPS, 3)
await update(Properties.ROW, 300)
let usageDoc = await getUsageQuotaDoc(db)
expect(usageDoc.usageQuota.apps).toEqual(3)
expect(usageDoc.usageQuota.rows).toEqual(300)
// create an extra app to test the migration
await config.createApp("quota-test")
// migrate
await runIfRequired()
await syncApps.run()
// assert the migration worked
usageDoc = await getUsageQuotaDoc(db)
expect(usageDoc.usageQuota.apps).toEqual(2)
expect(usageDoc.usageQuota.rows).toEqual(0)
})
})

43
packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js Normal file
View File

@ -0,0 +1,43 @@
const { getGlobalDB } = require("@budibase/backend-core/tenancy")
const TestConfig = require("../../../tests/utilities/TestConfiguration")
const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
const syncRows = require("../../usageQuotas/syncRows")
const env = require("../../../environment")
describe("syncRows", () => {
let config = new TestConfig(false)
beforeEach(async () => {
await config.init()
env._set("USE_QUOTAS", 1)
})
afterAll(config.end)
it("runs successfully", async () => {
// create the usage quota doc and mock usages
const db = getGlobalDB()
await getUsageQuotaDoc(db)
await update(Properties.ROW, 300)
let usageDoc = await getUsageQuotaDoc(db)
expect(usageDoc.usageQuota.rows).toEqual(300)
// app 1
await config.createTable()
await config.createRow()
// app 2
await config.createApp()
await config.createTable()
await config.createRow()
await config.createRow()
// migrate
await syncRows.run()
// assert the migration worked
usageDoc = await getUsageQuotaDoc(db)
expect(usageDoc.usageQuota.rows).toEqual(3)
})
})

24
packages/server/src/migrations/usageQuotas/index.js Normal file
View File

@ -0,0 +1,24 @@
const {
MIGRATIONS,
MIGRATION_DBS,
migrateIfRequired,
} = require("@budibase/backend-core/migrations")
const { useQuotas } = require("../../utilities/usageQuota")
const syncApps = require("./syncApps")
const syncRows = require("./syncRows")
exports.run = async () => {
if (!useQuotas()) {
return
}
// Jan 2022
await migrateIfRequired(
MIGRATION_DBS.GLOBAL_DB,
MIGRATIONS.QUOTAS_1,
async () => {
await syncApps.run()
await syncRows.run()
}
)
}

16
packages/server/src/migrations/usageQuotas/syncApps.js Normal file
View File

@ -0,0 +1,16 @@
const { getGlobalDB } = require("@budibase/backend-core/tenancy")
const { getAllApps } = require("@budibase/backend-core/db")
const CouchDB = require("../../db")
const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
exports.run = async () => {
const db = getGlobalDB()
// get app count
const devApps = await getAllApps(CouchDB, { dev: true })
const appCount = devApps ? devApps.length : 0
// sync app count
const usageDoc = await getUsageQuotaDoc(db)
usageDoc.usageQuota.apps = appCount
await db.put(usageDoc)
}

18
packages/server/src/migrations/usageQuotas/syncRows.js Normal file
View File

@ -0,0 +1,18 @@
const { getGlobalDB } = require("@budibase/backend-core/tenancy")
const { getAllApps } = require("@budibase/backend-core/db")
const CouchDB = require("../../db")
const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
const { getUniqueRows } = require("../../utilities/usageQuota/rows")
exports.run = async () => {
const db = getGlobalDB()
// get all rows in all apps
const allApps = await getAllApps(CouchDB, { all: true })
const appIds = allApps ? allApps.map(app => app.appId) : []
const rows = await getUniqueRows(appIds)
// sync row count
const usageDoc = await getUsageQuotaDoc(db)
usageDoc.usageQuota.rows = rows.length
await db.put(usageDoc)
}