Refactor to allow for future quota migrations

packages/backend-core/src/migrations/index.js

@@ -7,13 +7,13 @@ exports.MIGRATION_DBS = {
 
 exports.MIGRATIONS = {
   USER_EMAIL_VIEW_CASING: "user_email_view_casing",
-  SYNC_APP_AND_RESET_ROWS_QUOTAS: "sync_app_and_reset_rows_quotas",
+  QUOTAS_1: "quotas_1",
 }
 
 const DB_LOOKUP = {
   [exports.MIGRATION_DBS.GLOBAL_DB]: [
     exports.MIGRATIONS.USER_EMAIL_VIEW_CASING,
-    exports.MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS,
+    exports.MIGRATIONS.QUOTAS_1,
   ],
 }
 

packages/server/src/middleware/usageQuota.js

@@ -5,7 +5,7 @@ const {
   isExternalTable,
   isRowId: isExternalRowId,
 } = require("../integrations/utils")
-const quotaMigration = require("../migrations/sync_app_and_reset_rows_quotas")
+const migration = require("../migrations/usageQuotas")
 
 // currently only counting new writes and deletes
 const METHOD_MAP = {
@@ -74,7 +74,7 @@ module.exports = async (ctx, next) => {
     usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
   try {
-    await quotaMigration.runIfRequired()
+    await migration.run()
     await performRequest(ctx, next, property, usage)
   } catch (err) {
     ctx.throw(400, err)
@@ -142,6 +142,13 @@ const appPostDelete = async (ctx, usageContext) => {
   }
 }
 
+// const appPostCreate = async (ctx, usageContext) => {
+// if (ctx.request) {
+//   const rowCount = await getUniqueRows([ctx.appId]).length
+//   await usageQuota.update(usageQuota.Properties.ROW, -rowCount)
+// }
+// }
+
 const PRE_DELETE = {
   [usageQuota.Properties.APPS]: appPreDelete,
 }
@@ -152,4 +159,6 @@ const POST_DELETE = {
 
 const PRE_CREATE = {}
 
-const POST_CREATE = {}
+const POST_CREATE = {
+  // [usageQuota.Properties.APPS]: appPostCreate,
+}

packages/server/src/migrations/sync_app_and_reset_rows_quotas.js

@@ -1,48 +0,0 @@
-const {
-  MIGRATIONS,
-  MIGRATION_DBS,
-  migrateIfRequired,
-} = require("@budibase/backend-core/migrations")
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const { getAllApps } = require("@budibase/backend-core/db")
-const CouchDB = require("../db")
-const { getUsageQuotaDoc, useQuotas } = require("../utilities/usageQuota")
-const { getUniqueRows } = require("../utilities/usageQuota/rows")
-
-const syncRowsQuota = async db => {
-  // get all rows in all apps
-  const allApps = await getAllApps(CouchDB, { all: true })
-  const appIds = allApps ? allApps.map(app => app.appId) : []
-  const rows = await getUniqueRows(appIds)
-
-  // sync row count
-  const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.rows = rows.length
-  await db.put(usageDoc)
-}
-
-const syncAppsQuota = async db => {
-  // get app count
-  const devApps = await getAllApps(CouchDB, { dev: true })
-  const appCount = devApps ? devApps.length : 0
-
-  // sync app count
-  const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.apps = appCount
-  await db.put(usageDoc)
-}
-
-exports.runIfRequired = async () => {
-  await migrateIfRequired(
-    MIGRATION_DBS.GLOBAL_DB,
-    MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS,
-    async () => {
-      if (!useQuotas()) {
-        return
-      }
-      const db = getGlobalDB()
-      await syncAppsQuota(db)
-      await syncRowsQuota(db)
-    }
-  )
-}

packages/server/src/migrations/tests/usageQuotas/index.spec.js

@@ -0,0 +1,27 @@
+const env = require("../../../environment")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+
+const syncApps = jest.fn()
+const syncRows = jest.fn()
+
+jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) )
+jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) )
+
+const migrations = require("../../usageQuotas")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs the required migrations", async () => {
+    await migrations.run()
+    expect(syncApps).toHaveBeenCalledTimes(1)
+    expect(syncRows).toHaveBeenCalledTimes(1)
+  })
+})

packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js

@@ -1,10 +1,10 @@
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const TestConfig = require("../../tests/utilities/TestConfiguration")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
-const { getUsageQuotaDoc, update, Properties } = require("../../utilities/usageQuota")
+const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
-const { runIfRequired } = require("../sync_app_and_reset_rows_quotas")
+const syncApps = require("../../usageQuotas/syncApps")
-const env = require("../../environment")
+const env = require("../../../environment")
 
-describe("Sync App And Reset Rows Quotas Migration", () => {
+describe("syncApps", () => {
   let config = new TestConfig(false)
 
   beforeEach(async () => {
@@ -14,26 +14,24 @@ describe("Sync App And Reset Rows Quotas Migration", () => {
 
   afterAll(config.end)  
 
-  it("migrates successfully", async () => {
+  it("runs successfully", async () => {
     // create the usage quota doc and mock usages
     const db = getGlobalDB()
     await getUsageQuotaDoc(db)
     await update(Properties.APPS, 3)
-    await update(Properties.ROW, 300)
 
     let usageDoc = await getUsageQuotaDoc(db)
     expect(usageDoc.usageQuota.apps).toEqual(3)
-    expect(usageDoc.usageQuota.rows).toEqual(300)
 
     // create an extra app to test the migration
     await config.createApp("quota-test")
 
     // migrate
-    await runIfRequired()
+    await syncApps.run()
 
     // assert the migration worked 
     usageDoc = await getUsageQuotaDoc(db)
     expect(usageDoc.usageQuota.apps).toEqual(2)
-    expect(usageDoc.usageQuota.rows).toEqual(0)
   })
 })
+

packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js

@@ -0,0 +1,43 @@
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
+const syncRows = require("../../usageQuotas/syncRows")
+const env = require("../../../environment")
+
+describe("syncRows", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    // create the usage quota doc and mock usages
+    const db = getGlobalDB()
+    await getUsageQuotaDoc(db)
+    await update(Properties.ROW, 300)
+
+    let usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.rows).toEqual(300)
+
+    // app 1
+    await config.createTable()
+    await config.createRow()
+    // app 2
+    await config.createApp()
+    await config.createTable()
+    await config.createRow()
+    await config.createRow()
+    
+    // migrate
+    await syncRows.run()
+
+    // assert the migration worked 
+    usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.rows).toEqual(3)
+  })
+})
+

packages/server/src/migrations/usageQuotas/index.js

@@ -0,0 +1,24 @@
+const {
+  MIGRATIONS,
+  MIGRATION_DBS,
+  migrateIfRequired,
+} = require("@budibase/backend-core/migrations")
+const { useQuotas } = require("../../utilities/usageQuota")
+const syncApps = require("./syncApps")
+const syncRows = require("./syncRows")
+
+exports.run = async () => {
+  if (!useQuotas()) {
+    return
+  }
+
+  // Jan 2022
+  await migrateIfRequired(
+    MIGRATION_DBS.GLOBAL_DB,
+    MIGRATIONS.QUOTAS_1,
+    async () => {
+      await syncApps.run()
+      await syncRows.run()
+    }
+  )
+}

packages/server/src/migrations/usageQuotas/syncApps.js

@@ -0,0 +1,16 @@
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { getAllApps } = require("@budibase/backend-core/db")
+const CouchDB = require("../../db")
+const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+
+exports.run = async () => {
+  const db = getGlobalDB()
+  // get app count
+  const devApps = await getAllApps(CouchDB, { dev: true })
+  const appCount = devApps ? devApps.length : 0
+
+  // sync app count
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.apps = appCount
+  await db.put(usageDoc)
+}

packages/server/src/migrations/usageQuotas/syncRows.js

@@ -0,0 +1,18 @@
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { getAllApps } = require("@budibase/backend-core/db")
+const CouchDB = require("../../db")
+const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+const { getUniqueRows } = require("../../utilities/usageQuota/rows")
+
+exports.run = async () => {
+  const db = getGlobalDB()
+  // get all rows in all apps
+  const allApps = await getAllApps(CouchDB, { all: true })
+  const appIds = allApps ? allApps.map(app => app.appId) : []
+  const rows = await getUniqueRows(appIds)
+
+  // sync row count
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.rows = rows.length
+  await db.put(usageDoc)
+}