From 726315afde528d18a068f67ecb7048367045a11d Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Tue, 11 Jan 2022 17:49:42 +0000
Subject: [PATCH 01/12] Sync app quota to correct number, reset and disable
 rows quota

---
 packages/auth/src/db/constants.js             |  2 +-
 packages/auth/src/db/utils.js                 |  2 +-
 packages/auth/src/index.js                    |  1 +
 packages/auth/src/migrations/index.js         |  2 +
 packages/server/src/middleware/usageQuota.js  | 10 ++---
 .../sync_app_and_reset_rows_quotas.js         | 27 +++++++++++++
 .../sync_app_and_reset_rows_quotas.spec.js    | 39 +++++++++++++++++++
 packages/server/src/utilities/usageQuota.js   | 32 +++++----------
 .../server/src/utilities/usageQuoteReset.js   | 18 +++++++++
 .../src/api/controllers/global/users.js       |  4 +-
 10 files changed, 105 insertions(+), 32 deletions(-)
 create mode 100644 packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
 create mode 100644 packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js
 create mode 100644 packages/server/src/utilities/usageQuoteReset.js

diff --git a/packages/auth/src/db/constants.js b/packages/auth/src/db/constants.js
index ecdaae5bad..2affb09c7c 100644
--- a/packages/auth/src/db/constants.js
+++ b/packages/auth/src/db/constants.js
@@ -21,6 +21,7 @@ exports.StaticDatabases = {
     name: "global-db",
     docs: {
       apiKeys: "apikeys",
+      usageQuota: "usage_quota",
     },
   },
   // contains information about tenancy and so on
@@ -28,7 +29,6 @@ exports.StaticDatabases = {
     name: "global-info",
     docs: {
       tenants: "tenants",
-      usageQuota: "usage_quota",
     },
   },
 }
diff --git a/packages/auth/src/db/utils.js b/packages/auth/src/db/utils.js
index 5830de4721..2bc5462646 100644
--- a/packages/auth/src/db/utils.js
+++ b/packages/auth/src/db/utils.js
@@ -450,7 +450,7 @@ async function getScopedConfig(db, params) {
 
 function generateNewUsageQuotaDoc() {
   return {
-    _id: StaticDatabases.PLATFORM_INFO.docs.usageQuota,
+    _id: StaticDatabases.GLOBAL.docs.usageQuota,
     quotaReset: Date.now() + 2592000000,
     usageQuota: {
       automationRuns: 0,
diff --git a/packages/auth/src/index.js b/packages/auth/src/index.js
index 4aa2c8ab96..bd8d7c2118 100644
--- a/packages/auth/src/index.js
+++ b/packages/auth/src/index.js
@@ -68,4 +68,5 @@ module.exports = {
   },
   StaticDatabases,
   constants: require("./constants"),
+  migrations: require("./migrations"),
 }
diff --git a/packages/auth/src/migrations/index.js b/packages/auth/src/migrations/index.js
index 7492e94511..30e04e198b 100644
--- a/packages/auth/src/migrations/index.js
+++ b/packages/auth/src/migrations/index.js
@@ -7,11 +7,13 @@ exports.MIGRATION_DBS = {
 
 exports.MIGRATIONS = {
   USER_EMAIL_VIEW_CASING: "user_email_view_casing",
+  SYNC_APP_AND_RESET_ROWS_QUOTAS: "sync_app_and_reset_rows_quotas",
 }
 
 const DB_LOOKUP = {
   [exports.MIGRATION_DBS.GLOBAL_DB]: [
     exports.MIGRATIONS.USER_EMAIL_VIEW_CASING,
+    exports.MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS,
   ],
 }
 
diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js
index 2b189b8660..f9e5d4b6ee 100644
--- a/packages/server/src/middleware/usageQuota.js
+++ b/packages/server/src/middleware/usageQuota.js
@@ -17,14 +17,14 @@ const METHOD_MAP = {
 }
 
 const DOMAIN_MAP = {
-  rows: usageQuota.Properties.ROW,
-  upload: usageQuota.Properties.UPLOAD,
-  views: usageQuota.Properties.VIEW,
-  users: usageQuota.Properties.USER,
+  // rows: usageQuota.Properties.ROW,                 // works - disabled
+  // upload: usageQuota.Properties.UPLOAD,            // doesn't work yet
+  // views: usageQuota.Properties.VIEW,               // doesn't work yet
+  // users: usageQuota.Properties.USER,               // doesn't work yet
   applications: usageQuota.Properties.APPS,
   // this will not be updated by endpoint calls
   // instead it will be updated by triggerInfo
-  automationRuns: usageQuota.Properties.AUTOMATION,
+  // automationRuns: usageQuota.Properties.AUTOMATION, // doesn't work yet
 }
 
 function getProperty(url) {
diff --git a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
new file mode 100644
index 0000000000..39781471dd
--- /dev/null
+++ b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
@@ -0,0 +1,27 @@
+const { MIGRATIONS, MIGRATION_DBS, migrateIfRequired } =
+  require("@budibase/auth").migrations
+const { getGlobalDB } = require("@budibase/auth/tenancy")
+const { getUsageQuotaDoc } = require("../utilities/usageQuota")
+const { getAllApps } = require("@budibase/auth/db")
+const CouchDB = require("../db")
+
+exports.migrate = async () => {
+  await migrateIfRequired(
+    MIGRATION_DBS.GLOBAL_DB,
+    MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS,
+    async () => {
+      const globalDb = getGlobalDB()
+      const usageDoc = await getUsageQuotaDoc(globalDb)
+
+      // reset the rows
+      usageDoc.usageQuota.rows = 0
+
+      // sync the apps
+      const apps = await getAllApps(CouchDB, { dev: true })
+      const appCount = apps ? apps.length : 0
+      usageDoc.usageQuota.apps = appCount
+
+      await globalDb.put(usageDoc)
+    }
+  )
+}
diff --git a/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js b/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js
new file mode 100644
index 0000000000..a5b9f2d9c8
--- /dev/null
+++ b/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js
@@ -0,0 +1,39 @@
+const { getGlobalDB } = require("@budibase/auth/tenancy")
+const TestConfig = require("../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../utilities/usageQuota")
+const { migrate } = require("../sync_app_and_reset_rows_quotas")
+const env = require("../../environment")
+
+describe("Sync App And Reset Rows Quotas Migration", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)
+
+  it("migrates successfully", async () => {
+    // create the usage quota doc and mock usages
+    const db = getGlobalDB()
+    await getUsageQuotaDoc(db)
+    await update(Properties.APPS, 3)
+    await update(Properties.ROW, 300)
+
+    let usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.apps).toEqual(3)
+    expect(usageDoc.usageQuota.rows).toEqual(300)
+
+    // create an extra app to test the migration
+    await config.createApp("quota-test")
+
+    // migrate
+    await migrate()
+
+    // assert the migration worked 
+    usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.apps).toEqual(2)
+    expect(usageDoc.usageQuota.rows).toEqual(0)
+  })
+})
diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js
index 1980b13ef0..e669dfcefc 100644
--- a/packages/server/src/utilities/usageQuota.js
+++ b/packages/server/src/utilities/usageQuota.js
@@ -5,24 +5,20 @@ const {
   generateNewUsageQuotaDoc,
 } = require("@budibase/auth/db")
 
-function getNewQuotaReset() {
-  return Date.now() + 2592000000
-}
-
 exports.Properties = {
-  ROW: "rows",
-  UPLOAD: "storage",
-  VIEW: "views",
-  USER: "users",
-  AUTOMATION: "automationRuns",
-  APPS: "apps",
-  EMAILS: "emails",
+  ROW: "rows", // mostly works - disabled - app / table deletion not yet accounted for
+  UPLOAD: "storage", // doesn't work yet
+  VIEW: "views", // doesn't work yet
+  USER: "users", // doesn't work yet
+  AUTOMATION: "automationRuns", // doesn't work yet
+  APPS: "apps", // works
+  EMAILS: "emails", // doesn't work yet
 }
 
-async function getUsageQuotaDoc(db) {
+exports.getUsageQuotaDoc = async db => {
   let quota
   try {
-    quota = await db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota)
+    quota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota)
   } catch (err) {
     // doc doesn't exist. Create it
     quota = await db.post(generateNewUsageQuotaDoc())
@@ -45,15 +41,7 @@ exports.update = async (property, usage) => {
 
   try {
     const db = getGlobalDB()
-    const quota = await getUsageQuotaDoc(db)
-
-    // Check if the quota needs reset
-    if (Date.now() >= quota.quotaReset) {
-      quota.quotaReset = getNewQuotaReset()
-      for (let prop of Object.keys(quota.usageQuota)) {
-        quota.usageQuota[prop] = 0
-      }
-    }
+    const quota = await exports.getUsageQuotaDoc(db)
 
     // increment the quota
     quota.usageQuota[property] += usage
diff --git a/packages/server/src/utilities/usageQuoteReset.js b/packages/server/src/utilities/usageQuoteReset.js
new file mode 100644
index 0000000000..ff5a1aa00e
--- /dev/null
+++ b/packages/server/src/utilities/usageQuoteReset.js
@@ -0,0 +1,18 @@
+// UNUSED CODE
+// Preserved for future use
+
+/* eslint-disable no-unused-vars */
+
+function getNewQuotaReset() {
+  return Date.now() + 2592000000
+}
+
+function resetQuotasIfRequired(quota) {
+  // Check if the quota needs reset
+  if (Date.now() >= quota.quotaReset) {
+    quota.quotaReset = getNewQuotaReset()
+    for (let prop of Object.keys(quota.usageQuota)) {
+      quota.usageQuota[prop] = 0
+    }
+  }
+}
diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js
index 87194a7ceb..22f155b663 100644
--- a/packages/worker/src/api/controllers/global/users.js
+++ b/packages/worker/src/api/controllers/global/users.js
@@ -69,9 +69,7 @@ exports.adminUser = async ctx => {
   if (!env.SELF_HOSTED) {
     // could be a scenario where it exists, make sure its clean
     try {
-      const usageQuota = await db.get(
-        StaticDatabases.PLATFORM_INFO.docs.usageQuota
-      )
+      const usageQuota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota)
       if (usageQuota) {
         await db.remove(usageQuota._id, usageQuota._rev)
       }

From 979313f9669427af8bf77b82000ecdd51fb828b7 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Tue, 11 Jan 2022 18:38:18 +0000
Subject: [PATCH 02/12] Enable migration

---
 .../src/middleware/tests/usageQuota.spec.js   | 35 ++++++++++---------
 packages/server/src/middleware/usageQuota.js  |  2 ++
 .../sync_app_and_reset_rows_quotas.js         | 10 +++---
 .../sync_app_and_reset_rows_quotas.spec.js    |  4 +--
 packages/server/src/utilities/usageQuota.js   |  4 ++-
 .../src/api/controllers/global/users.js       |  2 +-
 6 files changed, 31 insertions(+), 26 deletions(-)

diff --git a/packages/server/src/middleware/tests/usageQuota.spec.js b/packages/server/src/middleware/tests/usageQuota.spec.js
index 7a64b1a039..42a7ee524f 100644
--- a/packages/server/src/middleware/tests/usageQuota.spec.js
+++ b/packages/server/src/middleware/tests/usageQuota.spec.js
@@ -29,7 +29,7 @@ class TestConfiguration {
       },
       req: {
         method: "POST",
-        url: "/rows"
+        url: "/applications"
       }
     }
   }
@@ -117,24 +117,25 @@ describe("usageQuota middleware", () => {
 
     await config.executeMiddleware()
 
-    expect(usageQuota.update).toHaveBeenCalledWith("rows", 1)
+    // expect(usageQuota.update).toHaveBeenCalledWith("rows", 1)
+    expect(usageQuota.update).not.toHaveBeenCalledWith("rows", 1)
     expect(config.next).toHaveBeenCalled()
   })
 
-  it("calculates the correct file size from a file upload call and adds it to quota", async () => {
-    config.setUrl("/upload")
-    config.setProd(true)
-    config.setFiles([
-      {
-        size: 100
-      },
-      {
-        size: 10000
-      },
-    ])
-    await config.executeMiddleware()
+  // it("calculates the correct file size from a file upload call and adds it to quota", async () => {
+  //   config.setUrl("/upload")
+  //   config.setProd(true)
+  //   config.setFiles([
+  //     {
+  //       size: 100
+  //     },
+  //     {
+  //       size: 10000
+  //     },
+  //   ])
+  //   await config.executeMiddleware()
 
-    expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
-    expect(config.next).toHaveBeenCalled()
-  })
+  //   expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
+  //   expect(config.next).toHaveBeenCalled()
+  // })
 })
\ No newline at end of file
diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js
index f9e5d4b6ee..e487b555a0 100644
--- a/packages/server/src/middleware/usageQuota.js
+++ b/packages/server/src/middleware/usageQuota.js
@@ -6,6 +6,7 @@ const {
   isExternalTable,
   isRowId: isExternalRowId,
 } = require("../integrations/utils")
+const quotaMigration = require("../migrations/sync_app_and_reset_rows_quotas")
 
 // tenants without limits
 const EXCLUDED_TENANTS = ["bb", "default", "bbtest", "bbstaging"]
@@ -80,6 +81,7 @@ module.exports = async (ctx, next) => {
     usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
   try {
+    await quotaMigration.runIfRequired()
     await usageQuota.update(property, usage)
     return next()
   } catch (err) {
diff --git a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
index 39781471dd..6795d062a2 100644
--- a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
+++ b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
@@ -1,17 +1,17 @@
 const { MIGRATIONS, MIGRATION_DBS, migrateIfRequired } =
   require("@budibase/auth").migrations
 const { getGlobalDB } = require("@budibase/auth/tenancy")
-const { getUsageQuotaDoc } = require("../utilities/usageQuota")
 const { getAllApps } = require("@budibase/auth/db")
 const CouchDB = require("../db")
+const { getUsageQuotaDoc } = require("../utilities/usageQuota")
 
-exports.migrate = async () => {
+exports.runIfRequired = async () => {
   await migrateIfRequired(
     MIGRATION_DBS.GLOBAL_DB,
     MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS,
     async () => {
-      const globalDb = getGlobalDB()
-      const usageDoc = await getUsageQuotaDoc(globalDb)
+      const db = getGlobalDB()
+      const usageDoc = await getUsageQuotaDoc(db)
 
       // reset the rows
       usageDoc.usageQuota.rows = 0
@@ -21,7 +21,7 @@ exports.migrate = async () => {
       const appCount = apps ? apps.length : 0
       usageDoc.usageQuota.apps = appCount
 
-      await globalDb.put(usageDoc)
+      await db.put(usageDoc)
     }
   )
 }
diff --git a/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js b/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js
index a5b9f2d9c8..69ffd0a9e2 100644
--- a/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js
+++ b/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js
@@ -1,7 +1,7 @@
 const { getGlobalDB } = require("@budibase/auth/tenancy")
 const TestConfig = require("../../tests/utilities/TestConfiguration")
 const { getUsageQuotaDoc, update, Properties } = require("../../utilities/usageQuota")
-const { migrate } = require("../sync_app_and_reset_rows_quotas")
+const { runIfRequired } = require("../sync_app_and_reset_rows_quotas")
 const env = require("../../environment")
 
 describe("Sync App And Reset Rows Quotas Migration", () => {
@@ -29,7 +29,7 @@ describe("Sync App And Reset Rows Quotas Migration", () => {
     await config.createApp("quota-test")
 
     // migrate
-    await migrate()
+    await runIfRequired()
 
     // assert the migration worked 
     usageDoc = await getUsageQuotaDoc(db)
diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js
index e669dfcefc..7bc3e2eb6e 100644
--- a/packages/server/src/utilities/usageQuota.js
+++ b/packages/server/src/utilities/usageQuota.js
@@ -21,7 +21,9 @@ exports.getUsageQuotaDoc = async db => {
     quota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota)
   } catch (err) {
     // doc doesn't exist. Create it
-    quota = await db.post(generateNewUsageQuotaDoc())
+    quota = generateNewUsageQuotaDoc()
+    const response = await db.put(quota)
+    quota._rev = response.rev
   }
 
   return quota
diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js
index 22f155b663..1f85e316a2 100644
--- a/packages/worker/src/api/controllers/global/users.js
+++ b/packages/worker/src/api/controllers/global/users.js
@@ -76,7 +76,7 @@ exports.adminUser = async ctx => {
     } catch (err) {
       // don't worry about errors
     }
-    await db.post(generateNewUsageQuotaDoc())
+    await db.put(generateNewUsageQuotaDoc())
   }
 
   if (response.rows.some(row => row.doc.admin)) {

From da12a33cc101f91666b3c0d77da7c0d5bf1cc136 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Tue, 11 Jan 2022 18:47:42 +0000
Subject: [PATCH 03/12] Add local dev override for quotas

---
 packages/server/src/middleware/usageQuota.js | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js
index e487b555a0..3f8528d27f 100644
--- a/packages/server/src/middleware/usageQuota.js
+++ b/packages/server/src/middleware/usageQuota.js
@@ -8,6 +8,8 @@ const {
 } = require("../integrations/utils")
 const quotaMigration = require("../migrations/sync_app_and_reset_rows_quotas")
 
+const testing = false
+
 // tenants without limits
 const EXCLUDED_TENANTS = ["bb", "default", "bbtest", "bbstaging"]
 
@@ -40,7 +42,10 @@ module.exports = async (ctx, next) => {
   const tenantId = getTenantId()
 
   // if in development or a self hosted cloud usage quotas should not be executed
-  if (env.isDev() || env.SELF_HOSTED || EXCLUDED_TENANTS.includes(tenantId)) {
+  if (
+    (env.isDev() || env.SELF_HOSTED || EXCLUDED_TENANTS.includes(tenantId)) &&
+    !testing
+  ) {
     return next()
   }
 

From adcb692652a82f6df8c4cbbbb9d9703f69c26ae8 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Wed, 12 Jan 2022 09:53:48 +0000
Subject: [PATCH 04/12] Replace auth with backend-core

---
 .../server/src/migrations/sync_app_and_reset_rows_quotas.js | 6 +++---
 .../migrations/tests/sync_app_and_reset_rows_quotas.spec.js | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
index 6795d062a2..3aa3ca7354 100644
--- a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
+++ b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
@@ -1,7 +1,7 @@
 const { MIGRATIONS, MIGRATION_DBS, migrateIfRequired } =
-  require("@budibase/auth").migrations
-const { getGlobalDB } = require("@budibase/auth/tenancy")
-const { getAllApps } = require("@budibase/auth/db")
+  require("@budibase/backend-core").migrations
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { getAllApps } = require("@budibase/backend-core/db")
 const CouchDB = require("../db")
 const { getUsageQuotaDoc } = require("../utilities/usageQuota")
 
diff --git a/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js b/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js
index 69ffd0a9e2..cd533e3f06 100644
--- a/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js
+++ b/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js
@@ -1,4 +1,4 @@
-const { getGlobalDB } = require("@budibase/auth/tenancy")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 const TestConfig = require("../../tests/utilities/TestConfiguration")
 const { getUsageQuotaDoc, update, Properties } = require("../../utilities/usageQuota")
 const { runIfRequired } = require("../sync_app_and_reset_rows_quotas")

From 8fc60af820f4c23991cb99e42ecd316ffe7404a0 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Mon, 17 Jan 2022 12:44:53 +0000
Subject: [PATCH 05/12] Calculate total rows in migration, centralise quota
 enabled logic and tidy to use env vars only

---
 .../templates/app-service-deployment.yaml     |  4 +-
 charts/budibase/values.yaml                   |  5 +-
 .../server/src/automations/steps/createRow.js |  5 +-
 packages/server/src/environment.js            |  1 +
 packages/server/src/integrations/s3.ts        |  2 +-
 .../src/middleware/tests/usageQuota.spec.js   | 11 +--
 packages/server/src/middleware/usageQuota.js  | 17 +---
 .../sync_app_and_reset_rows_quotas.js         | 79 ++++++++++++++++---
 .../src/utilities/tests/usageQuota.spec.js    | 72 +++++++++++++++++
 packages/server/src/utilities/usageQuota.js   | 25 +++++-
 10 files changed, 175 insertions(+), 46 deletions(-)
 create mode 100644 packages/server/src/utilities/tests/usageQuota.spec.js

diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml
index 8086c0ab20..cd43631992 100644
--- a/charts/budibase/templates/app-service-deployment.yaml
+++ b/charts/budibase/templates/app-service-deployment.yaml
@@ -99,7 +99,9 @@ spec:
         - name: PLATFORM_URL
           value: {{ .Values.globals.platformUrl | quote }}
         - name: USE_QUOTAS
-          value: "1" 
+          value: {{ .Values.globals.useQuotas | quote }}
+        - name: EXCLUDE_QUOTAS_TENANTS
+          value: {{ .Values.globals.excludeQuotasTenants | quote }}
         - name: ACCOUNT_PORTAL_URL
           value: {{ .Values.globals.accountPortalUrl | quote }}
         - name: ACCOUNT_PORTAL_API_KEY
diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml
index 4666d01c70..9ea055c6c0 100644
--- a/charts/budibase/values.yaml
+++ b/charts/budibase/values.yaml
@@ -93,6 +93,8 @@ globals:
   logLevel: info
   selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
   multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs
+  useQuotas: "0"
+  excludeQuotasTenants: "" # comma seperated list of tenants to exclude from quotas
   accountPortalUrl: ""
   accountPortalApiKey: ""
   cookieDomain: ""
@@ -239,7 +241,8 @@ couchdb:
     hosts:
       - chart-example.local
     path: /
-    annotations: []
+    annotations:
+      []
       # kubernetes.io/ingress.class: nginx
       # kubernetes.io/tls-acme: "true"
     tls:
diff --git a/packages/server/src/automations/steps/createRow.js b/packages/server/src/automations/steps/createRow.js
index 8e5b44cc06..816cd829ab 100644
--- a/packages/server/src/automations/steps/createRow.js
+++ b/packages/server/src/automations/steps/createRow.js
@@ -1,6 +1,5 @@
 const rowController = require("../../api/controllers/row")
 const automationUtils = require("../automationUtils")
-const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")
 const { buildCtx } = require("./utils")
 
@@ -83,9 +82,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
       inputs.row.tableId,
       inputs.row
     )
-    if (env.USE_QUOTAS) {
-      await usage.update(usage.Properties.ROW, 1)
-    }
+    await usage.update(usage.Properties.ROW, 1)
     await rowController.save(ctx)
     return {
       row: inputs.row,
diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js
index a92e113851..614f41a29f 100644
--- a/packages/server/src/environment.js
+++ b/packages/server/src/environment.js
@@ -38,6 +38,7 @@ module.exports = {
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   USE_QUOTAS: process.env.USE_QUOTAS,
+  EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS,
   REDIS_URL: process.env.REDIS_URL,
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts
index 25b439fd58..273f221575 100644
--- a/packages/server/src/integrations/s3.ts
+++ b/packages/server/src/integrations/s3.ts
@@ -38,7 +38,7 @@ module S3Module {
       signatureVersion: {
         type: "string",
         required: false,
-        default: "v4"
+        default: "v4",
       },
     },
     query: {
diff --git a/packages/server/src/middleware/tests/usageQuota.spec.js b/packages/server/src/middleware/tests/usageQuota.spec.js
index 2e92eb9957..1282615a50 100644
--- a/packages/server/src/middleware/tests/usageQuota.spec.js
+++ b/packages/server/src/middleware/tests/usageQuota.spec.js
@@ -1,11 +1,5 @@
 jest.mock("../../db")
 jest.mock("../../utilities/usageQuota")
-jest.mock("../../environment", () => ({
-  isTest: () => true,
-  isProd: () => false,
-  isDev: () => true,
-  _set: () => {},
-}))
 jest.mock("@budibase/backend-core/tenancy", () => ({
   getTenantId: () => "testing123"
 }))
@@ -32,6 +26,7 @@ class TestConfiguration {
         url: "/applications"
       }
     }
+    usageQuota.useQuotas = () => true
   }
 
   executeMiddleware() {
@@ -113,12 +108,10 @@ describe("usageQuota middleware", () => {
 
   it("calculates and persists the correct usage quota for the relevant action", async () => {
     config.setUrl("/rows")
-    config.setProd(true)
 
     await config.executeMiddleware()
 
-    // expect(usageQuota.update).toHaveBeenCalledWith("rows", 1)
-    expect(usageQuota.update).not.toHaveBeenCalledWith("rows", 1)
+    expect(usageQuota.update).toHaveBeenCalledWith("rows", 1)
     expect(config.next).toHaveBeenCalled()
   })
 
diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js
index bce13816d1..fb2e0722a7 100644
--- a/packages/server/src/middleware/usageQuota.js
+++ b/packages/server/src/middleware/usageQuota.js
@@ -1,18 +1,11 @@
 const CouchDB = require("../db")
 const usageQuota = require("../utilities/usageQuota")
-const env = require("../environment")
-const { getTenantId } = require("@budibase/backend-core/tenancy")
 const {
   isExternalTable,
   isRowId: isExternalRowId,
 } = require("../integrations/utils")
 const quotaMigration = require("../migrations/sync_app_and_reset_rows_quotas")
 
-const testing = false
-
-// tenants without limits
-const EXCLUDED_TENANTS = ["bb", "default", "bbtest", "bbstaging"]
-
 // currently only counting new writes and deletes
 const METHOD_MAP = {
   POST: 1,
@@ -20,7 +13,7 @@ const METHOD_MAP = {
 }
 
 const DOMAIN_MAP = {
-  // rows: usageQuota.Properties.ROW,                 // works - disabled
+  rows: usageQuota.Properties.ROW,
   // upload: usageQuota.Properties.UPLOAD,            // doesn't work yet
   // views: usageQuota.Properties.VIEW,               // doesn't work yet
   // users: usageQuota.Properties.USER,               // doesn't work yet
@@ -39,13 +32,7 @@ function getProperty(url) {
 }
 
 module.exports = async (ctx, next) => {
-  const tenantId = getTenantId()
-
-  // if in development or a self hosted cloud usage quotas should not be executed
-  if (
-    (env.isDev() || env.SELF_HOSTED || EXCLUDED_TENANTS.includes(tenantId)) &&
-    !testing
-  ) {
+  if (!usageQuota.useQuotas()) {
     return next()
   }
 
diff --git a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
index 23651ec9b7..61e6385aa6 100644
--- a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
+++ b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
@@ -6,25 +6,80 @@ const {
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 const { getAllApps } = require("@budibase/backend-core/db")
 const CouchDB = require("../db")
-const { getUsageQuotaDoc } = require("../utilities/usageQuota")
+const { getUsageQuotaDoc, useQuotas } = require("../utilities/usageQuota")
+const { getRowParams } = require("../db/utils")
+
+/**
+ * Get all rows in the given app ids.
+ *
+ * The returned rows may contan duplicates if there
+ * is a production and dev app.
+ */
+const getAllRows = async appIds => {
+  const allRows = []
+  let appDb
+  for (let appId of appIds) {
+    try {
+      appDb = new CouchDB(appId)
+      const response = await appDb.allDocs(
+        getRowParams(null, null, {
+          include_docs: false,
+        })
+      )
+      allRows.push(...response.rows.map(r => r.id))
+    } catch (e) {
+      // don't error out if we can't count the app rows, just continue
+    }
+  }
+
+  return allRows
+}
+
+/**
+ * Get all rows in the given app ids.
+ *
+ * The returned rows will be unique, duplicated rows across
+ * production and dev apps will be removed.
+ */
+const getUniqueRows = async appIds => {
+  const allRows = await getAllRows(appIds)
+  return new Set(allRows)
+}
+
+const syncRowsQuota = async db => {
+  // get all rows in all apps
+  const allApps = await getAllApps(CouchDB, { all: true })
+  const appIds = allApps ? allApps.map(app => app.appId) : []
+  const rows = await getUniqueRows(appIds)
+
+  // sync row count
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.rows = rows.size
+  await db.put(usageDoc)
+}
+
+const syncAppsQuota = async db => {
+  // get app count
+  const devApps = await getAllApps(CouchDB, { dev: true })
+  const appCount = devApps ? devApps.length : 0
+
+  // sync app count
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.apps = appCount
+  await db.put(usageDoc)
+}
 
 exports.runIfRequired = async () => {
   await migrateIfRequired(
     MIGRATION_DBS.GLOBAL_DB,
     MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS,
     async () => {
+      if (!useQuotas()) {
+        return
+      }
       const db = getGlobalDB()
-      const usageDoc = await getUsageQuotaDoc(db)
-
-      // reset the rows
-      usageDoc.usageQuota.rows = 0
-
-      // sync the apps
-      const apps = await getAllApps(CouchDB, { dev: true })
-      const appCount = apps ? apps.length : 0
-      usageDoc.usageQuota.apps = appCount
-
-      await db.put(usageDoc)
+      await syncAppsQuota(db)
+      await syncRowsQuota(db)
     }
   )
 }
diff --git a/packages/server/src/utilities/tests/usageQuota.spec.js b/packages/server/src/utilities/tests/usageQuota.spec.js
new file mode 100644
index 0000000000..6023b08fbe
--- /dev/null
+++ b/packages/server/src/utilities/tests/usageQuota.spec.js
@@ -0,0 +1,72 @@
+const getTenantId = jest.fn()
+jest.mock("@budibase/backend-core/tenancy", () => ({
+  getTenantId
+}))
+const usageQuota = require("../usageQuota")
+const env = require("../../environment")
+
+class TestConfiguration {
+  constructor() {
+    this.enableQuotas()
+  }
+
+  enableQuotas = () => {
+    env.USE_QUOTAS = 1
+  }
+
+  disableQuotas = () => {
+    env.USE_QUOTAS = null
+  }
+
+  setTenantId = (tenantId) => {
+    getTenantId.mockReturnValue(tenantId)
+  } 
+
+  setExcludedTenants = (tenants) => {
+    env.EXCLUDE_QUOTAS_TENANTS = tenants
+  }
+
+  reset = () => {
+    this.disableQuotas()
+    this.setExcludedTenants(null)
+  }
+}
+
+describe("usageQuota", () => {
+  let config
+
+  beforeEach(() => {
+    config = new TestConfiguration()
+  })
+
+  afterEach(() => {
+    config.reset()
+  })
+
+  describe("useQuotas", () => {
+    it("works when no settings have been provided", () => {
+      config.reset()
+      expect(usageQuota.useQuotas()).toBe(false)
+    })
+    it("honours USE_QUOTAS setting", () => {
+      config.disableQuotas()
+      expect(usageQuota.useQuotas()).toBe(false)
+
+      config.enableQuotas()
+      expect(usageQuota.useQuotas()).toBe(true)
+    })
+    it("honours EXCLUDE_QUOTAS_TENANTS setting", () => {
+      config.setTenantId("test")
+
+      // tenantId is in the list
+      config.setExcludedTenants("test, test2, test2")
+      expect(usageQuota.useQuotas()).toBe(false)
+      config.setExcludedTenants("test,test2,test2")
+      expect(usageQuota.useQuotas()).toBe(false)
+
+      // tenantId is not in the list
+      config.setTenantId("other")
+      expect(usageQuota.useQuotas()).toBe(true)
+    })
+  })
+})
\ No newline at end of file
diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js
index 4d88b610e0..5965334205 100644
--- a/packages/server/src/utilities/usageQuota.js
+++ b/packages/server/src/utilities/usageQuota.js
@@ -1,12 +1,31 @@
 const env = require("../environment")
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
 const {
   StaticDatabases,
   generateNewUsageQuotaDoc,
 } = require("@budibase/backend-core/db")
 
+exports.useQuotas = () => {
+  // check if quotas are enabled
+  if (env.USE_QUOTAS) {
+    // check if there are any tenants without limits
+    if (env.EXCLUDE_QUOTAS_TENANTS) {
+      const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace(
+        /\s/g,
+        ""
+      ).split(",")
+      const tenantId = getTenantId()
+      if (excludedTenants.includes(tenantId)) {
+        return false
+      }
+    }
+    return true
+  }
+  return false
+}
+
 exports.Properties = {
-  ROW: "rows", // mostly works - disabled - app / table deletion not yet accounted for
+  ROW: "rows", // mostly works - app / table deletion not yet accounted for
   UPLOAD: "storage", // doesn't work yet
   VIEW: "views", // doesn't work yet
   USER: "users", // doesn't work yet
@@ -37,7 +56,7 @@ exports.getUsageQuotaDoc = async db => {
  * also been reset after this call.
  */
 exports.update = async (property, usage) => {
-  if (!env.USE_QUOTAS) {
+  if (!exports.useQuotas()) {
     return
   }
 

From ff887f8f88fd1e31363bead75eb968abc3d5c766 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Mon, 17 Jan 2022 18:07:26 +0000
Subject: [PATCH 06/12] Sync row usage with app deletion

---
 packages/server/src/middleware/usageQuota.js  | 77 ++++++++++++++++++-
 .../sync_app_and_reset_rows_quotas.js         | 39 +---------
 .../tests/{ => usageQuota}/usageQuota.spec.js |  2 +-
 .../{usageQuota.js => usageQuota/index.js}    | 18 ++++-
 .../server/src/utilities/usageQuota/rows.js   | 52 +++++++++++++
 .../{ => usageQuota}/usageQuoteReset.js       |  0
 6 files changed, 143 insertions(+), 45 deletions(-)
 rename packages/server/src/utilities/tests/{ => usageQuota}/usageQuota.spec.js (97%)
 rename packages/server/src/utilities/{usageQuota.js => usageQuota/index.js} (81%)
 create mode 100644 packages/server/src/utilities/usageQuota/rows.js
 rename packages/server/src/utilities/{ => usageQuota}/usageQuoteReset.js (100%)

diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js
index fb2e0722a7..2d55aac61d 100644
--- a/packages/server/src/middleware/usageQuota.js
+++ b/packages/server/src/middleware/usageQuota.js
@@ -1,5 +1,6 @@
 const CouchDB = require("../db")
 const usageQuota = require("../utilities/usageQuota")
+const { getUniqueRows } = require("../utilities/usageQuota/rows")
 const {
   isExternalTable,
   isRowId: isExternalRowId,
@@ -74,9 +75,81 @@ module.exports = async (ctx, next) => {
   }
   try {
     await quotaMigration.runIfRequired()
-    await usageQuota.update(property, usage)
-    return next()
+    await performRequest(ctx, next, property, usage)
   } catch (err) {
     ctx.throw(400, err)
   }
 }
+
+const performRequest = async (ctx, next, property, usage) => {
+  const usageContext = {
+    skipNext: false,
+    skipUsage: false,
+    [usageQuota.Properties.APPS]: {},
+  }
+
+  if (usage === -1) {
+    if (PRE_DELETE[property]) {
+      await PRE_DELETE[property](ctx, usageContext)
+    }
+  } else {
+    if (PRE_CREATE[property]) {
+      await PRE_CREATE[property](ctx, usageContext)
+    }
+  }
+
+  // run the request
+  if (!usageContext.skipNext) {
+    await usageQuota.update(property, usage, { dryRun: true })
+    await next()
+  }
+
+  if (usage === -1) {
+    if (POST_DELETE[property]) {
+      await POST_DELETE[property](ctx, usageContext)
+    }
+  } else {
+    if (POST_CREATE[property]) {
+      await POST_CREATE[property](ctx, usageContext)
+    }
+  }
+
+  // update the usage
+  if (!usageContext.skipUsage) {
+    await usageQuota.update(property, usage)
+  }
+}
+
+const appPreDelete = async (ctx, usageContext) => {
+  if (ctx.query.unpublish) {
+    // don't run usage decrement for unpublish
+    usageContext.skipUsage = true
+    return
+  }
+
+  // store the row count to delete
+  const rows = await getUniqueRows([ctx.appId])
+  if (rows.size) {
+    usageContext[usageQuota.Properties.APPS] = { rowCount: rows.size }
+  }
+}
+
+const appPostDelete = async (ctx, usageContext) => {
+  // delete the app rows from usage
+  const rowCount = usageContext[usageQuota.Properties.APPS].rowCount
+  if (rowCount) {
+    await usageQuota.update(usageQuota.Properties.ROW, -rowCount)
+  }
+}
+
+const PRE_DELETE = {
+  [usageQuota.Properties.APPS]: appPreDelete,
+}
+
+const POST_DELETE = {
+  [usageQuota.Properties.APPS]: appPostDelete,
+}
+
+const PRE_CREATE = {}
+
+const POST_CREATE = {}
diff --git a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
index 61e6385aa6..5445919f26 100644
--- a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
+++ b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
@@ -7,44 +7,7 @@ const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 const { getAllApps } = require("@budibase/backend-core/db")
 const CouchDB = require("../db")
 const { getUsageQuotaDoc, useQuotas } = require("../utilities/usageQuota")
-const { getRowParams } = require("../db/utils")
-
-/**
- * Get all rows in the given app ids.
- *
- * The returned rows may contan duplicates if there
- * is a production and dev app.
- */
-const getAllRows = async appIds => {
-  const allRows = []
-  let appDb
-  for (let appId of appIds) {
-    try {
-      appDb = new CouchDB(appId)
-      const response = await appDb.allDocs(
-        getRowParams(null, null, {
-          include_docs: false,
-        })
-      )
-      allRows.push(...response.rows.map(r => r.id))
-    } catch (e) {
-      // don't error out if we can't count the app rows, just continue
-    }
-  }
-
-  return allRows
-}
-
-/**
- * Get all rows in the given app ids.
- *
- * The returned rows will be unique, duplicated rows across
- * production and dev apps will be removed.
- */
-const getUniqueRows = async appIds => {
-  const allRows = await getAllRows(appIds)
-  return new Set(allRows)
-}
+const { getUniqueRows } = require("../utilities/usageQuota/rows")
 
 const syncRowsQuota = async db => {
   // get all rows in all apps
diff --git a/packages/server/src/utilities/tests/usageQuota.spec.js b/packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js
similarity index 97%
rename from packages/server/src/utilities/tests/usageQuota.spec.js
rename to packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js
index 6023b08fbe..764d509b5e 100644
--- a/packages/server/src/utilities/tests/usageQuota.spec.js
+++ b/packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js
@@ -3,7 +3,7 @@ jest.mock("@budibase/backend-core/tenancy", () => ({
   getTenantId
 }))
 const usageQuota = require("../usageQuota")
-const env = require("../../environment")
+const env = require("../../../environment")
 
 class TestConfiguration {
   constructor() {
diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota/index.js
similarity index 81%
rename from packages/server/src/utilities/usageQuota.js
rename to packages/server/src/utilities/usageQuota/index.js
index 5965334205..110f8ef600 100644
--- a/packages/server/src/utilities/usageQuota.js
+++ b/packages/server/src/utilities/usageQuota/index.js
@@ -1,4 +1,4 @@
-const env = require("../environment")
+const env = require("../../environment")
 const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
 const {
   StaticDatabases,
@@ -55,7 +55,7 @@ exports.getUsageQuotaDoc = async db => {
  * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
  * also been reset after this call.
  */
-exports.update = async (property, usage) => {
+exports.update = async (property, usage, opts = { dryRun: false }) => {
   if (!exports.useQuotas()) {
     return
   }
@@ -67,14 +67,24 @@ exports.update = async (property, usage) => {
     // increment the quota
     quota.usageQuota[property] += usage
 
-    if (quota.usageQuota[property] > quota.usageLimits[property]) {
+    if (
+      quota.usageQuota[property] > quota.usageLimits[property] &&
+      usage > 0 // allow for decrementing usage when the quota is already exceeded
+    ) {
       throw new Error(
         `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
       )
     }
 
+    if (quota.usageQuota[property] < 0) {
+      // never go negative if the quota has previously been exceeded
+      quota.usageQuota[property] = 0
+    }
+
     // update the usage quotas
-    await db.put(quota)
+    if (!opts.dryRun) {
+      await db.put(quota)
+    }
   } catch (err) {
     console.error(`Error updating usage quotas for ${property}`, err)
     throw err
diff --git a/packages/server/src/utilities/usageQuota/rows.js b/packages/server/src/utilities/usageQuota/rows.js
new file mode 100644
index 0000000000..3a5602958b
--- /dev/null
+++ b/packages/server/src/utilities/usageQuota/rows.js
@@ -0,0 +1,52 @@
+const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils")
+const CouchDB = require("../../db")
+
+const ROW_EXCLUSIONS = [USER_METDATA_PREFIX]
+
+/**
+ * Get all rows in the given app ids.
+ *
+ * The returned rows may contan duplicates if there
+ * is a production and dev app.
+ */
+const getAllRows = async appIds => {
+  const allRows = []
+  let appDb
+  for (let appId of appIds) {
+    try {
+      appDb = new CouchDB(appId)
+      const response = await appDb.allDocs(
+        getRowParams(null, null, {
+          include_docs: false,
+        })
+      )
+      allRows.push(
+        ...response.rows
+          .map(r => r.id)
+          .filter(id => {
+            for (let exclusion of ROW_EXCLUSIONS) {
+              if (id.startsWith(exclusion)) {
+                return false
+              }
+            }
+            return true
+          })
+      )
+    } catch (e) {
+      // don't error out if we can't count the app rows, just continue
+    }
+  }
+
+  return allRows
+}
+
+/**
+ * Get all rows in the given app ids.
+ *
+ * The returned rows will be unique, duplicated rows across
+ * production and dev apps will be removed.
+ */
+exports.getUniqueRows = async appIds => {
+  const allRows = await getAllRows(appIds)
+  return new Set(allRows)
+}
diff --git a/packages/server/src/utilities/usageQuoteReset.js b/packages/server/src/utilities/usageQuota/usageQuoteReset.js
similarity index 100%
rename from packages/server/src/utilities/usageQuoteReset.js
rename to packages/server/src/utilities/usageQuota/usageQuoteReset.js

From 4d61e84c2b4e9fe05bfcf85118271e274692d579 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Tue, 18 Jan 2022 11:57:20 +0000
Subject: [PATCH 07/12] Handle table deletion and data import

---
 .../modals/CreateTableModal.svelte            | 25 ++++++++++++-------
 .../src/api/controllers/table/internal.js     |  2 ++
 .../server/src/api/controllers/table/utils.js |  5 ++++
 3 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
index 997864e165..a3b7ca81a6 100644
--- a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
@@ -53,16 +53,23 @@
     }
 
     // Create table
-    const table = await tables.save(newTable)
-    notifications.success(`Table ${name} created successfully.`)
-    analytics.captureEvent(Events.TABLE.CREATED, { name })
+    let table
+    try {
+      table = await tables.save(newTable)
+      notifications.success(`Table ${name} created successfully.`)
+      analytics.captureEvent(Events.TABLE.CREATED, { name })
 
-    // Navigate to new table
-    const currentUrl = $url()
-    const path = currentUrl.endsWith("data")
-      ? `./table/${table._id}`
-      : `../../table/${table._id}`
-    $goto(path)
+      // Navigate to new table
+      const currentUrl = $url()
+      const path = currentUrl.endsWith("data")
+        ? `./table/${table._id}`
+        : `../../table/${table._id}`
+      $goto(path)
+    } catch (e) {
+      notifications.error(e)
+      // reload in case the table was created
+      await tables.fetch()
+    }
   }
 </script>
 
diff --git a/packages/server/src/api/controllers/table/internal.js b/packages/server/src/api/controllers/table/internal.js
index 10a5c9746a..9f09e78219 100644
--- a/packages/server/src/api/controllers/table/internal.js
+++ b/packages/server/src/api/controllers/table/internal.js
@@ -8,6 +8,7 @@ const {
   getTable,
   handleDataImport,
 } = require("./utils")
+const usageQuota = require("../../../utilities/usageQuota")
 
 exports.save = async function (ctx) {
   const appId = ctx.appId
@@ -119,6 +120,7 @@ exports.destroy = async function (ctx) {
     })
   )
   await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
+  await usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length)
 
   // update linked rows
   await linkRows.updateLinks({
diff --git a/packages/server/src/api/controllers/table/utils.js b/packages/server/src/api/controllers/table/utils.js
index e4086e8071..e0311389e6 100644
--- a/packages/server/src/api/controllers/table/utils.js
+++ b/packages/server/src/api/controllers/table/utils.js
@@ -15,6 +15,7 @@ const {
 } = require("../../../integrations/utils")
 const { getViews, saveView } = require("../view/utils")
 const viewTemplate = require("../view/viewBuilder")
+const usageQuota = require("../../../utilities/usageQuota")
 
 exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
   let updatedRows = []
@@ -111,7 +112,11 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
     finalData.push(row)
   }
 
+  await usageQuota.update(usageQuota.Properties.ROW, finalData.length, {
+    dryRun: true,
+  })
   await db.bulkDocs(finalData)
+  await usageQuota.update(usageQuota.Properties.ROW, finalData.length)
   let response = await db.put(table)
   table._rev = response._rev
   return table

From 79c10fba7cf7a5d7952c9248ad07e61186e97650 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Tue, 18 Jan 2022 12:48:54 +0000
Subject: [PATCH 08/12] Account for duplicate row ids caused by app import

---
 packages/server/src/middleware/usageQuota.js  |  4 +-
 .../sync_app_and_reset_rows_quotas.js         |  2 +-
 .../server/src/utilities/usageQuota/rows.js   | 95 +++++++++++--------
 3 files changed, 61 insertions(+), 40 deletions(-)

diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js
index 2d55aac61d..c42442affe 100644
--- a/packages/server/src/middleware/usageQuota.js
+++ b/packages/server/src/middleware/usageQuota.js
@@ -129,8 +129,8 @@ const appPreDelete = async (ctx, usageContext) => {
 
   // store the row count to delete
   const rows = await getUniqueRows([ctx.appId])
-  if (rows.size) {
-    usageContext[usageQuota.Properties.APPS] = { rowCount: rows.size }
+  if (rows.length) {
+    usageContext[usageQuota.Properties.APPS] = { rowCount: rows.length }
   }
 }
 
diff --git a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
index 5445919f26..4ab2e413f3 100644
--- a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
+++ b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
@@ -17,7 +17,7 @@ const syncRowsQuota = async db => {
 
   // sync row count
   const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.rows = rows.size
+  usageDoc.usageQuota.rows = rows.length
   await db.put(usageDoc)
 }
 
diff --git a/packages/server/src/utilities/usageQuota/rows.js b/packages/server/src/utilities/usageQuota/rows.js
index 3a5602958b..19c3e5b8bf 100644
--- a/packages/server/src/utilities/usageQuota/rows.js
+++ b/packages/server/src/utilities/usageQuota/rows.js
@@ -1,52 +1,73 @@
 const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils")
 const CouchDB = require("../../db")
+const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db")
 
 const ROW_EXCLUSIONS = [USER_METDATA_PREFIX]
 
-/**
- * Get all rows in the given app ids.
- *
- * The returned rows may contan duplicates if there
- * is a production and dev app.
- */
-const getAllRows = async appIds => {
-  const allRows = []
-  let appDb
+const getAppPairs = appIds => {
+  // collect the app ids into dev / prod pairs
+  // keyed by the dev app id
+  const pairs = {}
   for (let appId of appIds) {
-    try {
-      appDb = new CouchDB(appId)
-      const response = await appDb.allDocs(
-        getRowParams(null, null, {
-          include_docs: false,
-        })
-      )
-      allRows.push(
-        ...response.rows
-          .map(r => r.id)
-          .filter(id => {
-            for (let exclusion of ROW_EXCLUSIONS) {
-              if (id.startsWith(exclusion)) {
-                return false
-              }
-            }
-            return true
-          })
-      )
-    } catch (e) {
-      // don't error out if we can't count the app rows, just continue
+    const devId = getDevelopmentAppID(appId)
+    if (!pairs[devId]) {
+      pairs[devId] = {}
+    }
+    if (isDevAppID(appId)) {
+      pairs[devId].devId = appId
+    } else {
+      pairs[devId].prodId = appId
     }
   }
+  return pairs
+}
 
-  return allRows
+const getAppRows = async appId => {
+  const appDb = new CouchDB(appId)
+  const response = await appDb.allDocs(
+    getRowParams(null, null, {
+      include_docs: false,
+    })
+  )
+  return response.rows
+    .map(r => r.id)
+    .filter(id => {
+      for (let exclusion of ROW_EXCLUSIONS) {
+        if (id.startsWith(exclusion)) {
+          return false
+        }
+      }
+      return true
+    })
 }
 
 /**
- * Get all rows in the given app ids.
- *
- * The returned rows will be unique, duplicated rows across
- * production and dev apps will be removed.
+ * Return a set of all rows in the given app ids.
+ * The returned rows will be unique on a per dev/prod app basis.
+ * Rows duplicates may exist across apps due to data import so they are not filtered out.
  */
 exports.getUniqueRows = async appIds => {
-  const allRows = await getAllRows(appIds)
-  return new Set(allRows)
+  let uniqueRows = []
+  const pairs = getAppPairs(appIds)
+
+  for (let pair of Object.values(pairs)) {
+    let appRows = []
+    for (let appId of [pair.devId, pair.prodId]) {
+      if (!appId) {
+        continue
+      }
+      try {
+        appRows.push(await getAppRows(appId))
+      } catch (e) {
+        // don't error out if we can't count the app rows, just continue
+      }
+    }
+
+    // ensure uniqueness on a per app pair basis
+    // this can't be done on all rows because app import results in
+    // duplicate row ids across apps
+    uniqueRows = uniqueRows.concat(...new Set(appRows))
+  }
+
+  return uniqueRows
 }

From 439fa9a010f7e1c9b5276f3c1cb628f7f0f7aae1 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Tue, 18 Jan 2022 14:43:24 +0000
Subject: [PATCH 09/12] Refactor to allow for future quota migrations

---
 packages/backend-core/src/migrations/index.js |  4 +-
 packages/server/src/middleware/usageQuota.js  | 15 ++++--
 .../sync_app_and_reset_rows_quotas.js         | 48 -------------------
 .../tests/usageQuotas/index.spec.js           | 27 +++++++++++
 .../syncApps.spec.js}                         | 20 ++++----
 .../tests/usageQuotas/syncRows.spec.js        | 43 +++++++++++++++++
 .../src/migrations/usageQuotas/index.js       | 24 ++++++++++
 .../src/migrations/usageQuotas/syncApps.js    | 16 +++++++
 .../src/migrations/usageQuotas/syncRows.js    | 18 +++++++
 9 files changed, 151 insertions(+), 64 deletions(-)
 delete mode 100644 packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
 create mode 100644 packages/server/src/migrations/tests/usageQuotas/index.spec.js
 rename packages/server/src/migrations/tests/{sync_app_and_reset_rows_quotas.spec.js => usageQuotas/syncApps.spec.js} (59%)
 create mode 100644 packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js
 create mode 100644 packages/server/src/migrations/usageQuotas/index.js
 create mode 100644 packages/server/src/migrations/usageQuotas/syncApps.js
 create mode 100644 packages/server/src/migrations/usageQuotas/syncRows.js

diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js
index 30e04e198b..4708776ec3 100644
--- a/packages/backend-core/src/migrations/index.js
+++ b/packages/backend-core/src/migrations/index.js
@@ -7,13 +7,13 @@ exports.MIGRATION_DBS = {
 
 exports.MIGRATIONS = {
   USER_EMAIL_VIEW_CASING: "user_email_view_casing",
-  SYNC_APP_AND_RESET_ROWS_QUOTAS: "sync_app_and_reset_rows_quotas",
+  QUOTAS_1: "quotas_1",
 }
 
 const DB_LOOKUP = {
   [exports.MIGRATION_DBS.GLOBAL_DB]: [
     exports.MIGRATIONS.USER_EMAIL_VIEW_CASING,
-    exports.MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS,
+    exports.MIGRATIONS.QUOTAS_1,
   ],
 }
 
diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js
index c42442affe..8e3daa6268 100644
--- a/packages/server/src/middleware/usageQuota.js
+++ b/packages/server/src/middleware/usageQuota.js
@@ -5,7 +5,7 @@ const {
   isExternalTable,
   isRowId: isExternalRowId,
 } = require("../integrations/utils")
-const quotaMigration = require("../migrations/sync_app_and_reset_rows_quotas")
+const migration = require("../migrations/usageQuotas")
 
 // currently only counting new writes and deletes
 const METHOD_MAP = {
@@ -74,7 +74,7 @@ module.exports = async (ctx, next) => {
     usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
   try {
-    await quotaMigration.runIfRequired()
+    await migration.run()
     await performRequest(ctx, next, property, usage)
   } catch (err) {
     ctx.throw(400, err)
@@ -142,6 +142,13 @@ const appPostDelete = async (ctx, usageContext) => {
   }
 }
 
+// const appPostCreate = async (ctx, usageContext) => {
+// if (ctx.request) {
+//   const rowCount = await getUniqueRows([ctx.appId]).length
+//   await usageQuota.update(usageQuota.Properties.ROW, -rowCount)
+// }
+// }
+
 const PRE_DELETE = {
   [usageQuota.Properties.APPS]: appPreDelete,
 }
@@ -152,4 +159,6 @@ const POST_DELETE = {
 
 const PRE_CREATE = {}
 
-const POST_CREATE = {}
+const POST_CREATE = {
+  // [usageQuota.Properties.APPS]: appPostCreate,
+}
diff --git a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
deleted file mode 100644
index 4ab2e413f3..0000000000
--- a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
+++ /dev/null
@@ -1,48 +0,0 @@
-const {
-  MIGRATIONS,
-  MIGRATION_DBS,
-  migrateIfRequired,
-} = require("@budibase/backend-core/migrations")
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const { getAllApps } = require("@budibase/backend-core/db")
-const CouchDB = require("../db")
-const { getUsageQuotaDoc, useQuotas } = require("../utilities/usageQuota")
-const { getUniqueRows } = require("../utilities/usageQuota/rows")
-
-const syncRowsQuota = async db => {
-  // get all rows in all apps
-  const allApps = await getAllApps(CouchDB, { all: true })
-  const appIds = allApps ? allApps.map(app => app.appId) : []
-  const rows = await getUniqueRows(appIds)
-
-  // sync row count
-  const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.rows = rows.length
-  await db.put(usageDoc)
-}
-
-const syncAppsQuota = async db => {
-  // get app count
-  const devApps = await getAllApps(CouchDB, { dev: true })
-  const appCount = devApps ? devApps.length : 0
-
-  // sync app count
-  const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.apps = appCount
-  await db.put(usageDoc)
-}
-
-exports.runIfRequired = async () => {
-  await migrateIfRequired(
-    MIGRATION_DBS.GLOBAL_DB,
-    MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS,
-    async () => {
-      if (!useQuotas()) {
-        return
-      }
-      const db = getGlobalDB()
-      await syncAppsQuota(db)
-      await syncRowsQuota(db)
-    }
-  )
-}
diff --git a/packages/server/src/migrations/tests/usageQuotas/index.spec.js b/packages/server/src/migrations/tests/usageQuotas/index.spec.js
new file mode 100644
index 0000000000..0c5b982909
--- /dev/null
+++ b/packages/server/src/migrations/tests/usageQuotas/index.spec.js
@@ -0,0 +1,27 @@
+const env = require("../../../environment")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+
+const syncApps = jest.fn()
+const syncRows = jest.fn()
+
+jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) )
+jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) )
+
+const migrations = require("../../usageQuotas")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs the required migrations", async () => {
+    await migrations.run()
+    expect(syncApps).toHaveBeenCalledTimes(1)
+    expect(syncRows).toHaveBeenCalledTimes(1)
+  })
+})
diff --git a/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js b/packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js
similarity index 59%
rename from packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js
rename to packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js
index cd533e3f06..160319a31b 100644
--- a/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js
+++ b/packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js
@@ -1,10 +1,10 @@
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const TestConfig = require("../../tests/utilities/TestConfiguration")
-const { getUsageQuotaDoc, update, Properties } = require("../../utilities/usageQuota")
-const { runIfRequired } = require("../sync_app_and_reset_rows_quotas")
-const env = require("../../environment")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
+const syncApps = require("../../usageQuotas/syncApps")
+const env = require("../../../environment")
 
-describe("Sync App And Reset Rows Quotas Migration", () => {
+describe("syncApps", () => {
   let config = new TestConfig(false)
 
   beforeEach(async () => {
@@ -12,28 +12,26 @@ describe("Sync App And Reset Rows Quotas Migration", () => {
     env._set("USE_QUOTAS", 1)
   })
 
-  afterAll(config.end)
+  afterAll(config.end)  
 
-  it("migrates successfully", async () => {
+  it("runs successfully", async () => {
     // create the usage quota doc and mock usages
     const db = getGlobalDB()
     await getUsageQuotaDoc(db)
     await update(Properties.APPS, 3)
-    await update(Properties.ROW, 300)
 
     let usageDoc = await getUsageQuotaDoc(db)
     expect(usageDoc.usageQuota.apps).toEqual(3)
-    expect(usageDoc.usageQuota.rows).toEqual(300)
 
     // create an extra app to test the migration
     await config.createApp("quota-test")
 
     // migrate
-    await runIfRequired()
+    await syncApps.run()
 
     // assert the migration worked 
     usageDoc = await getUsageQuotaDoc(db)
     expect(usageDoc.usageQuota.apps).toEqual(2)
-    expect(usageDoc.usageQuota.rows).toEqual(0)
   })
 })
+
diff --git a/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js b/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js
new file mode 100644
index 0000000000..a09bea60bd
--- /dev/null
+++ b/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js
@@ -0,0 +1,43 @@
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
+const syncRows = require("../../usageQuotas/syncRows")
+const env = require("../../../environment")
+
+describe("syncRows", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    // create the usage quota doc and mock usages
+    const db = getGlobalDB()
+    await getUsageQuotaDoc(db)
+    await update(Properties.ROW, 300)
+
+    let usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.rows).toEqual(300)
+
+    // app 1
+    await config.createTable()
+    await config.createRow()
+    // app 2
+    await config.createApp()
+    await config.createTable()
+    await config.createRow()
+    await config.createRow()
+    
+    // migrate
+    await syncRows.run()
+
+    // assert the migration worked 
+    usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.rows).toEqual(3)
+  })
+})
+
diff --git a/packages/server/src/migrations/usageQuotas/index.js b/packages/server/src/migrations/usageQuotas/index.js
new file mode 100644
index 0000000000..39744093c2
--- /dev/null
+++ b/packages/server/src/migrations/usageQuotas/index.js
@@ -0,0 +1,24 @@
+const {
+  MIGRATIONS,
+  MIGRATION_DBS,
+  migrateIfRequired,
+} = require("@budibase/backend-core/migrations")
+const { useQuotas } = require("../../utilities/usageQuota")
+const syncApps = require("./syncApps")
+const syncRows = require("./syncRows")
+
+exports.run = async () => {
+  if (!useQuotas()) {
+    return
+  }
+
+  // Jan 2022
+  await migrateIfRequired(
+    MIGRATION_DBS.GLOBAL_DB,
+    MIGRATIONS.QUOTAS_1,
+    async () => {
+      await syncApps.run()
+      await syncRows.run()
+    }
+  )
+}
diff --git a/packages/server/src/migrations/usageQuotas/syncApps.js b/packages/server/src/migrations/usageQuotas/syncApps.js
new file mode 100644
index 0000000000..c285bafe4e
--- /dev/null
+++ b/packages/server/src/migrations/usageQuotas/syncApps.js
@@ -0,0 +1,16 @@
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { getAllApps } = require("@budibase/backend-core/db")
+const CouchDB = require("../../db")
+const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+
+exports.run = async () => {
+  const db = getGlobalDB()
+  // get app count
+  const devApps = await getAllApps(CouchDB, { dev: true })
+  const appCount = devApps ? devApps.length : 0
+
+  // sync app count
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.apps = appCount
+  await db.put(usageDoc)
+}
diff --git a/packages/server/src/migrations/usageQuotas/syncRows.js b/packages/server/src/migrations/usageQuotas/syncRows.js
new file mode 100644
index 0000000000..73f6f1477d
--- /dev/null
+++ b/packages/server/src/migrations/usageQuotas/syncRows.js
@@ -0,0 +1,18 @@
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { getAllApps } = require("@budibase/backend-core/db")
+const CouchDB = require("../../db")
+const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+const { getUniqueRows } = require("../../utilities/usageQuota/rows")
+
+exports.run = async () => {
+  const db = getGlobalDB()
+  // get all rows in all apps
+  const allApps = await getAllApps(CouchDB, { all: true })
+  const appIds = allApps ? allApps.map(app => app.appId) : []
+  const rows = await getUniqueRows(appIds)
+
+  // sync row count
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.rows = rows.length
+  await db.put(usageDoc)
+}

From fc043e116d0d56c7b1e175aa94097207c183fcf4 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Tue, 18 Jan 2022 15:37:16 +0000
Subject: [PATCH 10/12] Support template and app import in quotas + logging

---
 packages/backend-core/src/migrations/index.js    | 12 ++++++++----
 packages/server/src/middleware/usageQuota.js     | 16 +++++++++-------
 .../src/migrations/usageQuotas/syncApps.js       |  4 +++-
 .../src/migrations/usageQuotas/syncRows.js       |  7 +++++--
 .../tests/usageQuota/usageQuota.spec.js          |  2 +-
 packages/server/src/utilities/usageQuota/rows.js |  1 +
 6 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js
index 4708776ec3..6b8eb3a95c 100644
--- a/packages/backend-core/src/migrations/index.js
+++ b/packages/backend-core/src/migrations/index.js
@@ -1,5 +1,5 @@
 const { DocumentTypes } = require("../db/constants")
-const { getGlobalDB } = require("../tenancy")
+const { getGlobalDB, getTenantId } = require("../tenancy")
 
 exports.MIGRATION_DBS = {
   GLOBAL_DB: "GLOBAL_DB",
@@ -29,6 +29,7 @@ exports.getMigrationsDoc = async db => {
 }
 
 exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
+  const tenantId = getTenantId()
   try {
     let db
     if (migrationDb === exports.MIGRATION_DBS.GLOBAL_DB) {
@@ -49,15 +50,18 @@ exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
       return
     }
 
-    console.log(`Performing migration: ${migrationName}`)
+    console.log(`[Tenant: ${tenantId}] Performing migration: ${migrationName}`)
     await migrateFn()
-    console.log(`Migration complete: ${migrationName}`)
+    console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`)
 
     // mark as complete
     doc[migrationName] = Date.now()
     await db.put(doc)
   } catch (err) {
-    console.error(`Error performing migration: ${migrationName}: `, err)
+    console.error(
+      `[Tenant: ${tenantId}] Error performing migration: ${migrationName}: `,
+      err
+    )
     throw err
   }
 }
diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js
index 8e3daa6268..4bafa75132 100644
--- a/packages/server/src/middleware/usageQuota.js
+++ b/packages/server/src/middleware/usageQuota.js
@@ -142,12 +142,14 @@ const appPostDelete = async (ctx, usageContext) => {
   }
 }
 
-// const appPostCreate = async (ctx, usageContext) => {
-// if (ctx.request) {
-//   const rowCount = await getUniqueRows([ctx.appId]).length
-//   await usageQuota.update(usageQuota.Properties.ROW, -rowCount)
-// }
-// }
+const appPostCreate = async ctx => {
+  // app import & template creation
+  if (ctx.request.body.useTemplate === "true") {
+    const rows = await getUniqueRows([ctx.response.body.appId])
+    const rowCount = rows ? rows.length : 0
+    await usageQuota.update(usageQuota.Properties.ROW, rowCount)
+  }
+}
 
 const PRE_DELETE = {
   [usageQuota.Properties.APPS]: appPreDelete,
@@ -160,5 +162,5 @@ const POST_DELETE = {
 const PRE_CREATE = {}
 
 const POST_CREATE = {
-  // [usageQuota.Properties.APPS]: appPostCreate,
+  [usageQuota.Properties.APPS]: appPostCreate,
 }
diff --git a/packages/server/src/migrations/usageQuotas/syncApps.js b/packages/server/src/migrations/usageQuotas/syncApps.js
index c285bafe4e..ee106129e6 100644
--- a/packages/server/src/migrations/usageQuotas/syncApps.js
+++ b/packages/server/src/migrations/usageQuotas/syncApps.js
@@ -1,4 +1,4 @@
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
 const { getAllApps } = require("@budibase/backend-core/db")
 const CouchDB = require("../../db")
 const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
@@ -10,6 +10,8 @@ exports.run = async () => {
   const appCount = devApps ? devApps.length : 0
 
   // sync app count
+  const tenantId = getTenantId()
+  console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
   const usageDoc = await getUsageQuotaDoc(db)
   usageDoc.usageQuota.apps = appCount
   await db.put(usageDoc)
diff --git a/packages/server/src/migrations/usageQuotas/syncRows.js b/packages/server/src/migrations/usageQuotas/syncRows.js
index 73f6f1477d..7990f405de 100644
--- a/packages/server/src/migrations/usageQuotas/syncRows.js
+++ b/packages/server/src/migrations/usageQuotas/syncRows.js
@@ -1,4 +1,4 @@
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
 const { getAllApps } = require("@budibase/backend-core/db")
 const CouchDB = require("../../db")
 const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
@@ -10,9 +10,12 @@ exports.run = async () => {
   const allApps = await getAllApps(CouchDB, { all: true })
   const appIds = allApps ? allApps.map(app => app.appId) : []
   const rows = await getUniqueRows(appIds)
+  const rowCount = rows ? rows.length : 0
 
   // sync row count
+  const tenantId = getTenantId()
+  console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
   const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.rows = rows.length
+  usageDoc.usageQuota.rows = rowCount
   await db.put(usageDoc)
 }
diff --git a/packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js b/packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js
index 764d509b5e..dcd7578f59 100644
--- a/packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js
+++ b/packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js
@@ -2,7 +2,7 @@ const getTenantId = jest.fn()
 jest.mock("@budibase/backend-core/tenancy", () => ({
   getTenantId
 }))
-const usageQuota = require("../usageQuota")
+const usageQuota = require("../../usageQuota")
 const env = require("../../../environment")
 
 class TestConfiguration {
diff --git a/packages/server/src/utilities/usageQuota/rows.js b/packages/server/src/utilities/usageQuota/rows.js
index 19c3e5b8bf..67ad07410d 100644
--- a/packages/server/src/utilities/usageQuota/rows.js
+++ b/packages/server/src/utilities/usageQuota/rows.js
@@ -59,6 +59,7 @@ exports.getUniqueRows = async appIds => {
       try {
         appRows.push(await getAppRows(appId))
       } catch (e) {
+        console.error(e)
         // don't error out if we can't count the app rows, just continue
       }
     }

From 604d316834630601a093ffe1199640870f0ffde7 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Tue, 18 Jan 2022 15:53:55 +0000
Subject: [PATCH 11/12] Add dryRun to automation row create and remove env
 conditional on delete

---
 packages/server/src/automations/steps/createRow.js | 3 ++-
 packages/server/src/automations/steps/deleteRow.js | 5 +----
 packages/server/src/utilities/usageQuota/index.js  | 4 ++--
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/automations/steps/createRow.js b/packages/server/src/automations/steps/createRow.js
index 816cd829ab..1937121062 100644
--- a/packages/server/src/automations/steps/createRow.js
+++ b/packages/server/src/automations/steps/createRow.js
@@ -82,8 +82,9 @@ exports.run = async function ({ inputs, appId, emitter }) {
       inputs.row.tableId,
       inputs.row
     )
-    await usage.update(usage.Properties.ROW, 1)
+    await usage.update(usage.Properties.ROW, 1, { dryRun: true })
     await rowController.save(ctx)
+    await usage.update(usage.Properties.ROW, 1)
     return {
       row: inputs.row,
       response: ctx.body,
diff --git a/packages/server/src/automations/steps/deleteRow.js b/packages/server/src/automations/steps/deleteRow.js
index c7bee577a5..e41e5ad263 100644
--- a/packages/server/src/automations/steps/deleteRow.js
+++ b/packages/server/src/automations/steps/deleteRow.js
@@ -1,5 +1,4 @@
 const rowController = require("../../api/controllers/row")
-const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")
 const { buildCtx } = require("./utils")
 const automationUtils = require("../automationUtils")
@@ -74,9 +73,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
   })
 
   try {
-    if (env.isProd()) {
-      await usage.update(usage.Properties.ROW, -1)
-    }
+    await usage.update(usage.Properties.ROW, -1)
     await rowController.destroy(ctx)
     return {
       response: ctx.body,
diff --git a/packages/server/src/utilities/usageQuota/index.js b/packages/server/src/utilities/usageQuota/index.js
index 110f8ef600..b0ff310aa3 100644
--- a/packages/server/src/utilities/usageQuota/index.js
+++ b/packages/server/src/utilities/usageQuota/index.js
@@ -25,12 +25,12 @@ exports.useQuotas = () => {
 }
 
 exports.Properties = {
-  ROW: "rows", // mostly works - app / table deletion not yet accounted for
+  ROW: "rows",
   UPLOAD: "storage", // doesn't work yet
   VIEW: "views", // doesn't work yet
   USER: "users", // doesn't work yet
   AUTOMATION: "automationRuns", // doesn't work yet
-  APPS: "apps", // works
+  APPS: "apps",
   EMAILS: "emails", // doesn't work yet
 }
 

From 58dc644ad86f2806a6aaf99d26914b54291cb4ee Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Fri, 21 Jan 2022 11:27:13 +0000
Subject: [PATCH 12/12] v1.0.46-alpha.3

---
 lerna.json                             | 2 +-
 packages/backend-core/package.json     | 2 +-
 packages/bbui/package.json             | 2 +-
 packages/builder/package.json          | 8 ++++----
 packages/cli/package.json              | 2 +-
 packages/client/package.json           | 6 +++---
 packages/server/package.json           | 8 ++++----
 packages/string-templates/package.json | 2 +-
 packages/worker/package.json           | 6 +++---
 9 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/lerna.json b/lerna.json
index 5ad1ffdd97..7564ea387c 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "1.0.46-alpha.2",
+  "version": "1.0.46-alpha.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*"
diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index c7e2f41fa8..c4f7dd1ae8 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.0.46-alpha.2",
+  "version": "1.0.46-alpha.3",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "src/index.js",
   "author": "Budibase",
diff --git a/packages/bbui/package.json b/packages/bbui/package.json
index 885e831cba..ecc5bf5a1f 100644
--- a/packages/bbui/package.json
+++ b/packages/bbui/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.46-alpha.2",
+  "version": "1.0.46-alpha.3",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
diff --git a/packages/builder/package.json b/packages/builder/package.json
index c94198316a..474213899e 100644
--- a/packages/builder/package.json
+++ b/packages/builder/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.46-alpha.2",
+  "version": "1.0.46-alpha.3",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.46-alpha.2",
-    "@budibase/client": "^1.0.46-alpha.2",
+    "@budibase/bbui": "^1.0.46-alpha.3",
+    "@budibase/client": "^1.0.46-alpha.3",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^1.0.46-alpha.2",
+    "@budibase/string-templates": "^1.0.46-alpha.3",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
diff --git a/packages/cli/package.json b/packages/cli/package.json
index 4df1e25045..e1df87bfd3 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.46-alpha.2",
+  "version": "1.0.46-alpha.3",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
diff --git a/packages/client/package.json b/packages/client/package.json
index b3eca904ee..5730b4a5e0 100644
--- a/packages/client/package.json
+++ b/packages/client/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.0.46-alpha.2",
+  "version": "1.0.46-alpha.3",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.46-alpha.2",
+    "@budibase/bbui": "^1.0.46-alpha.3",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^1.0.46-alpha.2",
+    "@budibase/string-templates": "^1.0.46-alpha.3",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"
diff --git a/packages/server/package.json b/packages/server/package.json
index 4c67f4edf8..8dd3a339ae 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.46-alpha.2",
+  "version": "1.0.46-alpha.3",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.46-alpha.2",
-    "@budibase/client": "^1.0.46-alpha.2",
-    "@budibase/string-templates": "^1.0.46-alpha.2",
+    "@budibase/backend-core": "^1.0.46-alpha.3",
+    "@budibase/client": "^1.0.46-alpha.3",
+    "@budibase/string-templates": "^1.0.46-alpha.3",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json
index 0dbda68cef..1a808c3f9b 100644
--- a/packages/string-templates/package.json
+++ b/packages/string-templates/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "1.0.46-alpha.2",
+  "version": "1.0.46-alpha.3",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",
diff --git a/packages/worker/package.json b/packages/worker/package.json
index f9bf55dd66..93c33b648c 100644
--- a/packages/worker/package.json
+++ b/packages/worker/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.0.46-alpha.2",
+  "version": "1.0.46-alpha.3",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -29,8 +29,8 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "^1.0.46-alpha.2",
-    "@budibase/string-templates": "^1.0.46-alpha.2",
+    "@budibase/backend-core": "^1.0.46-alpha.3",
+    "@budibase/string-templates": "^1.0.46-alpha.3",
     "@koa/router": "^8.0.0",
     "@sentry/node": "^6.0.0",
     "@techpass/passport-openidconnect": "^0.3.0",