From 8fc60af820f4c23991cb99e42ecd316ffe7404a0 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Mon, 17 Jan 2022 12:44:53 +0000
Subject: [PATCH 1/7] Calculate total rows in migration, centralise quota
 enabled logic and tidy to use env vars only

---
 .../templates/app-service-deployment.yaml     |  4 +-
 charts/budibase/values.yaml                   |  5 +-
 .../server/src/automations/steps/createRow.js |  5 +-
 packages/server/src/environment.js            |  1 +
 packages/server/src/integrations/s3.ts        |  2 +-
 .../src/middleware/tests/usageQuota.spec.js   | 11 +--
 packages/server/src/middleware/usageQuota.js  | 17 +---
 .../sync_app_and_reset_rows_quotas.js         | 79 ++++++++++++++++---
 .../src/utilities/tests/usageQuota.spec.js    | 72 +++++++++++++++++
 packages/server/src/utilities/usageQuota.js   | 25 +++++-
 10 files changed, 175 insertions(+), 46 deletions(-)
 create mode 100644 packages/server/src/utilities/tests/usageQuota.spec.js

diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml
index 8086c0ab20..cd43631992 100644
--- a/charts/budibase/templates/app-service-deployment.yaml
+++ b/charts/budibase/templates/app-service-deployment.yaml
@@ -99,7 +99,9 @@ spec:
         - name: PLATFORM_URL
           value: {{ .Values.globals.platformUrl | quote }}
         - name: USE_QUOTAS
-          value: "1" 
+          value: {{ .Values.globals.useQuotas | quote }}
+        - name: EXCLUDE_QUOTAS_TENANTS
+          value: {{ .Values.globals.excludeQuotasTenants | quote }}
         - name: ACCOUNT_PORTAL_URL
           value: {{ .Values.globals.accountPortalUrl | quote }}
         - name: ACCOUNT_PORTAL_API_KEY
diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml
index 4666d01c70..9ea055c6c0 100644
--- a/charts/budibase/values.yaml
+++ b/charts/budibase/values.yaml
@@ -93,6 +93,8 @@ globals:
   logLevel: info
   selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
   multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs
+  useQuotas: "0"
+  excludeQuotasTenants: "" # comma seperated list of tenants to exclude from quotas
   accountPortalUrl: ""
   accountPortalApiKey: ""
   cookieDomain: ""
@@ -239,7 +241,8 @@ couchdb:
     hosts:
       - chart-example.local
     path: /
-    annotations: []
+    annotations:
+      []
       # kubernetes.io/ingress.class: nginx
       # kubernetes.io/tls-acme: "true"
     tls:
diff --git a/packages/server/src/automations/steps/createRow.js b/packages/server/src/automations/steps/createRow.js
index 8e5b44cc06..816cd829ab 100644
--- a/packages/server/src/automations/steps/createRow.js
+++ b/packages/server/src/automations/steps/createRow.js
@@ -1,6 +1,5 @@
 const rowController = require("../../api/controllers/row")
 const automationUtils = require("../automationUtils")
-const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")
 const { buildCtx } = require("./utils")
 
@@ -83,9 +82,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
       inputs.row.tableId,
       inputs.row
     )
-    if (env.USE_QUOTAS) {
-      await usage.update(usage.Properties.ROW, 1)
-    }
+    await usage.update(usage.Properties.ROW, 1)
     await rowController.save(ctx)
     return {
       row: inputs.row,
diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js
index a92e113851..614f41a29f 100644
--- a/packages/server/src/environment.js
+++ b/packages/server/src/environment.js
@@ -38,6 +38,7 @@ module.exports = {
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   USE_QUOTAS: process.env.USE_QUOTAS,
+  EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS,
   REDIS_URL: process.env.REDIS_URL,
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
diff --git a/packages/server/src/integrations/s3.ts b/packages/server/src/integrations/s3.ts
index 25b439fd58..273f221575 100644
--- a/packages/server/src/integrations/s3.ts
+++ b/packages/server/src/integrations/s3.ts
@@ -38,7 +38,7 @@ module S3Module {
       signatureVersion: {
         type: "string",
         required: false,
-        default: "v4"
+        default: "v4",
       },
     },
     query: {
diff --git a/packages/server/src/middleware/tests/usageQuota.spec.js b/packages/server/src/middleware/tests/usageQuota.spec.js
index 2e92eb9957..1282615a50 100644
--- a/packages/server/src/middleware/tests/usageQuota.spec.js
+++ b/packages/server/src/middleware/tests/usageQuota.spec.js
@@ -1,11 +1,5 @@
 jest.mock("../../db")
 jest.mock("../../utilities/usageQuota")
-jest.mock("../../environment", () => ({
-  isTest: () => true,
-  isProd: () => false,
-  isDev: () => true,
-  _set: () => {},
-}))
 jest.mock("@budibase/backend-core/tenancy", () => ({
   getTenantId: () => "testing123"
 }))
@@ -32,6 +26,7 @@ class TestConfiguration {
         url: "/applications"
       }
     }
+    usageQuota.useQuotas = () => true
   }
 
   executeMiddleware() {
@@ -113,12 +108,10 @@ describe("usageQuota middleware", () => {
 
   it("calculates and persists the correct usage quota for the relevant action", async () => {
     config.setUrl("/rows")
-    config.setProd(true)
 
     await config.executeMiddleware()
 
-    // expect(usageQuota.update).toHaveBeenCalledWith("rows", 1)
-    expect(usageQuota.update).not.toHaveBeenCalledWith("rows", 1)
+    expect(usageQuota.update).toHaveBeenCalledWith("rows", 1)
     expect(config.next).toHaveBeenCalled()
   })
 
diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js
index bce13816d1..fb2e0722a7 100644
--- a/packages/server/src/middleware/usageQuota.js
+++ b/packages/server/src/middleware/usageQuota.js
@@ -1,18 +1,11 @@
 const CouchDB = require("../db")
 const usageQuota = require("../utilities/usageQuota")
-const env = require("../environment")
-const { getTenantId } = require("@budibase/backend-core/tenancy")
 const {
   isExternalTable,
   isRowId: isExternalRowId,
 } = require("../integrations/utils")
 const quotaMigration = require("../migrations/sync_app_and_reset_rows_quotas")
 
-const testing = false
-
-// tenants without limits
-const EXCLUDED_TENANTS = ["bb", "default", "bbtest", "bbstaging"]
-
 // currently only counting new writes and deletes
 const METHOD_MAP = {
   POST: 1,
@@ -20,7 +13,7 @@ const METHOD_MAP = {
 }
 
 const DOMAIN_MAP = {
-  // rows: usageQuota.Properties.ROW,                 // works - disabled
+  rows: usageQuota.Properties.ROW,
   // upload: usageQuota.Properties.UPLOAD,            // doesn't work yet
   // views: usageQuota.Properties.VIEW,               // doesn't work yet
   // users: usageQuota.Properties.USER,               // doesn't work yet
@@ -39,13 +32,7 @@ function getProperty(url) {
 }
 
 module.exports = async (ctx, next) => {
-  const tenantId = getTenantId()
-
-  // if in development or a self hosted cloud usage quotas should not be executed
-  if (
-    (env.isDev() || env.SELF_HOSTED || EXCLUDED_TENANTS.includes(tenantId)) &&
-    !testing
-  ) {
+  if (!usageQuota.useQuotas()) {
     return next()
   }
 
diff --git a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
index 23651ec9b7..61e6385aa6 100644
--- a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
+++ b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
@@ -6,25 +6,80 @@ const {
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 const { getAllApps } = require("@budibase/backend-core/db")
 const CouchDB = require("../db")
-const { getUsageQuotaDoc } = require("../utilities/usageQuota")
+const { getUsageQuotaDoc, useQuotas } = require("../utilities/usageQuota")
+const { getRowParams } = require("../db/utils")
+
+/**
+ * Get all rows in the given app ids.
+ *
+ * The returned rows may contan duplicates if there
+ * is a production and dev app.
+ */
+const getAllRows = async appIds => {
+  const allRows = []
+  let appDb
+  for (let appId of appIds) {
+    try {
+      appDb = new CouchDB(appId)
+      const response = await appDb.allDocs(
+        getRowParams(null, null, {
+          include_docs: false,
+        })
+      )
+      allRows.push(...response.rows.map(r => r.id))
+    } catch (e) {
+      // don't error out if we can't count the app rows, just continue
+    }
+  }
+
+  return allRows
+}
+
+/**
+ * Get all rows in the given app ids.
+ *
+ * The returned rows will be unique, duplicated rows across
+ * production and dev apps will be removed.
+ */
+const getUniqueRows = async appIds => {
+  const allRows = await getAllRows(appIds)
+  return new Set(allRows)
+}
+
+const syncRowsQuota = async db => {
+  // get all rows in all apps
+  const allApps = await getAllApps(CouchDB, { all: true })
+  const appIds = allApps ? allApps.map(app => app.appId) : []
+  const rows = await getUniqueRows(appIds)
+
+  // sync row count
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.rows = rows.size
+  await db.put(usageDoc)
+}
+
+const syncAppsQuota = async db => {
+  // get app count
+  const devApps = await getAllApps(CouchDB, { dev: true })
+  const appCount = devApps ? devApps.length : 0
+
+  // sync app count
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.apps = appCount
+  await db.put(usageDoc)
+}
 
 exports.runIfRequired = async () => {
   await migrateIfRequired(
     MIGRATION_DBS.GLOBAL_DB,
     MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS,
     async () => {
+      if (!useQuotas()) {
+        return
+      }
       const db = getGlobalDB()
-      const usageDoc = await getUsageQuotaDoc(db)
-
-      // reset the rows
-      usageDoc.usageQuota.rows = 0
-
-      // sync the apps
-      const apps = await getAllApps(CouchDB, { dev: true })
-      const appCount = apps ? apps.length : 0
-      usageDoc.usageQuota.apps = appCount
-
-      await db.put(usageDoc)
+      await syncAppsQuota(db)
+      await syncRowsQuota(db)
     }
   )
 }
diff --git a/packages/server/src/utilities/tests/usageQuota.spec.js b/packages/server/src/utilities/tests/usageQuota.spec.js
new file mode 100644
index 0000000000..6023b08fbe
--- /dev/null
+++ b/packages/server/src/utilities/tests/usageQuota.spec.js
@@ -0,0 +1,72 @@
+const getTenantId = jest.fn()
+jest.mock("@budibase/backend-core/tenancy", () => ({
+  getTenantId
+}))
+const usageQuota = require("../usageQuota")
+const env = require("../../environment")
+
+class TestConfiguration {
+  constructor() {
+    this.enableQuotas()
+  }
+
+  enableQuotas = () => {
+    env.USE_QUOTAS = 1
+  }
+
+  disableQuotas = () => {
+    env.USE_QUOTAS = null
+  }
+
+  setTenantId = (tenantId) => {
+    getTenantId.mockReturnValue(tenantId)
+  } 
+
+  setExcludedTenants = (tenants) => {
+    env.EXCLUDE_QUOTAS_TENANTS = tenants
+  }
+
+  reset = () => {
+    this.disableQuotas()
+    this.setExcludedTenants(null)
+  }
+}
+
+describe("usageQuota", () => {
+  let config
+
+  beforeEach(() => {
+    config = new TestConfiguration()
+  })
+
+  afterEach(() => {
+    config.reset()
+  })
+
+  describe("useQuotas", () => {
+    it("works when no settings have been provided", () => {
+      config.reset()
+      expect(usageQuota.useQuotas()).toBe(false)
+    })
+    it("honours USE_QUOTAS setting", () => {
+      config.disableQuotas()
+      expect(usageQuota.useQuotas()).toBe(false)
+
+      config.enableQuotas()
+      expect(usageQuota.useQuotas()).toBe(true)
+    })
+    it("honours EXCLUDE_QUOTAS_TENANTS setting", () => {
+      config.setTenantId("test")
+
+      // tenantId is in the list
+      config.setExcludedTenants("test, test2, test2")
+      expect(usageQuota.useQuotas()).toBe(false)
+      config.setExcludedTenants("test,test2,test2")
+      expect(usageQuota.useQuotas()).toBe(false)
+
+      // tenantId is not in the list
+      config.setTenantId("other")
+      expect(usageQuota.useQuotas()).toBe(true)
+    })
+  })
+})
\ No newline at end of file
diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js
index 4d88b610e0..5965334205 100644
--- a/packages/server/src/utilities/usageQuota.js
+++ b/packages/server/src/utilities/usageQuota.js
@@ -1,12 +1,31 @@
 const env = require("../environment")
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
 const {
   StaticDatabases,
   generateNewUsageQuotaDoc,
 } = require("@budibase/backend-core/db")
 
+exports.useQuotas = () => {
+  // check if quotas are enabled
+  if (env.USE_QUOTAS) {
+    // check if there are any tenants without limits
+    if (env.EXCLUDE_QUOTAS_TENANTS) {
+      const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace(
+        /\s/g,
+        ""
+      ).split(",")
+      const tenantId = getTenantId()
+      if (excludedTenants.includes(tenantId)) {
+        return false
+      }
+    }
+    return true
+  }
+  return false
+}
+
 exports.Properties = {
-  ROW: "rows", // mostly works - disabled - app / table deletion not yet accounted for
+  ROW: "rows", // mostly works - app / table deletion not yet accounted for
   UPLOAD: "storage", // doesn't work yet
   VIEW: "views", // doesn't work yet
   USER: "users", // doesn't work yet
@@ -37,7 +56,7 @@ exports.getUsageQuotaDoc = async db => {
  * also been reset after this call.
  */
 exports.update = async (property, usage) => {
-  if (!env.USE_QUOTAS) {
+  if (!exports.useQuotas()) {
     return
   }
 

From ff887f8f88fd1e31363bead75eb968abc3d5c766 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Mon, 17 Jan 2022 18:07:26 +0000
Subject: [PATCH 2/7] Sync row usage with app deletion

---
 packages/server/src/middleware/usageQuota.js  | 77 ++++++++++++++++++-
 .../sync_app_and_reset_rows_quotas.js         | 39 +---------
 .../tests/{ => usageQuota}/usageQuota.spec.js |  2 +-
 .../{usageQuota.js => usageQuota/index.js}    | 18 ++++-
 .../server/src/utilities/usageQuota/rows.js   | 52 +++++++++++++
 .../{ => usageQuota}/usageQuoteReset.js       |  0
 6 files changed, 143 insertions(+), 45 deletions(-)
 rename packages/server/src/utilities/tests/{ => usageQuota}/usageQuota.spec.js (97%)
 rename packages/server/src/utilities/{usageQuota.js => usageQuota/index.js} (81%)
 create mode 100644 packages/server/src/utilities/usageQuota/rows.js
 rename packages/server/src/utilities/{ => usageQuota}/usageQuoteReset.js (100%)

diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js
index fb2e0722a7..2d55aac61d 100644
--- a/packages/server/src/middleware/usageQuota.js
+++ b/packages/server/src/middleware/usageQuota.js
@@ -1,5 +1,6 @@
 const CouchDB = require("../db")
 const usageQuota = require("../utilities/usageQuota")
+const { getUniqueRows } = require("../utilities/usageQuota/rows")
 const {
   isExternalTable,
   isRowId: isExternalRowId,
@@ -74,9 +75,81 @@ module.exports = async (ctx, next) => {
   }
   try {
     await quotaMigration.runIfRequired()
-    await usageQuota.update(property, usage)
-    return next()
+    await performRequest(ctx, next, property, usage)
   } catch (err) {
     ctx.throw(400, err)
   }
 }
+
+const performRequest = async (ctx, next, property, usage) => {
+  const usageContext = {
+    skipNext: false,
+    skipUsage: false,
+    [usageQuota.Properties.APPS]: {},
+  }
+
+  if (usage === -1) {
+    if (PRE_DELETE[property]) {
+      await PRE_DELETE[property](ctx, usageContext)
+    }
+  } else {
+    if (PRE_CREATE[property]) {
+      await PRE_CREATE[property](ctx, usageContext)
+    }
+  }
+
+  // run the request
+  if (!usageContext.skipNext) {
+    await usageQuota.update(property, usage, { dryRun: true })
+    await next()
+  }
+
+  if (usage === -1) {
+    if (POST_DELETE[property]) {
+      await POST_DELETE[property](ctx, usageContext)
+    }
+  } else {
+    if (POST_CREATE[property]) {
+      await POST_CREATE[property](ctx, usageContext)
+    }
+  }
+
+  // update the usage
+  if (!usageContext.skipUsage) {
+    await usageQuota.update(property, usage)
+  }
+}
+
+const appPreDelete = async (ctx, usageContext) => {
+  if (ctx.query.unpublish) {
+    // don't run usage decrement for unpublish
+    usageContext.skipUsage = true
+    return
+  }
+
+  // store the row count to delete
+  const rows = await getUniqueRows([ctx.appId])
+  if (rows.size) {
+    usageContext[usageQuota.Properties.APPS] = { rowCount: rows.size }
+  }
+}
+
+const appPostDelete = async (ctx, usageContext) => {
+  // delete the app rows from usage
+  const rowCount = usageContext[usageQuota.Properties.APPS].rowCount
+  if (rowCount) {
+    await usageQuota.update(usageQuota.Properties.ROW, -rowCount)
+  }
+}
+
+const PRE_DELETE = {
+  [usageQuota.Properties.APPS]: appPreDelete,
+}
+
+const POST_DELETE = {
+  [usageQuota.Properties.APPS]: appPostDelete,
+}
+
+const PRE_CREATE = {}
+
+const POST_CREATE = {}
diff --git a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
index 61e6385aa6..5445919f26 100644
--- a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
+++ b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
@@ -7,44 +7,7 @@ const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 const { getAllApps } = require("@budibase/backend-core/db")
 const CouchDB = require("../db")
 const { getUsageQuotaDoc, useQuotas } = require("../utilities/usageQuota")
-const { getRowParams } = require("../db/utils")
-
-/**
- * Get all rows in the given app ids.
- *
- * The returned rows may contan duplicates if there
- * is a production and dev app.
- */
-const getAllRows = async appIds => {
-  const allRows = []
-  let appDb
-  for (let appId of appIds) {
-    try {
-      appDb = new CouchDB(appId)
-      const response = await appDb.allDocs(
-        getRowParams(null, null, {
-          include_docs: false,
-        })
-      )
-      allRows.push(...response.rows.map(r => r.id))
-    } catch (e) {
-      // don't error out if we can't count the app rows, just continue
-    }
-  }
-
-  return allRows
-}
-
-/**
- * Get all rows in the given app ids.
- *
- * The returned rows will be unique, duplicated rows across
- * production and dev apps will be removed.
- */
-const getUniqueRows = async appIds => {
-  const allRows = await getAllRows(appIds)
-  return new Set(allRows)
-}
+const { getUniqueRows } = require("../utilities/usageQuota/rows")
 
 const syncRowsQuota = async db => {
   // get all rows in all apps
diff --git a/packages/server/src/utilities/tests/usageQuota.spec.js b/packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js
similarity index 97%
rename from packages/server/src/utilities/tests/usageQuota.spec.js
rename to packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js
index 6023b08fbe..764d509b5e 100644
--- a/packages/server/src/utilities/tests/usageQuota.spec.js
+++ b/packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js
@@ -3,7 +3,7 @@ jest.mock("@budibase/backend-core/tenancy", () => ({
   getTenantId
 }))
 const usageQuota = require("../usageQuota")
-const env = require("../../environment")
+const env = require("../../../environment")
 
 class TestConfiguration {
   constructor() {
diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota/index.js
similarity index 81%
rename from packages/server/src/utilities/usageQuota.js
rename to packages/server/src/utilities/usageQuota/index.js
index 5965334205..110f8ef600 100644
--- a/packages/server/src/utilities/usageQuota.js
+++ b/packages/server/src/utilities/usageQuota/index.js
@@ -1,4 +1,4 @@
-const env = require("../environment")
+const env = require("../../environment")
 const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
 const {
   StaticDatabases,
@@ -55,7 +55,7 @@ exports.getUsageQuotaDoc = async db => {
  * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
  * also been reset after this call.
  */
-exports.update = async (property, usage) => {
+exports.update = async (property, usage, opts = { dryRun: false }) => {
   if (!exports.useQuotas()) {
     return
   }
@@ -67,14 +67,24 @@ exports.update = async (property, usage) => {
     // increment the quota
     quota.usageQuota[property] += usage
 
-    if (quota.usageQuota[property] > quota.usageLimits[property]) {
+    if (
+      quota.usageQuota[property] > quota.usageLimits[property] &&
+      usage > 0 // allow for decrementing usage when the quota is already exceeded
+    ) {
       throw new Error(
         `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
       )
     }
 
+    if (quota.usageQuota[property] < 0) {
+      // never go negative if the quota has previously been exceeded
+      quota.usageQuota[property] = 0
+    }
+
     // update the usage quotas
-    await db.put(quota)
+    if (!opts.dryRun) {
+      await db.put(quota)
+    }
   } catch (err) {
     console.error(`Error updating usage quotas for ${property}`, err)
     throw err
diff --git a/packages/server/src/utilities/usageQuota/rows.js b/packages/server/src/utilities/usageQuota/rows.js
new file mode 100644
index 0000000000..3a5602958b
--- /dev/null
+++ b/packages/server/src/utilities/usageQuota/rows.js
@@ -0,0 +1,52 @@
+const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils")
+const CouchDB = require("../../db")
+
+const ROW_EXCLUSIONS = [USER_METDATA_PREFIX]
+
+/**
+ * Get all rows in the given app ids.
+ *
+ * The returned rows may contan duplicates if there
+ * is a production and dev app.
+ */
+const getAllRows = async appIds => {
+  const allRows = []
+  let appDb
+  for (let appId of appIds) {
+    try {
+      appDb = new CouchDB(appId)
+      const response = await appDb.allDocs(
+        getRowParams(null, null, {
+          include_docs: false,
+        })
+      )
+      allRows.push(
+        ...response.rows
+          .map(r => r.id)
+          .filter(id => {
+            for (let exclusion of ROW_EXCLUSIONS) {
+              if (id.startsWith(exclusion)) {
+                return false
+              }
+            }
+            return true
+          })
+      )
+    } catch (e) {
+      // don't error out if we can't count the app rows, just continue
+    }
+  }
+
+  return allRows
+}
+
+/**
+ * Get all rows in the given app ids.
+ *
+ * The returned rows will be unique, duplicated rows across
+ * production and dev apps will be removed.
+ */
+exports.getUniqueRows = async appIds => {
+  const allRows = await getAllRows(appIds)
+  return new Set(allRows)
+}
diff --git a/packages/server/src/utilities/usageQuoteReset.js b/packages/server/src/utilities/usageQuota/usageQuoteReset.js
similarity index 100%
rename from packages/server/src/utilities/usageQuoteReset.js
rename to packages/server/src/utilities/usageQuota/usageQuoteReset.js

From 4d61e84c2b4e9fe05bfcf85118271e274692d579 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Tue, 18 Jan 2022 11:57:20 +0000
Subject: [PATCH 3/7] Handle table deletion and data import

---
 .../modals/CreateTableModal.svelte            | 25 ++++++++++++-------
 .../src/api/controllers/table/internal.js     |  2 ++
 .../server/src/api/controllers/table/utils.js |  5 ++++
 3 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
index 997864e165..a3b7ca81a6 100644
--- a/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
@@ -53,16 +53,23 @@
     }
 
     // Create table
-    const table = await tables.save(newTable)
-    notifications.success(`Table ${name} created successfully.`)
-    analytics.captureEvent(Events.TABLE.CREATED, { name })
+    let table
+    try {
+      table = await tables.save(newTable)
+      notifications.success(`Table ${name} created successfully.`)
+      analytics.captureEvent(Events.TABLE.CREATED, { name })
 
-    // Navigate to new table
-    const currentUrl = $url()
-    const path = currentUrl.endsWith("data")
-      ? `./table/${table._id}`
-      : `../../table/${table._id}`
-    $goto(path)
+      // Navigate to new table
+      const currentUrl = $url()
+      const path = currentUrl.endsWith("data")
+        ? `./table/${table._id}`
+        : `../../table/${table._id}`
+      $goto(path)
+    } catch (e) {
+      notifications.error(e)
+      // reload in case the table was created
+      await tables.fetch()
+    }
   }
 </script>
 
diff --git a/packages/server/src/api/controllers/table/internal.js b/packages/server/src/api/controllers/table/internal.js
index 10a5c9746a..9f09e78219 100644
--- a/packages/server/src/api/controllers/table/internal.js
+++ b/packages/server/src/api/controllers/table/internal.js
@@ -8,6 +8,7 @@ const {
   getTable,
   handleDataImport,
 } = require("./utils")
+const usageQuota = require("../../../utilities/usageQuota")
 
 exports.save = async function (ctx) {
   const appId = ctx.appId
@@ -119,6 +120,7 @@ exports.destroy = async function (ctx) {
     })
   )
   await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
+  await usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length)
 
   // update linked rows
   await linkRows.updateLinks({
diff --git a/packages/server/src/api/controllers/table/utils.js b/packages/server/src/api/controllers/table/utils.js
index e4086e8071..e0311389e6 100644
--- a/packages/server/src/api/controllers/table/utils.js
+++ b/packages/server/src/api/controllers/table/utils.js
@@ -15,6 +15,7 @@ const {
 } = require("../../../integrations/utils")
 const { getViews, saveView } = require("../view/utils")
 const viewTemplate = require("../view/viewBuilder")
+const usageQuota = require("../../../utilities/usageQuota")
 
 exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
   let updatedRows = []
@@ -111,7 +112,11 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
     finalData.push(row)
   }
 
+  await usageQuota.update(usageQuota.Properties.ROW, finalData.length, {
+    dryRun: true,
+  })
   await db.bulkDocs(finalData)
+  await usageQuota.update(usageQuota.Properties.ROW, finalData.length)
   let response = await db.put(table)
   table._rev = response._rev
   return table

From 79c10fba7cf7a5d7952c9248ad07e61186e97650 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Tue, 18 Jan 2022 12:48:54 +0000
Subject: [PATCH 4/7] Account for duplicate row ids caused by app import

---
 packages/server/src/middleware/usageQuota.js  |  4 +-
 .../sync_app_and_reset_rows_quotas.js         |  2 +-
 .../server/src/utilities/usageQuota/rows.js   | 95 +++++++++++--------
 3 files changed, 61 insertions(+), 40 deletions(-)

diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js
index 2d55aac61d..c42442affe 100644
--- a/packages/server/src/middleware/usageQuota.js
+++ b/packages/server/src/middleware/usageQuota.js
@@ -129,8 +129,8 @@ const appPreDelete = async (ctx, usageContext) => {
 
   // store the row count to delete
   const rows = await getUniqueRows([ctx.appId])
-  if (rows.size) {
-    usageContext[usageQuota.Properties.APPS] = { rowCount: rows.size }
+  if (rows.length) {
+    usageContext[usageQuota.Properties.APPS] = { rowCount: rows.length }
   }
 }
 
diff --git a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
index 5445919f26..4ab2e413f3 100644
--- a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
+++ b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
@@ -17,7 +17,7 @@ const syncRowsQuota = async db => {
 
   // sync row count
   const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.rows = rows.size
+  usageDoc.usageQuota.rows = rows.length
   await db.put(usageDoc)
 }
 
diff --git a/packages/server/src/utilities/usageQuota/rows.js b/packages/server/src/utilities/usageQuota/rows.js
index 3a5602958b..19c3e5b8bf 100644
--- a/packages/server/src/utilities/usageQuota/rows.js
+++ b/packages/server/src/utilities/usageQuota/rows.js
@@ -1,52 +1,73 @@
 const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils")
 const CouchDB = require("../../db")
+const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db")
 
 const ROW_EXCLUSIONS = [USER_METDATA_PREFIX]
 
-/**
- * Get all rows in the given app ids.
- *
- * The returned rows may contan duplicates if there
- * is a production and dev app.
- */
-const getAllRows = async appIds => {
-  const allRows = []
-  let appDb
+const getAppPairs = appIds => {
+  // collect the app ids into dev / prod pairs
+  // keyed by the dev app id
+  const pairs = {}
   for (let appId of appIds) {
-    try {
-      appDb = new CouchDB(appId)
-      const response = await appDb.allDocs(
-        getRowParams(null, null, {
-          include_docs: false,
-        })
-      )
-      allRows.push(
-        ...response.rows
-          .map(r => r.id)
-          .filter(id => {
-            for (let exclusion of ROW_EXCLUSIONS) {
-              if (id.startsWith(exclusion)) {
-                return false
-              }
-            }
-            return true
-          })
-      )
-    } catch (e) {
-      // don't error out if we can't count the app rows, just continue
+    const devId = getDevelopmentAppID(appId)
+    if (!pairs[devId]) {
+      pairs[devId] = {}
+    }
+    if (isDevAppID(appId)) {
+      pairs[devId].devId = appId
+    } else {
+      pairs[devId].prodId = appId
     }
   }
+  return pairs
+}
 
-  return allRows
+const getAppRows = async appId => {
+  const appDb = new CouchDB(appId)
+  const response = await appDb.allDocs(
+    getRowParams(null, null, {
+      include_docs: false,
+    })
+  )
+  return response.rows
+    .map(r => r.id)
+    .filter(id => {
+      for (let exclusion of ROW_EXCLUSIONS) {
+        if (id.startsWith(exclusion)) {
+          return false
+        }
+      }
+      return true
+    })
 }
 
 /**
- * Get all rows in the given app ids.
- *
- * The returned rows will be unique, duplicated rows across
- * production and dev apps will be removed.
+ * Return a set of all rows in the given app ids.
+ * The returned rows will be unique on a per dev/prod app basis.
+ * Rows duplicates may exist across apps due to data import so they are not filtered out.
  */
 exports.getUniqueRows = async appIds => {
-  const allRows = await getAllRows(appIds)
-  return new Set(allRows)
+  let uniqueRows = []
+  const pairs = getAppPairs(appIds)
+
+  for (let pair of Object.values(pairs)) {
+    let appRows = []
+    for (let appId of [pair.devId, pair.prodId]) {
+      if (!appId) {
+        continue
+      }
+      try {
+        appRows.push(await getAppRows(appId))
+      } catch (e) {
+        // don't error out if we can't count the app rows, just continue
+      }
+    }
+
+    // ensure uniqueness on a per app pair basis
+    // this can't be done on all rows because app import results in
+    // duplicate row ids across apps
+    uniqueRows = uniqueRows.concat(...new Set(appRows))
+  }
+
+  return uniqueRows
 }

From 439fa9a010f7e1c9b5276f3c1cb628f7f0f7aae1 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Tue, 18 Jan 2022 14:43:24 +0000
Subject: [PATCH 5/7] Refactor to allow for future quota migrations

---
 packages/backend-core/src/migrations/index.js |  4 +-
 packages/server/src/middleware/usageQuota.js  | 15 ++++--
 .../sync_app_and_reset_rows_quotas.js         | 48 -------------------
 .../tests/usageQuotas/index.spec.js           | 27 +++++++++++
 .../syncApps.spec.js}                         | 20 ++++----
 .../tests/usageQuotas/syncRows.spec.js        | 43 +++++++++++++++++
 .../src/migrations/usageQuotas/index.js       | 24 ++++++++++
 .../src/migrations/usageQuotas/syncApps.js    | 16 +++++++
 .../src/migrations/usageQuotas/syncRows.js    | 18 +++++++
 9 files changed, 151 insertions(+), 64 deletions(-)
 delete mode 100644 packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
 create mode 100644 packages/server/src/migrations/tests/usageQuotas/index.spec.js
 rename packages/server/src/migrations/tests/{sync_app_and_reset_rows_quotas.spec.js => usageQuotas/syncApps.spec.js} (59%)
 create mode 100644 packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js
 create mode 100644 packages/server/src/migrations/usageQuotas/index.js
 create mode 100644 packages/server/src/migrations/usageQuotas/syncApps.js
 create mode 100644 packages/server/src/migrations/usageQuotas/syncRows.js

diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js
index 30e04e198b..4708776ec3 100644
--- a/packages/backend-core/src/migrations/index.js
+++ b/packages/backend-core/src/migrations/index.js
@@ -7,13 +7,13 @@ exports.MIGRATION_DBS = {
 
 exports.MIGRATIONS = {
   USER_EMAIL_VIEW_CASING: "user_email_view_casing",
-  SYNC_APP_AND_RESET_ROWS_QUOTAS: "sync_app_and_reset_rows_quotas",
+  QUOTAS_1: "quotas_1",
 }
 
 const DB_LOOKUP = {
   [exports.MIGRATION_DBS.GLOBAL_DB]: [
     exports.MIGRATIONS.USER_EMAIL_VIEW_CASING,
-    exports.MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS,
+    exports.MIGRATIONS.QUOTAS_1,
   ],
 }
 
diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js
index c42442affe..8e3daa6268 100644
--- a/packages/server/src/middleware/usageQuota.js
+++ b/packages/server/src/middleware/usageQuota.js
@@ -5,7 +5,7 @@ const {
   isExternalTable,
   isRowId: isExternalRowId,
 } = require("../integrations/utils")
-const quotaMigration = require("../migrations/sync_app_and_reset_rows_quotas")
+const migration = require("../migrations/usageQuotas")
 
 // currently only counting new writes and deletes
 const METHOD_MAP = {
@@ -74,7 +74,7 @@ module.exports = async (ctx, next) => {
     usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
   try {
-    await quotaMigration.runIfRequired()
+    await migration.run()
     await performRequest(ctx, next, property, usage)
   } catch (err) {
     ctx.throw(400, err)
@@ -142,6 +142,13 @@ const appPostDelete = async (ctx, usageContext) => {
   }
 }
 
+// const appPostCreate = async (ctx, usageContext) => {
+// if (ctx.request) {
+//   const rowCount = await getUniqueRows([ctx.appId]).length
+//   await usageQuota.update(usageQuota.Properties.ROW, -rowCount)
+// }
+// }
+
 const PRE_DELETE = {
   [usageQuota.Properties.APPS]: appPreDelete,
 }
@@ -152,4 +159,6 @@ const POST_DELETE = {
 
 const PRE_CREATE = {}
 
-const POST_CREATE = {}
+const POST_CREATE = {
+  // [usageQuota.Properties.APPS]: appPostCreate,
+}
diff --git a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js b/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
deleted file mode 100644
index 4ab2e413f3..0000000000
--- a/packages/server/src/migrations/sync_app_and_reset_rows_quotas.js
+++ /dev/null
@@ -1,48 +0,0 @@
-const {
-  MIGRATIONS,
-  MIGRATION_DBS,
-  migrateIfRequired,
-} = require("@budibase/backend-core/migrations")
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const { getAllApps } = require("@budibase/backend-core/db")
-const CouchDB = require("../db")
-const { getUsageQuotaDoc, useQuotas } = require("../utilities/usageQuota")
-const { getUniqueRows } = require("../utilities/usageQuota/rows")
-
-const syncRowsQuota = async db => {
-  // get all rows in all apps
-  const allApps = await getAllApps(CouchDB, { all: true })
-  const appIds = allApps ? allApps.map(app => app.appId) : []
-  const rows = await getUniqueRows(appIds)
-
-  // sync row count
-  const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.rows = rows.length
-  await db.put(usageDoc)
-}
-
-const syncAppsQuota = async db => {
-  // get app count
-  const devApps = await getAllApps(CouchDB, { dev: true })
-  const appCount = devApps ? devApps.length : 0
-
-  // sync app count
-  const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.apps = appCount
-  await db.put(usageDoc)
-}
-
-exports.runIfRequired = async () => {
-  await migrateIfRequired(
-    MIGRATION_DBS.GLOBAL_DB,
-    MIGRATIONS.SYNC_APP_AND_RESET_ROWS_QUOTAS,
-    async () => {
-      if (!useQuotas()) {
-        return
-      }
-      const db = getGlobalDB()
-      await syncAppsQuota(db)
-      await syncRowsQuota(db)
-    }
-  )
-}
diff --git a/packages/server/src/migrations/tests/usageQuotas/index.spec.js b/packages/server/src/migrations/tests/usageQuotas/index.spec.js
new file mode 100644
index 0000000000..0c5b982909
--- /dev/null
+++ b/packages/server/src/migrations/tests/usageQuotas/index.spec.js
@@ -0,0 +1,27 @@
+const env = require("../../../environment")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+
+const syncApps = jest.fn()
+const syncRows = jest.fn()
+
+jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) )
+jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) )
+
+const migrations = require("../../usageQuotas")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs the required migrations", async () => {
+    await migrations.run()
+    expect(syncApps).toHaveBeenCalledTimes(1)
+    expect(syncRows).toHaveBeenCalledTimes(1)
+  })
+})
diff --git a/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js b/packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js
similarity index 59%
rename from packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js
rename to packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js
index cd533e3f06..160319a31b 100644
--- a/packages/server/src/migrations/tests/sync_app_and_reset_rows_quotas.spec.js
+++ b/packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js
@@ -1,10 +1,10 @@
 const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const TestConfig = require("../../tests/utilities/TestConfiguration")
-const { getUsageQuotaDoc, update, Properties } = require("../../utilities/usageQuota")
-const { runIfRequired } = require("../sync_app_and_reset_rows_quotas")
-const env = require("../../environment")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
+const syncApps = require("../../usageQuotas/syncApps")
+const env = require("../../../environment")
 
-describe("Sync App And Reset Rows Quotas Migration", () => {
+describe("syncApps", () => {
   let config = new TestConfig(false)
 
   beforeEach(async () => {
@@ -12,28 +12,26 @@ describe("Sync App And Reset Rows Quotas Migration", () => {
     env._set("USE_QUOTAS", 1)
   })
 
-  afterAll(config.end)
+  afterAll(config.end)  
 
-  it("migrates successfully", async () => {
+  it("runs successfully", async () => {
     // create the usage quota doc and mock usages
     const db = getGlobalDB()
     await getUsageQuotaDoc(db)
     await update(Properties.APPS, 3)
-    await update(Properties.ROW, 300)
 
     let usageDoc = await getUsageQuotaDoc(db)
     expect(usageDoc.usageQuota.apps).toEqual(3)
-    expect(usageDoc.usageQuota.rows).toEqual(300)
 
     // create an extra app to test the migration
     await config.createApp("quota-test")
 
     // migrate
-    await runIfRequired()
+    await syncApps.run()
 
     // assert the migration worked 
     usageDoc = await getUsageQuotaDoc(db)
     expect(usageDoc.usageQuota.apps).toEqual(2)
-    expect(usageDoc.usageQuota.rows).toEqual(0)
   })
 })
+
diff --git a/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js b/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js
new file mode 100644
index 0000000000..a09bea60bd
--- /dev/null
+++ b/packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js
@@ -0,0 +1,43 @@
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
+const syncRows = require("../../usageQuotas/syncRows")
+const env = require("../../../environment")
+
+describe("syncRows", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    // create the usage quota doc and mock usages
+    const db = getGlobalDB()
+    await getUsageQuotaDoc(db)
+    await update(Properties.ROW, 300)
+
+    let usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.rows).toEqual(300)
+
+    // app 1
+    await config.createTable()
+    await config.createRow()
+    // app 2
+    await config.createApp()
+    await config.createTable()
+    await config.createRow()
+    await config.createRow()
+    
+    // migrate
+    await syncRows.run()
+
+    // assert the migration worked 
+    usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.rows).toEqual(3)
+  })
+})
+
diff --git a/packages/server/src/migrations/usageQuotas/index.js b/packages/server/src/migrations/usageQuotas/index.js
new file mode 100644
index 0000000000..39744093c2
--- /dev/null
+++ b/packages/server/src/migrations/usageQuotas/index.js
@@ -0,0 +1,24 @@
+const {
+  MIGRATIONS,
+  MIGRATION_DBS,
+  migrateIfRequired,
+} = require("@budibase/backend-core/migrations")
+const { useQuotas } = require("../../utilities/usageQuota")
+const syncApps = require("./syncApps")
+const syncRows = require("./syncRows")
+
+exports.run = async () => {
+  if (!useQuotas()) {
+    return
+  }
+
+  // Jan 2022
+  await migrateIfRequired(
+    MIGRATION_DBS.GLOBAL_DB,
+    MIGRATIONS.QUOTAS_1,
+    async () => {
+      await syncApps.run()
+      await syncRows.run()
+    }
+  )
+}
diff --git a/packages/server/src/migrations/usageQuotas/syncApps.js b/packages/server/src/migrations/usageQuotas/syncApps.js
new file mode 100644
index 0000000000..c285bafe4e
--- /dev/null
+++ b/packages/server/src/migrations/usageQuotas/syncApps.js
@@ -0,0 +1,16 @@
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { getAllApps } = require("@budibase/backend-core/db")
+const CouchDB = require("../../db")
+const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+
+exports.run = async () => {
+  const db = getGlobalDB()
+  // get app count
+  const devApps = await getAllApps(CouchDB, { dev: true })
+  const appCount = devApps ? devApps.length : 0
+
+  // sync app count
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.apps = appCount
+  await db.put(usageDoc)
+}
diff --git a/packages/server/src/migrations/usageQuotas/syncRows.js b/packages/server/src/migrations/usageQuotas/syncRows.js
new file mode 100644
index 0000000000..73f6f1477d
--- /dev/null
+++ b/packages/server/src/migrations/usageQuotas/syncRows.js
@@ -0,0 +1,18 @@
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { getAllApps } = require("@budibase/backend-core/db")
+const CouchDB = require("../../db")
+const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+const { getUniqueRows } = require("../../utilities/usageQuota/rows")
+
+exports.run = async () => {
+  const db = getGlobalDB()
+  // get all rows in all apps
+  const allApps = await getAllApps(CouchDB, { all: true })
+  const appIds = allApps ? allApps.map(app => app.appId) : []
+  const rows = await getUniqueRows(appIds)
+
+  // sync row count
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.rows = rows.length
+  await db.put(usageDoc)
+}

From fc043e116d0d56c7b1e175aa94097207c183fcf4 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Tue, 18 Jan 2022 15:37:16 +0000
Subject: [PATCH 6/7] Support template and app import in quotas + logging

---
 packages/backend-core/src/migrations/index.js    | 12 ++++++++----
 packages/server/src/middleware/usageQuota.js     | 16 +++++++++-------
 .../src/migrations/usageQuotas/syncApps.js       |  4 +++-
 .../src/migrations/usageQuotas/syncRows.js       |  7 +++++--
 .../tests/usageQuota/usageQuota.spec.js          |  2 +-
 packages/server/src/utilities/usageQuota/rows.js |  1 +
 6 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/packages/backend-core/src/migrations/index.js b/packages/backend-core/src/migrations/index.js
index 4708776ec3..6b8eb3a95c 100644
--- a/packages/backend-core/src/migrations/index.js
+++ b/packages/backend-core/src/migrations/index.js
@@ -1,5 +1,5 @@
 const { DocumentTypes } = require("../db/constants")
-const { getGlobalDB } = require("../tenancy")
+const { getGlobalDB, getTenantId } = require("../tenancy")
 
 exports.MIGRATION_DBS = {
   GLOBAL_DB: "GLOBAL_DB",
@@ -29,6 +29,7 @@ exports.getMigrationsDoc = async db => {
 }
 
 exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
+  const tenantId = getTenantId()
   try {
     let db
     if (migrationDb === exports.MIGRATION_DBS.GLOBAL_DB) {
@@ -49,15 +50,18 @@ exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
       return
     }
 
-    console.log(`Performing migration: ${migrationName}`)
+    console.log(`[Tenant: ${tenantId}] Performing migration: ${migrationName}`)
     await migrateFn()
-    console.log(`Migration complete: ${migrationName}`)
+    console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`)
 
     // mark as complete
     doc[migrationName] = Date.now()
     await db.put(doc)
   } catch (err) {
-    console.error(`Error performing migration: ${migrationName}: `, err)
+    console.error(
+      `[Tenant: ${tenantId}] Error performing migration: ${migrationName}: `,
+      err
+    )
     throw err
   }
 }
diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js
index 8e3daa6268..4bafa75132 100644
--- a/packages/server/src/middleware/usageQuota.js
+++ b/packages/server/src/middleware/usageQuota.js
@@ -142,12 +142,14 @@ const appPostDelete = async (ctx, usageContext) => {
   }
 }
 
-// const appPostCreate = async (ctx, usageContext) => {
-// if (ctx.request) {
-//   const rowCount = await getUniqueRows([ctx.appId]).length
-//   await usageQuota.update(usageQuota.Properties.ROW, -rowCount)
-// }
-// }
+const appPostCreate = async ctx => {
+  // app import & template creation
+  if (ctx.request.body.useTemplate === "true") {
+    const rows = await getUniqueRows([ctx.response.body.appId])
+    const rowCount = rows ? rows.length : 0
+    await usageQuota.update(usageQuota.Properties.ROW, rowCount)
+  }
+}
 
 const PRE_DELETE = {
   [usageQuota.Properties.APPS]: appPreDelete,
@@ -160,5 +162,5 @@ const POST_DELETE = {
 const PRE_CREATE = {}
 
 const POST_CREATE = {
-  // [usageQuota.Properties.APPS]: appPostCreate,
+  [usageQuota.Properties.APPS]: appPostCreate,
 }
diff --git a/packages/server/src/migrations/usageQuotas/syncApps.js b/packages/server/src/migrations/usageQuotas/syncApps.js
index c285bafe4e..ee106129e6 100644
--- a/packages/server/src/migrations/usageQuotas/syncApps.js
+++ b/packages/server/src/migrations/usageQuotas/syncApps.js
@@ -1,4 +1,4 @@
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
 const { getAllApps } = require("@budibase/backend-core/db")
 const CouchDB = require("../../db")
 const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
@@ -10,6 +10,8 @@ exports.run = async () => {
   const appCount = devApps ? devApps.length : 0
 
   // sync app count
+  const tenantId = getTenantId()
+  console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
   const usageDoc = await getUsageQuotaDoc(db)
   usageDoc.usageQuota.apps = appCount
   await db.put(usageDoc)
diff --git a/packages/server/src/migrations/usageQuotas/syncRows.js b/packages/server/src/migrations/usageQuotas/syncRows.js
index 73f6f1477d..7990f405de 100644
--- a/packages/server/src/migrations/usageQuotas/syncRows.js
+++ b/packages/server/src/migrations/usageQuotas/syncRows.js
@@ -1,4 +1,4 @@
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
 const { getAllApps } = require("@budibase/backend-core/db")
 const CouchDB = require("../../db")
 const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
@@ -10,9 +10,12 @@ exports.run = async () => {
   const allApps = await getAllApps(CouchDB, { all: true })
   const appIds = allApps ? allApps.map(app => app.appId) : []
   const rows = await getUniqueRows(appIds)
+  const rowCount = rows ? rows.length : 0
 
   // sync row count
+  const tenantId = getTenantId()
+  console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
   const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.rows = rows.length
+  usageDoc.usageQuota.rows = rowCount
   await db.put(usageDoc)
 }
diff --git a/packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js b/packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js
index 764d509b5e..dcd7578f59 100644
--- a/packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js
+++ b/packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js
@@ -2,7 +2,7 @@ const getTenantId = jest.fn()
 jest.mock("@budibase/backend-core/tenancy", () => ({
   getTenantId
 }))
-const usageQuota = require("../usageQuota")
+const usageQuota = require("../../usageQuota")
 const env = require("../../../environment")
 
 class TestConfiguration {
diff --git a/packages/server/src/utilities/usageQuota/rows.js b/packages/server/src/utilities/usageQuota/rows.js
index 19c3e5b8bf..67ad07410d 100644
--- a/packages/server/src/utilities/usageQuota/rows.js
+++ b/packages/server/src/utilities/usageQuota/rows.js
@@ -59,6 +59,7 @@ exports.getUniqueRows = async appIds => {
       try {
         appRows.push(await getAppRows(appId))
       } catch (e) {
+        console.error(e)
         // don't error out if we can't count the app rows, just continue
       }
     }

From 604d316834630601a093ffe1199640870f0ffde7 Mon Sep 17 00:00:00 2001
From: Rory Powell <rory.codes@gmail.com>
Date: Tue, 18 Jan 2022 15:53:55 +0000
Subject: [PATCH 7/7] Add dryRun to automation row create and remove env
 conditional on delete

---
 packages/server/src/automations/steps/createRow.js | 3 ++-
 packages/server/src/automations/steps/deleteRow.js | 5 +----
 packages/server/src/utilities/usageQuota/index.js  | 4 ++--
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/automations/steps/createRow.js b/packages/server/src/automations/steps/createRow.js
index 816cd829ab..1937121062 100644
--- a/packages/server/src/automations/steps/createRow.js
+++ b/packages/server/src/automations/steps/createRow.js
@@ -82,8 +82,9 @@ exports.run = async function ({ inputs, appId, emitter }) {
       inputs.row.tableId,
       inputs.row
     )
-    await usage.update(usage.Properties.ROW, 1)
+    await usage.update(usage.Properties.ROW, 1, { dryRun: true })
     await rowController.save(ctx)
+    await usage.update(usage.Properties.ROW, 1)
     return {
       row: inputs.row,
       response: ctx.body,
diff --git a/packages/server/src/automations/steps/deleteRow.js b/packages/server/src/automations/steps/deleteRow.js
index c7bee577a5..e41e5ad263 100644
--- a/packages/server/src/automations/steps/deleteRow.js
+++ b/packages/server/src/automations/steps/deleteRow.js
@@ -1,5 +1,4 @@
 const rowController = require("../../api/controllers/row")
-const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")
 const { buildCtx } = require("./utils")
 const automationUtils = require("../automationUtils")
@@ -74,9 +73,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
   })
 
   try {
-    if (env.isProd()) {
-      await usage.update(usage.Properties.ROW, -1)
-    }
+    await usage.update(usage.Properties.ROW, -1)
     await rowController.destroy(ctx)
     return {
       response: ctx.body,
diff --git a/packages/server/src/utilities/usageQuota/index.js b/packages/server/src/utilities/usageQuota/index.js
index 110f8ef600..b0ff310aa3 100644
--- a/packages/server/src/utilities/usageQuota/index.js
+++ b/packages/server/src/utilities/usageQuota/index.js
@@ -25,12 +25,12 @@ exports.useQuotas = () => {
 }
 
 exports.Properties = {
-  ROW: "rows", // mostly works - app / table deletion not yet accounted for
+  ROW: "rows",
   UPLOAD: "storage", // doesn't work yet
   VIEW: "views", // doesn't work yet
   USER: "users", // doesn't work yet
   AUTOMATION: "automationRuns", // doesn't work yet
-  APPS: "apps", // works
+  APPS: "apps",
   EMAILS: "emails", // doesn't work yet
 }