Merge branch 'develop' of github.com:Budibase/budibase into feature/formula-filtering

charts/budibase/templates/app-service-deployment.yaml

@@ -99,7 +99,9 @@ spec:
         - name: PLATFORM_URL
           value: {{ .Values.globals.platformUrl | quote }}
         - name: USE_QUOTAS
-          value: "1" 
+          value: {{ .Values.globals.useQuotas | quote }}
+        - name: EXCLUDE_QUOTAS_TENANTS
+          value: {{ .Values.globals.excludeQuotasTenants | quote }}
         - name: ACCOUNT_PORTAL_URL
           value: {{ .Values.globals.accountPortalUrl | quote }}
         - name: ACCOUNT_PORTAL_API_KEY

charts/budibase/values.yaml

@@ -93,6 +93,8 @@ globals:
   logLevel: info
   selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
   multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs
+  useQuotas: "0"
+  excludeQuotasTenants: "" # comma seperated list of tenants to exclude from quotas
   accountPortalUrl: ""
   accountPortalApiKey: ""
   cookieDomain: ""
@@ -239,7 +241,8 @@ couchdb:
     hosts:
       - chart-example.local
     path: /
-    annotations: []
+    annotations:
+      []
       # kubernetes.io/ingress.class: nginx
       # kubernetes.io/tls-acme: "true"
     tls:

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.46-alpha.2",
+  "version": "1.0.46-alpha.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/migrations.js

@@ -0,0 +1 @@
+module.exports = require("./src/migrations")

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.0.46-alpha.2",
+  "version": "1.0.46-alpha.3",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "src/index.js",
   "author": "Budibase",

packages/backend-core/src/db/constants.js

@@ -21,6 +21,7 @@ exports.StaticDatabases = {
     name: "global-db",
     docs: {
       apiKeys: "apikeys",
+      usageQuota: "usage_quota",
     },
   },
   // contains information about tenancy and so on
@@ -28,7 +29,6 @@ exports.StaticDatabases = {
     name: "global-info",
     docs: {
       tenants: "tenants",
-      usageQuota: "usage_quota",
     },
   },
 }

packages/backend-core/src/db/utils.js

@@ -450,7 +450,7 @@ async function getScopedConfig(db, params) {
 
 function generateNewUsageQuotaDoc() {
   return {
-    _id: StaticDatabases.PLATFORM_INFO.docs.usageQuota,
+    _id: StaticDatabases.GLOBAL.docs.usageQuota,
     quotaReset: Date.now() + 2592000000,
     usageQuota: {
       automationRuns: 0,

packages/backend-core/src/index.js

@@ -14,4 +14,5 @@ module.exports = {
   cache: require("../cache"),
   auth: require("../auth"),
   constants: require("../constants"),
+  migrations: require("../migrations"),
 }

packages/backend-core/src/migrations/index.js

@@ -1,5 +1,5 @@
 const { DocumentTypes } = require("../db/constants")
-const { getGlobalDB } = require("../tenancy")
+const { getGlobalDB, getTenantId } = require("../tenancy")
 
 exports.MIGRATION_DBS = {
   GLOBAL_DB: "GLOBAL_DB",
@@ -7,11 +7,13 @@ exports.MIGRATION_DBS = {
 
 exports.MIGRATIONS = {
   USER_EMAIL_VIEW_CASING: "user_email_view_casing",
+  QUOTAS_1: "quotas_1",
 }
 
 const DB_LOOKUP = {
   [exports.MIGRATION_DBS.GLOBAL_DB]: [
     exports.MIGRATIONS.USER_EMAIL_VIEW_CASING,
+    exports.MIGRATIONS.QUOTAS_1,
   ],
 }
 
@@ -27,6 +29,7 @@ exports.getMigrationsDoc = async db => {
 }
 
 exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
+  const tenantId = getTenantId()
   try {
     let db
     if (migrationDb === exports.MIGRATION_DBS.GLOBAL_DB) {
@@ -47,15 +50,18 @@ exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
       return
     }
 
-    console.log(`Performing migration: ${migrationName}`)
+    console.log(`[Tenant: ${tenantId}] Performing migration: ${migrationName}`)
     await migrateFn()
-    console.log(`Migration complete: ${migrationName}`)
+    console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`)
 
     // mark as complete
     doc[migrationName] = Date.now()
     await db.put(doc)
   } catch (err) {
-    console.error(`Error performing migration: ${migrationName}: `, err)
+    console.error(
+      `[Tenant: ${tenantId}] Error performing migration: ${migrationName}: `,
+      err
+    )
     throw err
   }
 }

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.46-alpha.2",
+  "version": "1.0.46-alpha.3",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.46-alpha.2",
+  "version": "1.0.46-alpha.3",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.46-alpha.2",
-    "@budibase/client": "^1.0.46-alpha.2",
+    "@budibase/bbui": "^1.0.46-alpha.3",
+    "@budibase/client": "^1.0.46-alpha.3",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^1.0.46-alpha.2",
+    "@budibase/string-templates": "^1.0.46-alpha.3",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte

@@ -53,7 +53,9 @@
     }
 
     // Create table
-    const table = await tables.save(newTable)
+    let table
+    try {
+      table = await tables.save(newTable)
       notifications.success(`Table ${name} created successfully.`)
       analytics.captureEvent(Events.TABLE.CREATED, { name })
 
@@ -63,6 +65,11 @@
         ? `./table/${table._id}`
         : `../../table/${table._id}`
       $goto(path)
+    } catch (e) {
+      notifications.error(e)
+      // reload in case the table was created
+      await tables.fetch()
+    }
   }
 </script>
 

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.46-alpha.2",
+  "version": "1.0.46-alpha.3",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.0.46-alpha.2",
+  "version": "1.0.46-alpha.3",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.46-alpha.2",
+    "@budibase/bbui": "^1.0.46-alpha.3",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^1.0.46-alpha.2",
+    "@budibase/string-templates": "^1.0.46-alpha.3",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.46-alpha.2",
+  "version": "1.0.46-alpha.3",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.46-alpha.2",
-    "@budibase/client": "^1.0.46-alpha.2",
-    "@budibase/string-templates": "^1.0.46-alpha.2",
+    "@budibase/backend-core": "^1.0.46-alpha.3",
+    "@budibase/client": "^1.0.46-alpha.3",
+    "@budibase/string-templates": "^1.0.46-alpha.3",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",

packages/server/src/api/controllers/table/internal.js

@@ -8,6 +8,7 @@ const {
   getTable,
   handleDataImport,
 } = require("./utils")
+const usageQuota = require("../../../utilities/usageQuota")
 
 exports.save = async function (ctx) {
   const appId = ctx.appId
@@ -119,6 +120,7 @@ exports.destroy = async function (ctx) {
     })
   )
   await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
+  await usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length)
 
   // update linked rows
   await linkRows.updateLinks({

packages/server/src/api/controllers/table/utils.js

@@ -16,6 +16,7 @@ const {
 } = require("../../../integrations/utils")
 const { getViews, saveView } = require("../view/utils")
 const viewTemplate = require("../view/viewBuilder")
+const usageQuota = require("../../../utilities/usageQuota")
 
 exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
   let updatedRows = []
@@ -112,7 +113,11 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
     finalData.push(row)
   }
 
+  await usageQuota.update(usageQuota.Properties.ROW, finalData.length, {
+    dryRun: true,
+  })
   await db.bulkDocs(finalData)
+  await usageQuota.update(usageQuota.Properties.ROW, finalData.length)
   let response = await db.put(table)
   table._rev = response._rev
   return table

packages/server/src/automations/steps/createRow.js

@@ -1,6 +1,5 @@
 const rowController = require("../../api/controllers/row")
 const automationUtils = require("../automationUtils")
-const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")
 const { buildCtx } = require("./utils")
 
@@ -83,10 +82,9 @@ exports.run = async function ({ inputs, appId, emitter }) {
       inputs.row.tableId,
       inputs.row
     )
-    if (env.USE_QUOTAS) {
-      await usage.update(usage.Properties.ROW, 1)
-    }
+    await usage.update(usage.Properties.ROW, 1, { dryRun: true })
     await rowController.save(ctx)
+    await usage.update(usage.Properties.ROW, 1)
     return {
       row: inputs.row,
       response: ctx.body,

packages/server/src/automations/steps/deleteRow.js

@@ -1,5 +1,4 @@
 const rowController = require("../../api/controllers/row")
-const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")
 const { buildCtx } = require("./utils")
 const automationUtils = require("../automationUtils")
@@ -74,9 +73,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
   })
 
   try {
-    if (env.isProd()) {
     await usage.update(usage.Properties.ROW, -1)
-    }
     await rowController.destroy(ctx)
     return {
       response: ctx.body,

packages/server/src/environment.js

@@ -38,6 +38,7 @@ module.exports = {
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   USE_QUOTAS: process.env.USE_QUOTAS,
+  EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS,
   REDIS_URL: process.env.REDIS_URL,
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,

packages/server/src/middleware/tests/usageQuota.spec.js

@@ -1,11 +1,5 @@
 jest.mock("../../db")
 jest.mock("../../utilities/usageQuota")
-jest.mock("../../environment", () => ({
-  isTest: () => true,
-  isProd: () => false,
-  isDev: () => true,
-  _set: () => {},
-}))
 jest.mock("@budibase/backend-core/tenancy", () => ({
   getTenantId: () => "testing123"
 }))
@@ -29,9 +23,10 @@ class TestConfiguration {
       },
       req: {
         method: "POST",
-        url: "/rows"
+        url: "/applications"
       }
     }
+    usageQuota.useQuotas = () => true
   }
 
   executeMiddleware() {
@@ -113,7 +108,6 @@ describe("usageQuota middleware", () => {
 
   it("calculates and persists the correct usage quota for the relevant action", async () => {
     config.setUrl("/rows")
-    config.setProd(true)
 
     await config.executeMiddleware()
 
@@ -121,20 +115,20 @@ describe("usageQuota middleware", () => {
     expect(config.next).toHaveBeenCalled()
   })
 
-  it("calculates the correct file size from a file upload call and adds it to quota", async () => {
-    config.setUrl("/upload")
-    config.setProd(true)
-    config.setFiles([
-      {
-        size: 100
-      },
-      {
-        size: 10000
-      },
-    ])
-    await config.executeMiddleware()
+  // it("calculates the correct file size from a file upload call and adds it to quota", async () => {
+  //   config.setUrl("/upload")
+  //   config.setProd(true)
+  //   config.setFiles([
+  //     {
+  //       size: 100
+  //     },
+  //     {
+  //       size: 10000
+  //     },
+  //   ])
+  //   await config.executeMiddleware()
 
-    expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
-    expect(config.next).toHaveBeenCalled()
-  })
+  //   expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
+  //   expect(config.next).toHaveBeenCalled()
+  // })
 })

packages/server/src/middleware/usageQuota.js

@@ -1,14 +1,11 @@
 const CouchDB = require("../db")
 const usageQuota = require("../utilities/usageQuota")
-const env = require("../environment")
-const { getTenantId } = require("@budibase/backend-core/tenancy")
+const { getUniqueRows } = require("../utilities/usageQuota/rows")
 const {
   isExternalTable,
   isRowId: isExternalRowId,
 } = require("../integrations/utils")
-
-// tenants without limits
-const EXCLUDED_TENANTS = ["bb", "default", "bbtest", "bbstaging"]
+const migration = require("../migrations/usageQuotas")
 
 // currently only counting new writes and deletes
 const METHOD_MAP = {
@@ -18,13 +15,13 @@ const METHOD_MAP = {
 
 const DOMAIN_MAP = {
   rows: usageQuota.Properties.ROW,
-  upload: usageQuota.Properties.UPLOAD,
-  views: usageQuota.Properties.VIEW,
-  users: usageQuota.Properties.USER,
+  // upload: usageQuota.Properties.UPLOAD,            // doesn't work yet
+  // views: usageQuota.Properties.VIEW,               // doesn't work yet
+  // users: usageQuota.Properties.USER,               // doesn't work yet
   applications: usageQuota.Properties.APPS,
   // this will not be updated by endpoint calls
   // instead it will be updated by triggerInfo
-  automationRuns: usageQuota.Properties.AUTOMATION,
+  // automationRuns: usageQuota.Properties.AUTOMATION, // doesn't work yet
 }
 
 function getProperty(url) {
@@ -36,10 +33,7 @@ function getProperty(url) {
 }
 
 module.exports = async (ctx, next) => {
-  const tenantId = getTenantId()
-
-  // if in development or a self hosted cloud usage quotas should not be executed
-  if (env.isDev() || env.SELF_HOSTED || EXCLUDED_TENANTS.includes(tenantId)) {
+  if (!usageQuota.useQuotas()) {
     return next()
   }
 
@@ -80,9 +74,93 @@ module.exports = async (ctx, next) => {
     usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
   try {
-    await usageQuota.update(property, usage)
-    return next()
+    await migration.run()
+    await performRequest(ctx, next, property, usage)
   } catch (err) {
     ctx.throw(400, err)
   }
 }
+
+const performRequest = async (ctx, next, property, usage) => {
+  const usageContext = {
+    skipNext: false,
+    skipUsage: false,
+    [usageQuota.Properties.APPS]: {},
+  }
+
+  if (usage === -1) {
+    if (PRE_DELETE[property]) {
+      await PRE_DELETE[property](ctx, usageContext)
+    }
+  } else {
+    if (PRE_CREATE[property]) {
+      await PRE_CREATE[property](ctx, usageContext)
+    }
+  }
+
+  // run the request
+  if (!usageContext.skipNext) {
+    await usageQuota.update(property, usage, { dryRun: true })
+    await next()
+  }
+
+  if (usage === -1) {
+    if (POST_DELETE[property]) {
+      await POST_DELETE[property](ctx, usageContext)
+    }
+  } else {
+    if (POST_CREATE[property]) {
+      await POST_CREATE[property](ctx, usageContext)
+    }
+  }
+
+  // update the usage
+  if (!usageContext.skipUsage) {
+    await usageQuota.update(property, usage)
+  }
+}
+
+const appPreDelete = async (ctx, usageContext) => {
+  if (ctx.query.unpublish) {
+    // don't run usage decrement for unpublish
+    usageContext.skipUsage = true
+    return
+  }
+
+  // store the row count to delete
+  const rows = await getUniqueRows([ctx.appId])
+  if (rows.length) {
+    usageContext[usageQuota.Properties.APPS] = { rowCount: rows.length }
+  }
+}
+
+const appPostDelete = async (ctx, usageContext) => {
+  // delete the app rows from usage
+  const rowCount = usageContext[usageQuota.Properties.APPS].rowCount
+  if (rowCount) {
+    await usageQuota.update(usageQuota.Properties.ROW, -rowCount)
+  }
+}
+
+const appPostCreate = async ctx => {
+  // app import & template creation
+  if (ctx.request.body.useTemplate === "true") {
+    const rows = await getUniqueRows([ctx.response.body.appId])
+    const rowCount = rows ? rows.length : 0
+    await usageQuota.update(usageQuota.Properties.ROW, rowCount)
+  }
+}
+
+const PRE_DELETE = {
+  [usageQuota.Properties.APPS]: appPreDelete,
+}
+
+const POST_DELETE = {
+  [usageQuota.Properties.APPS]: appPostDelete,
+}
+
+const PRE_CREATE = {}
+
+const POST_CREATE = {
+  [usageQuota.Properties.APPS]: appPostCreate,
+}

packages/server/src/migrations/tests/usageQuotas/index.spec.js

@@ -0,0 +1,27 @@
+const env = require("../../../environment")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+
+const syncApps = jest.fn()
+const syncRows = jest.fn()
+
+jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) )
+jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) )
+
+const migrations = require("../../usageQuotas")
+
+describe("run", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs the required migrations", async () => {
+    await migrations.run()
+    expect(syncApps).toHaveBeenCalledTimes(1)
+    expect(syncRows).toHaveBeenCalledTimes(1)
+  })
+})

packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js

@@ -0,0 +1,37 @@
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
+const syncApps = require("../../usageQuotas/syncApps")
+const env = require("../../../environment")
+
+describe("syncApps", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    // create the usage quota doc and mock usages
+    const db = getGlobalDB()
+    await getUsageQuotaDoc(db)
+    await update(Properties.APPS, 3)
+
+    let usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.apps).toEqual(3)
+
+    // create an extra app to test the migration
+    await config.createApp("quota-test")
+
+    // migrate
+    await syncApps.run()
+
+    // assert the migration worked 
+    usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.apps).toEqual(2)
+  })
+})
+

packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js

@@ -0,0 +1,43 @@
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const TestConfig = require("../../../tests/utilities/TestConfiguration")
+const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
+const syncRows = require("../../usageQuotas/syncRows")
+const env = require("../../../environment")
+
+describe("syncRows", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    // create the usage quota doc and mock usages
+    const db = getGlobalDB()
+    await getUsageQuotaDoc(db)
+    await update(Properties.ROW, 300)
+
+    let usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.rows).toEqual(300)
+
+    // app 1
+    await config.createTable()
+    await config.createRow()
+    // app 2
+    await config.createApp()
+    await config.createTable()
+    await config.createRow()
+    await config.createRow()
+    
+    // migrate
+    await syncRows.run()
+
+    // assert the migration worked 
+    usageDoc = await getUsageQuotaDoc(db)
+    expect(usageDoc.usageQuota.rows).toEqual(3)
+  })
+})
+

packages/server/src/migrations/usageQuotas/index.js

@@ -0,0 +1,24 @@
+const {
+  MIGRATIONS,
+  MIGRATION_DBS,
+  migrateIfRequired,
+} = require("@budibase/backend-core/migrations")
+const { useQuotas } = require("../../utilities/usageQuota")
+const syncApps = require("./syncApps")
+const syncRows = require("./syncRows")
+
+exports.run = async () => {
+  if (!useQuotas()) {
+    return
+  }
+
+  // Jan 2022
+  await migrateIfRequired(
+    MIGRATION_DBS.GLOBAL_DB,
+    MIGRATIONS.QUOTAS_1,
+    async () => {
+      await syncApps.run()
+      await syncRows.run()
+    }
+  )
+}

packages/server/src/migrations/usageQuotas/syncApps.js

@@ -0,0 +1,18 @@
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
+const { getAllApps } = require("@budibase/backend-core/db")
+const CouchDB = require("../../db")
+const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+
+exports.run = async () => {
+  const db = getGlobalDB()
+  // get app count
+  const devApps = await getAllApps(CouchDB, { dev: true })
+  const appCount = devApps ? devApps.length : 0
+
+  // sync app count
+  const tenantId = getTenantId()
+  console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.apps = appCount
+  await db.put(usageDoc)
+}

packages/server/src/migrations/usageQuotas/syncRows.js

@@ -0,0 +1,21 @@
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
+const { getAllApps } = require("@budibase/backend-core/db")
+const CouchDB = require("../../db")
+const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
+const { getUniqueRows } = require("../../utilities/usageQuota/rows")
+
+exports.run = async () => {
+  const db = getGlobalDB()
+  // get all rows in all apps
+  const allApps = await getAllApps(CouchDB, { all: true })
+  const appIds = allApps ? allApps.map(app => app.appId) : []
+  const rows = await getUniqueRows(appIds)
+  const rowCount = rows ? rows.length : 0
+
+  // sync row count
+  const tenantId = getTenantId()
+  console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.rows = rowCount
+  await db.put(usageDoc)
+}

packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js

@@ -0,0 +1,72 @@
+const getTenantId = jest.fn()
+jest.mock("@budibase/backend-core/tenancy", () => ({
+  getTenantId
+}))
+const usageQuota = require("../../usageQuota")
+const env = require("../../../environment")
+
+class TestConfiguration {
+  constructor() {
+    this.enableQuotas()
+  }
+
+  enableQuotas = () => {
+    env.USE_QUOTAS = 1
+  }
+
+  disableQuotas = () => {
+    env.USE_QUOTAS = null
+  }
+
+  setTenantId = (tenantId) => {
+    getTenantId.mockReturnValue(tenantId)
+  } 
+
+  setExcludedTenants = (tenants) => {
+    env.EXCLUDE_QUOTAS_TENANTS = tenants
+  }
+
+  reset = () => {
+    this.disableQuotas()
+    this.setExcludedTenants(null)
+  }
+}
+
+describe("usageQuota", () => {
+  let config
+
+  beforeEach(() => {
+    config = new TestConfiguration()
+  })
+
+  afterEach(() => {
+    config.reset()
+  })
+
+  describe("useQuotas", () => {
+    it("works when no settings have been provided", () => {
+      config.reset()
+      expect(usageQuota.useQuotas()).toBe(false)
+    })
+    it("honours USE_QUOTAS setting", () => {
+      config.disableQuotas()
+      expect(usageQuota.useQuotas()).toBe(false)
+
+      config.enableQuotas()
+      expect(usageQuota.useQuotas()).toBe(true)
+    })
+    it("honours EXCLUDE_QUOTAS_TENANTS setting", () => {
+      config.setTenantId("test")
+
+      // tenantId is in the list
+      config.setExcludedTenants("test, test2, test2")
+      expect(usageQuota.useQuotas()).toBe(false)
+      config.setExcludedTenants("test,test2,test2")
+      expect(usageQuota.useQuotas()).toBe(false)
+
+      // tenantId is not in the list
+      config.setTenantId("other")
+      expect(usageQuota.useQuotas()).toBe(true)
+    })
+  })
+})

packages/server/src/utilities/usageQuota.js

@@ -1,73 +0,0 @@
-const env = require("../environment")
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const {
-  StaticDatabases,
-  generateNewUsageQuotaDoc,
-} = require("@budibase/backend-core/db")
-
-function getNewQuotaReset() {
-  return Date.now() + 2592000000
-}
-
-exports.Properties = {
-  ROW: "rows",
-  UPLOAD: "storage",
-  VIEW: "views",
-  USER: "users",
-  AUTOMATION: "automationRuns",
-  APPS: "apps",
-  EMAILS: "emails",
-}
-
-async function getUsageQuotaDoc(db) {
-  let quota
-  try {
-    quota = await db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota)
-  } catch (err) {
-    // doc doesn't exist. Create it
-    quota = await db.post(generateNewUsageQuotaDoc())
-  }
-
-  return quota
-}
-
-/**
- * Given a specified tenantId this will add to the usage object for the specified property.
- * @param {string} property The property which is to be added to (within the nested usageQuota object).
- * @param {number} usage The amount (this can be negative) to adjust the number by.
- * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
- * also been reset after this call.
- */
-exports.update = async (property, usage) => {
-  if (!env.USE_QUOTAS) {
-    return
-  }
-
-  try {
-    const db = getGlobalDB()
-    const quota = await getUsageQuotaDoc(db)
-
-    // Check if the quota needs reset
-    if (Date.now() >= quota.quotaReset) {
-      quota.quotaReset = getNewQuotaReset()
-      for (let prop of Object.keys(quota.usageQuota)) {
-        quota.usageQuota[prop] = 0
-      }
-    }
-
-    // increment the quota
-    quota.usageQuota[property] += usage
-
-    if (quota.usageQuota[property] > quota.usageLimits[property]) {
-      throw new Error(
-        `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
-      )
-    }
-
-    // update the usage quotas
-    await db.put(quota)
-  } catch (err) {
-    console.error(`Error updating usage quotas for ${property}`, err)
-    throw err
-  }
-}

packages/server/src/utilities/usageQuota/index.js

@@ -0,0 +1,92 @@
+const env = require("../../environment")
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
+const {
+  StaticDatabases,
+  generateNewUsageQuotaDoc,
+} = require("@budibase/backend-core/db")
+
+exports.useQuotas = () => {
+  // check if quotas are enabled
+  if (env.USE_QUOTAS) {
+    // check if there are any tenants without limits
+    if (env.EXCLUDE_QUOTAS_TENANTS) {
+      const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace(
+        /\s/g,
+        ""
+      ).split(",")
+      const tenantId = getTenantId()
+      if (excludedTenants.includes(tenantId)) {
+        return false
+      }
+    }
+    return true
+  }
+  return false
+}
+
+exports.Properties = {
+  ROW: "rows",
+  UPLOAD: "storage", // doesn't work yet
+  VIEW: "views", // doesn't work yet
+  USER: "users", // doesn't work yet
+  AUTOMATION: "automationRuns", // doesn't work yet
+  APPS: "apps",
+  EMAILS: "emails", // doesn't work yet
+}
+
+exports.getUsageQuotaDoc = async db => {
+  let quota
+  try {
+    quota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota)
+  } catch (err) {
+    // doc doesn't exist. Create it
+    quota = generateNewUsageQuotaDoc()
+    const response = await db.put(quota)
+    quota._rev = response.rev
+  }
+
+  return quota
+}
+
+/**
+ * Given a specified tenantId this will add to the usage object for the specified property.
+ * @param {string} property The property which is to be added to (within the nested usageQuota object).
+ * @param {number} usage The amount (this can be negative) to adjust the number by.
+ * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
+ * also been reset after this call.
+ */
+exports.update = async (property, usage, opts = { dryRun: false }) => {
+  if (!exports.useQuotas()) {
+    return
+  }
+
+  try {
+    const db = getGlobalDB()
+    const quota = await exports.getUsageQuotaDoc(db)
+
+    // increment the quota
+    quota.usageQuota[property] += usage
+
+    if (
+      quota.usageQuota[property] > quota.usageLimits[property] &&
+      usage > 0 // allow for decrementing usage when the quota is already exceeded
+    ) {
+      throw new Error(
+        `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
+      )
+    }
+
+    if (quota.usageQuota[property] < 0) {
+      // never go negative if the quota has previously been exceeded
+      quota.usageQuota[property] = 0
+    }
+
+    // update the usage quotas
+    if (!opts.dryRun) {
+      await db.put(quota)
+    }
+  } catch (err) {
+    console.error(`Error updating usage quotas for ${property}`, err)
+    throw err
+  }
+}

packages/server/src/utilities/usageQuota/rows.js

@@ -0,0 +1,74 @@
+const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils")
+const CouchDB = require("../../db")
+const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db")
+
+const ROW_EXCLUSIONS = [USER_METDATA_PREFIX]
+
+const getAppPairs = appIds => {
+  // collect the app ids into dev / prod pairs
+  // keyed by the dev app id
+  const pairs = {}
+  for (let appId of appIds) {
+    const devId = getDevelopmentAppID(appId)
+    if (!pairs[devId]) {
+      pairs[devId] = {}
+    }
+    if (isDevAppID(appId)) {
+      pairs[devId].devId = appId
+    } else {
+      pairs[devId].prodId = appId
+    }
+  }
+  return pairs
+}
+
+const getAppRows = async appId => {
+  const appDb = new CouchDB(appId)
+  const response = await appDb.allDocs(
+    getRowParams(null, null, {
+      include_docs: false,
+    })
+  )
+  return response.rows
+    .map(r => r.id)
+    .filter(id => {
+      for (let exclusion of ROW_EXCLUSIONS) {
+        if (id.startsWith(exclusion)) {
+          return false
+        }
+      }
+      return true
+    })
+}
+
+/**
+ * Return a set of all rows in the given app ids.
+ * The returned rows will be unique on a per dev/prod app basis.
+ * Rows duplicates may exist across apps due to data import so they are not filtered out.
+ */
+exports.getUniqueRows = async appIds => {
+  let uniqueRows = []
+  const pairs = getAppPairs(appIds)
+
+  for (let pair of Object.values(pairs)) {
+    let appRows = []
+    for (let appId of [pair.devId, pair.prodId]) {
+      if (!appId) {
+        continue
+      }
+      try {
+        appRows.push(await getAppRows(appId))
+      } catch (e) {
+        console.error(e)
+        // don't error out if we can't count the app rows, just continue
+      }
+    }
+
+    // ensure uniqueness on a per app pair basis
+    // this can't be done on all rows because app import results in
+    // duplicate row ids across apps
+    uniqueRows = uniqueRows.concat(...new Set(appRows))
+  }
+
+  return uniqueRows
+}

packages/server/src/utilities/usageQuota/usageQuoteReset.js

@@ -0,0 +1,18 @@
+// UNUSED CODE
+// Preserved for future use
+
+/* eslint-disable no-unused-vars */
+
+function getNewQuotaReset() {
+  return Date.now() + 2592000000
+}
+
+function resetQuotasIfRequired(quota) {
+  // Check if the quota needs reset
+  if (Date.now() >= quota.quotaReset) {
+    quota.quotaReset = getNewQuotaReset()
+    for (let prop of Object.keys(quota.usageQuota)) {
+      quota.usageQuota[prop] = 0
+    }
+  }
+}

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "1.0.46-alpha.2",
+  "version": "1.0.46-alpha.3",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.0.46-alpha.2",
+  "version": "1.0.46-alpha.3",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -29,8 +29,8 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "^1.0.46-alpha.2",
-    "@budibase/string-templates": "^1.0.46-alpha.2",
+    "@budibase/backend-core": "^1.0.46-alpha.3",
+    "@budibase/string-templates": "^1.0.46-alpha.3",
     "@koa/router": "^8.0.0",
     "@sentry/node": "^6.0.0",
     "@techpass/passport-openidconnect": "^0.3.0",

packages/worker/src/api/controllers/global/users.js

@@ -73,16 +73,14 @@ exports.adminUser = async ctx => {
   if (!env.SELF_HOSTED) {
     // could be a scenario where it exists, make sure its clean
     try {
-      const usageQuota = await db.get(
-        StaticDatabases.PLATFORM_INFO.docs.usageQuota
-      )
+      const usageQuota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota)
       if (usageQuota) {
         await db.remove(usageQuota._id, usageQuota._rev)
       }
     } catch (err) {
       // don't worry about errors
     }
-    await db.post(generateNewUsageQuotaDoc())
+    await db.put(generateNewUsageQuotaDoc())
   }
 
   if (response.rows.some(row => row.doc.admin)) {