Integrate usage quotas with licensing

packages/backend-core/src/db/utils.js

@@ -425,33 +425,8 @@ async function getScopedConfig(db, params) {
   return configDoc && configDoc.config ? configDoc.config : configDoc
 }
 
-function generateNewUsageQuotaDoc() {
-  return {
-    _id: StaticDatabases.GLOBAL.docs.usageQuota,
-    quotaReset: Date.now() + 2592000000,
-    usageQuota: {
-      automationRuns: 0,
-      rows: 0,
-      storage: 0,
-      apps: 0,
-      users: 0,
-      views: 0,
-      emails: 0,
-    },
-    usageLimits: {
-      automationRuns: 1000,
-      rows: 4000,
-      apps: 4,
-      storage: 1000,
-      users: 10,
-      emails: 50,
-    },
-  }
-}
-
 exports.Replication = Replication
 exports.getScopedConfig = getScopedConfig
 exports.generateConfigID = generateConfigID
 exports.getConfigParams = getConfigParams
 exports.getScopedFullConfig = getScopedFullConfig
-exports.generateNewUsageQuotaDoc = generateNewUsageQuotaDoc

packages/backend-core/src/environment.js

@@ -28,6 +28,8 @@ module.exports = {
   SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
   COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
   PLATFORM_URL: process.env.PLATFORM_URL,
+  USE_QUOTAS: process.env.USE_QUOTAS,
+  EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS,
   isTest,
   _set(key, value) {
     process.env[key] = value

packages/server/nodemon.json

@@ -1,5 +1,5 @@
 {
-  "watch": ["src", "../backend-core"],
+  "watch": ["src", "../backend-core", "../../../budibase-pro/packages/pro"],
   "ext": "js,ts,json",
   "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js"],
   "exec": "ts-node src/index.ts"

packages/server/src/api/controllers/table/internal.js

@@ -7,7 +7,7 @@ const {
   getTable,
   handleDataImport,
 } = require("./utils")
-const usageQuota = require("../../../utilities/usageQuota")
+const { quotas, StaticQuotaName, QuotaUsageType } = require("@budibase/pro")
 const { getAppDB } = require("@budibase/backend-core/context")
 const env = require("../../../environment")
 const { cleanupAttachments } = require("../../../utilities/rowProcessor")
@@ -120,7 +120,11 @@ exports.destroy = async function (ctx) {
     })
   )
   await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
-  await usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length)
+  await quotas.updateUsage(
+    -rows.rows.length,
+    StaticQuotaName.ROWS,
+    QuotaUsageType.STATIC
+  )
 
   // update linked rows
   await linkRows.updateLinks({

packages/server/src/api/controllers/table/utils.js

@@ -24,7 +24,7 @@ const {
 } = require("../../../integrations/utils")
 const { getViews, saveView } = require("../view/utils")
 const viewTemplate = require("../view/viewBuilder")
-const usageQuota = require("../../../utilities/usageQuota")
+const { quotas, StaticQuotaName, QuotaUsageType } = require("@budibase/pro")
 const { getAppDB } = require("@budibase/backend-core/context")
 const { cloneDeep } = require("lodash/fp")
 
@@ -143,11 +143,20 @@ exports.handleDataImport = async (user, table, dataImport) => {
     finalData.push(row)
   }
 
-  await usageQuota.update(usageQuota.Properties.ROW, finalData.length, {
+  await quotas.updateUsage(
+    finalData.length,
+    StaticQuotaName.ROWS,
+    QuotaUsageType.STATIC,
+    {
       dryRun: true,
-  })
+    }
+  )
   await db.bulkDocs(finalData)
-  await usageQuota.update(usageQuota.Properties.ROW, finalData.length)
+  await quotas.updateUsage(
+    finalData.length,
+    StaticQuotaName.ROWS,
+    QuotaUsageType.STATIC
+  )
   let response = await db.put(table)
   table._rev = response._rev
   return table

packages/server/src/app.ts

@@ -1,12 +1,6 @@
 // need to load environment first
 import { ExtendableContext } from "koa"
-
 import * as env from "./environment"
-
-// temp for testing
-import * as poc from "./pro-poc"
-poc.run()
-
 const CouchDB = require("./db")
 require("@budibase/backend-core").init(CouchDB)
 const Koa = require("koa")

packages/server/src/automations/steps/createRow.js

@@ -1,6 +1,6 @@
 const rowController = require("../../api/controllers/row")
 const automationUtils = require("../automationUtils")
-const usage = require("../../utilities/usageQuota")
+const { quotas, StaticQuotaName, QuotaUsageType } = require("@budibase/pro")
 const { buildCtx } = require("./utils")
 
 exports.definition = {
@@ -81,9 +81,11 @@ exports.run = async function ({ inputs, appId, emitter }) {
       inputs.row.tableId,
       inputs.row
     )
-    await usage.update(usage.Properties.ROW, 1, { dryRun: true })
+    await quotas.updateUsage(1, StaticQuotaName.ROWS, QuotaUsageType.STATIC, {
+      dryRun: true,
+    })
     await rowController.save(ctx)
-    await usage.update(usage.Properties.ROW, 1)
+    await quotas.updateUsage(1, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
     return {
       row: inputs.row,
       response: ctx.body,

packages/server/src/automations/steps/deleteRow.js

@@ -1,5 +1,5 @@
 const rowController = require("../../api/controllers/row")
-const usage = require("../../utilities/usageQuota")
+const { quotas, StaticQuotaName, QuotaUsageType } = require("@budibase/pro")
 const { buildCtx } = require("./utils")
 const automationUtils = require("../automationUtils")
 
@@ -73,7 +73,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
   })
 
   try {
-    await usage.update(usage.Properties.ROW, -1)
+    await quotas.updateUsage(-1, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
     await rowController.destroy(ctx)
     return {
       response: ctx.body,

packages/server/src/middleware/usageQuota.js

@@ -1,164 +0,0 @@
-const usageQuota = require("../utilities/usageQuota")
-const { getUniqueRows } = require("../utilities/usageQuota/rows")
-const {
-  isExternalTable,
-  isRowId: isExternalRowId,
-} = require("../integrations/utils")
-const { getAppDB } = require("@budibase/backend-core/context")
-
-// currently only counting new writes and deletes
-const METHOD_MAP = {
-  POST: 1,
-  DELETE: -1,
-}
-
-const DOMAIN_MAP = {
-  rows: usageQuota.Properties.ROW,
-  // upload: usageQuota.Properties.UPLOAD,            // doesn't work yet
-  // views: usageQuota.Properties.VIEW,               // doesn't work yet
-  // users: usageQuota.Properties.USER,               // doesn't work yet
-  applications: usageQuota.Properties.APPS,
-  // this will not be updated by endpoint calls
-  // instead it will be updated by triggerInfo
-  // automationRuns: usageQuota.Properties.AUTOMATION, // doesn't work yet
-}
-
-function getProperty(url) {
-  for (let domain of Object.keys(DOMAIN_MAP)) {
-    if (url.indexOf(domain) !== -1) {
-      return DOMAIN_MAP[domain]
-    }
-  }
-}
-
-module.exports = async (ctx, next) => {
-  if (!usageQuota.useQuotas()) {
-    return next()
-  }
-
-  let usage = METHOD_MAP[ctx.req.method]
-  const property = getProperty(ctx.req.url)
-  if (usage == null || property == null) {
-    return next()
-  }
-  // post request could be a save of a pre-existing entry
-  if (ctx.request.body && ctx.request.body._id && ctx.request.body._rev) {
-    const usageId = ctx.request.body._id
-    try {
-      if (ctx.appId) {
-        const db = getAppDB()
-        await db.get(usageId)
-      }
-      return next()
-    } catch (err) {
-      if (
-        isExternalTable(usageId) ||
-        (ctx.request.body.tableId &&
-          isExternalTable(ctx.request.body.tableId)) ||
-        isExternalRowId(usageId)
-      ) {
-        return next()
-      } else {
-        ctx.throw(404, `${usageId} does not exist`)
-      }
-    }
-  }
-
-  // update usage for uploads to be the total size
-  if (property === usageQuota.Properties.UPLOAD) {
-    const files =
-      ctx.request.files.file.length > 1
-        ? Array.from(ctx.request.files.file)
-        : [ctx.request.files.file]
-    usage = files.map(file => file.size).reduce((total, size) => total + size)
-  }
-  try {
-    await performRequest(ctx, next, property, usage)
-  } catch (err) {
-    ctx.throw(400, err)
-  }
-}
-
-const performRequest = async (ctx, next, property, usage) => {
-  const usageContext = {
-    skipNext: false,
-    skipUsage: false,
-    [usageQuota.Properties.APPS]: {},
-  }
-
-  if (usage === -1) {
-    if (PRE_DELETE[property]) {
-      await PRE_DELETE[property](ctx, usageContext)
-    }
-  } else {
-    if (PRE_CREATE[property]) {
-      await PRE_CREATE[property](ctx, usageContext)
-    }
-  }
-
-  // run the request
-  if (!usageContext.skipNext) {
-    await usageQuota.update(property, usage, { dryRun: true })
-    await next()
-  }
-
-  if (usage === -1) {
-    if (POST_DELETE[property]) {
-      await POST_DELETE[property](ctx, usageContext)
-    }
-  } else {
-    if (POST_CREATE[property]) {
-      await POST_CREATE[property](ctx, usageContext)
-    }
-  }
-
-  // update the usage
-  if (!usageContext.skipUsage) {
-    await usageQuota.update(property, usage)
-  }
-}
-
-const appPreDelete = async (ctx, usageContext) => {
-  if (ctx.query.unpublish) {
-    // don't run usage decrement for unpublish
-    usageContext.skipUsage = true
-    return
-  }
-
-  // store the row count to delete
-  const rows = await getUniqueRows([ctx.appId])
-  if (rows.length) {
-    usageContext[usageQuota.Properties.APPS] = { rowCount: rows.length }
-  }
-}
-
-const appPostDelete = async (ctx, usageContext) => {
-  // delete the app rows from usage
-  const rowCount = usageContext[usageQuota.Properties.APPS].rowCount
-  if (rowCount) {
-    await usageQuota.update(usageQuota.Properties.ROW, -rowCount)
-  }
-}
-
-const appPostCreate = async ctx => {
-  // app import & template creation
-  if (ctx.request.body.useTemplate === "true") {
-    const rows = await getUniqueRows([ctx.response.body.appId])
-    const rowCount = rows ? rows.length : 0
-    await usageQuota.update(usageQuota.Properties.ROW, rowCount)
-  }
-}
-
-const PRE_DELETE = {
-  [usageQuota.Properties.APPS]: appPreDelete,
-}
-
-const POST_DELETE = {
-  [usageQuota.Properties.APPS]: appPostDelete,
-}
-
-const PRE_CREATE = {}
-
-const POST_CREATE = {
-  [usageQuota.Properties.APPS]: appPostCreate,
-}

packages/server/src/middleware/usageQuota.ts

@@ -0,0 +1,170 @@
+import { quotas, StaticQuotaName, QuotaUsageType } from "@budibase/pro"
+const { getUniqueRows } = require("../utilities/usageQuota/rows")
+const {
+  isExternalTable,
+  isRowId: isExternalRowId,
+} = require("../integrations/utils")
+const { getAppDB } = require("@budibase/backend-core/context")
+
+// currently only counting new writes and deletes
+const METHOD_MAP: any = {
+  POST: 1,
+  DELETE: -1,
+}
+
+const DOMAIN_MAP: any = {
+  rows: {
+    name: StaticQuotaName.ROWS,
+    type: QuotaUsageType.STATIC,
+  },
+  applications: {
+    name: StaticQuotaName.APPS,
+    type: QuotaUsageType.STATIC,
+  },
+}
+
+function getQuotaInfo(url: string) {
+  for (let domain of Object.keys(DOMAIN_MAP)) {
+    if (url.indexOf(domain) !== -1) {
+      return DOMAIN_MAP[domain]
+    }
+  }
+}
+
+module.exports = async (ctx: any, next: any) => {
+  if (!quotas.useQuotas()) {
+    return next()
+  }
+
+  let usage = METHOD_MAP[ctx.req.method]
+  const quotaInfo = getQuotaInfo(ctx.req.url)
+  if (usage == null || quotaInfo == null) {
+    return next()
+  }
+  // post request could be a save of a pre-existing entry
+  if (ctx.request.body && ctx.request.body._id && ctx.request.body._rev) {
+    const usageId = ctx.request.body._id
+    try {
+      if (ctx.appId) {
+        const db = getAppDB()
+        await db.get(usageId)
+      }
+      return next()
+    } catch (err) {
+      if (
+        isExternalTable(usageId) ||
+        (ctx.request.body.tableId &&
+          isExternalTable(ctx.request.body.tableId)) ||
+        isExternalRowId(usageId)
+      ) {
+        return next()
+      } else {
+        ctx.throw(404, `${usageId} does not exist`)
+      }
+    }
+  }
+  try {
+    await performRequest(ctx, next, quotaInfo, usage)
+  } catch (err) {
+    ctx.throw(400, err)
+  }
+}
+
+const performRequest = async (
+  ctx: any,
+  next: any,
+  quotaInfo: any,
+  usage: number
+) => {
+  const usageContext = {
+    skipNext: false,
+    skipUsage: false,
+    [StaticQuotaName.APPS]: {},
+  }
+
+  const quotaName = quotaInfo.name
+
+  if (usage === -1) {
+    if (PRE_DELETE[quotaName]) {
+      await PRE_DELETE[quotaName](ctx, usageContext)
+    }
+  } else {
+    if (PRE_CREATE[quotaName]) {
+      await PRE_CREATE[quotaName](ctx, usageContext)
+    }
+  }
+
+  // run the request
+  if (!usageContext.skipNext) {
+    await quotas.updateUsage(usage, quotaName, quotaInfo.type, { dryRun: true })
+    await next()
+  }
+
+  if (usage === -1) {
+    if (POST_DELETE[quotaName]) {
+      await POST_DELETE[quotaName](ctx, usageContext)
+    }
+  } else {
+    if (POST_CREATE[quotaName]) {
+      await POST_CREATE[quotaName](ctx)
+    }
+  }
+
+  // update the usage
+  if (!usageContext.skipUsage) {
+    await quotas.updateUsage(usage, quotaName, quotaInfo.type)
+  }
+}
+
+const appPreDelete = async (ctx: any, usageContext: any) => {
+  if (ctx.query.unpublish) {
+    // don't run usage decrement for unpublish
+    usageContext.skipUsage = true
+    return
+  }
+
+  // store the row count to delete
+  const rows = await getUniqueRows([ctx.appId])
+  if (rows.length) {
+    usageContext[StaticQuotaName.APPS] = { rowCount: rows.length }
+  }
+}
+
+const appPostDelete = async (ctx: any, usageContext: any) => {
+  // delete the app rows from usage
+  const rowCount = usageContext[StaticQuotaName.ROWS].rowCount
+  if (rowCount) {
+    await quotas.updateUsage(
+      -rowCount,
+      StaticQuotaName.ROWS,
+      QuotaUsageType.STATIC
+    )
+  }
+}
+
+const appPostCreate = async (ctx: any) => {
+  // app import & template creation
+  if (ctx.request.body.useTemplate === "true") {
+    const rows = await getUniqueRows([ctx.response.body.appId])
+    const rowCount = rows ? rows.length : 0
+    await quotas.updateUsage(
+      rowCount,
+      StaticQuotaName.ROWS,
+      QuotaUsageType.STATIC
+    )
+  }
+}
+
+const PRE_DELETE: any = {
+  [StaticQuotaName.APPS]: appPreDelete,
+}
+
+const POST_DELETE: any = {
+  [StaticQuotaName.APPS]: appPostDelete,
+}
+
+const PRE_CREATE: any = {}
+
+const POST_CREATE: any = {
+  [StaticQuotaName.APPS]: appPostCreate,
+}

packages/server/src/migrations/functions/usageQuotas/index.ts

@@ -1,7 +1,7 @@
-const { useQuotas } = require("../../../utilities/usageQuota")
+import { quotas } from "@budibase/pro"
 
 export const runQuotaMigration = async (migration: Function) => {
-  if (!useQuotas()) {
+  if (!quotas.useQuotas()) {
     return
   }
   await migration()

packages/server/src/migrations/functions/usageQuotas/syncApps.ts

@@ -1,9 +1,8 @@
-import { getGlobalDB, getTenantId } from "@budibase/backend-core/tenancy"
+import { getTenantId } from "@budibase/backend-core/tenancy"
 import { getAllApps } from "@budibase/backend-core/db"
-import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
 
 export const run = async () => {
-  const db = getGlobalDB()
   // get app count
   // @ts-ignore
   const devApps = await getAllApps({ dev: true })
@@ -12,7 +11,5 @@ export const run = async () => {
   // sync app count
   const tenantId = getTenantId()
   console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
-  const usageDoc = await getUsageQuotaDoc(db)
+  await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
-  usageDoc.usageQuota.apps = appCount
-  await db.put(usageDoc)
 }

packages/server/src/migrations/functions/usageQuotas/syncRows.ts

@@ -1,10 +1,9 @@
-import { getGlobalDB, getTenantId } from "@budibase/backend-core/tenancy"
+import { getTenantId } from "@budibase/backend-core/tenancy"
 import { getAllApps } from "@budibase/backend-core/db"
-import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
 import { getUniqueRows } from "../../../utilities/usageQuota/rows"
 
 export const run = async () => {
-  const db = getGlobalDB()
   // get all rows in all apps
   // @ts-ignore
   const allApps = await getAllApps({ all: true })
@@ -16,7 +15,5 @@ export const run = async () => {
   // sync row count
   const tenantId = getTenantId()
   console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
-  const usageDoc = await getUsageQuotaDoc(db)
+  await quotas.setUsage(rowCount, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
-  usageDoc.usageQuota.rows = rowCount
-  await db.put(usageDoc)
 }

packages/server/src/migrations/index.ts

@@ -28,7 +28,7 @@ export interface Migration {
  */
 export interface MigrationOptions {
   tenantIds?: string[]
-  forced?: {
+  force?: {
     [type: string]: string[]
   }
 }

packages/server/src/threads/automation.js

@@ -1,13 +1,11 @@
 require("./utils").threadSetup()
-const env = require("../environment")
 const actions = require("../automations/actions")
 const automationUtils = require("../automations/automationUtils")
 const AutomationEmitter = require("../events/AutomationEmitter")
 const { processObject } = require("@budibase/string-templates")
 const { DEFAULT_TENANT_ID } = require("@budibase/backend-core/constants")
-const { DocumentTypes, isDevAppID } = require("../db/utils")
+const { DocumentTypes } = require("../db/utils")
 const { doInTenant } = require("@budibase/backend-core/tenancy")
-const usage = require("../utilities/usageQuota")
 const { definitions: triggerDefs } = require("../automations/triggerInfo")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
 
@@ -120,11 +118,6 @@ class Orchestrator {
         return err
       }
     }
-
-    // Increment quota for automation runs
-    if (!env.SELF_HOSTED && !isDevAppID(this._appId)) {
-      await usage.update(usage.Properties.AUTOMATION, 1)
-    }
     return this.executionOutput
   }
 }

packages/server/src/utilities/usageQuota/index.js

@@ -1,93 +0,0 @@
-const env = require("../../environment")
-const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
-const {
-  StaticDatabases,
-  generateNewUsageQuotaDoc,
-} = require("@budibase/backend-core/db")
-
-exports.useQuotas = () => {
-  // check if quotas are enabled
-  if (env.USE_QUOTAS) {
-    // check if there are any tenants without limits
-    if (env.EXCLUDE_QUOTAS_TENANTS) {
-      const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace(
-        /\s/g,
-        ""
-      ).split(",")
-      const tenantId = getTenantId()
-      if (excludedTenants.includes(tenantId)) {
-        return false
-      }
-    }
-    return true
-  }
-  return false
-}
-
-exports.Properties = {
-  ROW: "rows",
-  UPLOAD: "storage", // doesn't work yet
-  VIEW: "views", // doesn't work yet
-  USER: "users", // doesn't work yet
-  AUTOMATION: "automationRuns", // doesn't work yet
-  APPS: "apps",
-  EMAILS: "emails", // doesn't work yet
-}
-
-exports.getUsageQuotaDoc = async db => {
-  let quota
-  try {
-    quota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota)
-  } catch (err) {
-    // doc doesn't exist. Create it
-    quota = generateNewUsageQuotaDoc()
-    const response = await db.put(quota)
-    quota._rev = response.rev
-  }
-
-  return quota
-}
-
-/**
- * Given a specified tenantId this will add to the usage object for the specified property.
- * @param {string} property The property which is to be added to (within the nested usageQuota object).
- * @param {number} usage The amount (this can be negative) to adjust the number by.
- * @param {object} opts optional - options such as dryRun, to check what update will do.
- * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
- * also been reset after this call.
- */
-exports.update = async (property, usage, opts = { dryRun: false }) => {
-  if (!exports.useQuotas()) {
-    return
-  }
-
-  try {
-    const db = getGlobalDB()
-    const quota = await exports.getUsageQuotaDoc(db)
-
-    // increment the quota
-    quota.usageQuota[property] += usage
-
-    if (
-      quota.usageQuota[property] > quota.usageLimits[property] &&
-      usage > 0 // allow for decrementing usage when the quota is already exceeded
-    ) {
-      throw new Error(
-        `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
-      )
-    }
-
-    if (quota.usageQuota[property] < 0) {
-      // never go negative if the quota has previously been exceeded
-      quota.usageQuota[property] = 0
-    }
-
-    // update the usage quotas
-    if (!opts.dryRun) {
-      await db.put(quota)
-    }
-  } catch (err) {
-    console.error(`Error updating usage quotas for ${property}`, err)
-    throw err
-  }
-}