Merge pull request #2633 from Budibase/cloud-limits

Cloud limits

packages/auth/src/db/constants.js

@@ -12,6 +12,7 @@ exports.StaticDatabases = {
     name: "global-info",
     docs: {
       tenants: "tenants",
+      usageQuota: "usage_quota",
     },
   },
 }

packages/auth/src/db/utils.js

@@ -368,8 +368,33 @@ async function getScopedConfig(db, params) {
   return configDoc && configDoc.config ? configDoc.config : configDoc
 }
 
+function generateNewUsageQuotaDoc() {
+  return {
+    _id: StaticDatabases.PLATFORM_INFO.docs.usageQuota,
+    quotaReset: Date.now() + 2592000000,
+    usageQuota: {
+      automationRuns: 0,
+      rows: 0,
+      storage: 0,
+      apps: 0,
+      users: 0,
+      views: 0,
+      emails: 0,
+    },
+    usageLimits: {
+      automationRuns: 1000,
+      rows: 4000,
+      apps: 4,
+      storage: 1000,
+      users: 10,
+      emails: 50,
+    },
+  }
+}
+
 exports.Replication = Replication
 exports.getScopedConfig = getScopedConfig
 exports.generateConfigID = generateConfigID
 exports.getConfigParams = getConfigParams
 exports.getScopedFullConfig = getScopedFullConfig
+exports.generateNewUsageQuotaDoc = generateNewUsageQuotaDoc

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -546,7 +546,7 @@ module External {
         },
         meta: {
           table,
-        }
+        },
       }
       // can't really use response right now
       const response = await makeExternalQuery(appId, json)

packages/server/src/api/routes/application.js

@@ -2,11 +2,12 @@ const Router = require("@koa/router")
 const controller = require("../controllers/application")
 const authorized = require("../../middleware/authorized")
 const { BUILDER } = require("@budibase/auth/permissions")
+const usage = require("../../middleware/usageQuota")
 
 const router = Router()
 
 router
-  .post("/api/applications", authorized(BUILDER), controller.create)
+  .post("/api/applications", authorized(BUILDER), usage, controller.create)
   .get("/api/applications/:appId/definition", controller.fetchAppDefinition)
   .get("/api/applications", controller.fetch)
   .get("/api/applications/:appId/appPackage", controller.fetchAppPackage)
@@ -21,6 +22,11 @@ router
     authorized(BUILDER),
     controller.revertClient
   )
-  .delete("/api/applications/:appId", authorized(BUILDER), controller.delete)
+  .delete(
+    "/api/applications/:appId",
+    authorized(BUILDER),
+    usage,
+    controller.delete
+  )
 
 module.exports = router

packages/server/src/api/routes/user.js

@@ -5,7 +5,6 @@ const {
   PermissionLevels,
   PermissionTypes,
 } = require("@budibase/auth/permissions")
-const usage = require("../../middleware/usageQuota")
 
 const router = Router()
 
@@ -28,13 +27,11 @@ router
   .post(
     "/api/users/metadata/self",
     authorized(PermissionTypes.USER, PermissionLevels.WRITE),
-    usage,
     controller.updateSelfMetadata
   )
   .delete(
     "/api/users/metadata/:id",
     authorized(PermissionTypes.USER, PermissionLevels.WRITE),
-    usage,
     controller.destroyMetadata
   )
 

packages/server/src/api/routes/view.js

@@ -8,7 +8,6 @@ const {
   PermissionTypes,
   PermissionLevels,
 } = require("@budibase/auth/permissions")
-const usage = require("../../middleware/usageQuota")
 
 const router = Router()
 
@@ -25,9 +24,8 @@ router
     "/api/views/:viewName",
     paramResource("viewName"),
     authorized(BUILDER),
-    usage,
     viewController.destroy
   )
-  .post("/api/views", authorized(BUILDER), usage, viewController.save)
+  .post("/api/views", authorized(BUILDER), viewController.save)
 
 module.exports = router

packages/server/src/automations/steps/createRow.js

@@ -60,7 +60,7 @@ exports.definition = {
   },
 }
 
-exports.run = async function ({ inputs, appId, apiKey, emitter }) {
+exports.run = async function ({ inputs, appId, emitter }) {
   if (inputs.row == null || inputs.row.tableId == null) {
     return {
       success: false,
@@ -84,7 +84,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) {
       inputs.row
     )
     if (env.USE_QUOTAS) {
-      await usage.update(apiKey, usage.Properties.ROW, 1)
+      await usage.update(usage.Properties.ROW, 1)
     }
     await rowController.save(ctx)
     return {

packages/server/src/automations/steps/deleteRow.js

@@ -52,7 +52,7 @@ exports.definition = {
   },
 }
 
-exports.run = async function ({ inputs, appId, apiKey, emitter }) {
+exports.run = async function ({ inputs, appId, emitter }) {
   if (inputs.id == null || inputs.revision == null) {
     return {
       success: false,
@@ -74,7 +74,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) {
 
   try {
     if (env.isProd()) {
-      await usage.update(apiKey, usage.Properties.ROW, -1)
+      await usage.update(usage.Properties.ROW, -1)
     }
     await rowController.destroy(ctx)
     return {

packages/server/src/automations/steps/sendSmtpEmail.js

@@ -53,7 +53,7 @@ exports.run = async function ({ inputs }) {
     contents = "<h1>No content</h1>"
   }
   try {
-    let response = await sendSmtpEmail(to, from, subject, contents)
+    let response = await sendSmtpEmail(to, from, subject, contents, true)
     return {
       success: true,
       response,

packages/server/src/automations/tests/automation.spec.js

@@ -13,8 +13,6 @@ const { makePartial } = require("../../tests/utilities")
 const { cleanInputValues } = require("../automationUtils")
 const setup = require("./utilities")
 
-usageQuota.getAPIKey.mockReturnValue({ apiKey: "test" })
-
 describe("Run through some parts of the automations system", () => {
   let config = setup.getConfig()
 

packages/server/src/automations/tests/createRow.spec.js

@@ -46,7 +46,7 @@ describe("test the create row action", () => {
       await setup.runStep(setup.actions.CREATE_ROW.stepId, {
         row
       })
-      expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", 1)
+      expect(usageQuota.update).toHaveBeenCalledWith("rows", 1)
     })
   })
 

packages/server/src/automations/tests/deleteRow.spec.js

@@ -37,7 +37,7 @@ describe("test the delete row action", () => {
   it("check usage quota attempts", async () => {
     await setup.runInProd(async () => {
       await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs)
-      expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", -1)
+      expect(usageQuota.update).toHaveBeenCalledWith("rows", -1)
     })
   })
 

packages/server/src/automations/thread.js

@@ -4,8 +4,10 @@ const AutomationEmitter = require("../events/AutomationEmitter")
 const { processObject } = require("@budibase/string-templates")
 const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants
 const CouchDB = require("../db")
-const { DocumentTypes } = require("../db/utils")
+const { DocumentTypes, isDevAppID } = require("../db/utils")
 const { doInTenant } = require("@budibase/auth/tenancy")
+const env = require("../environment")
+const usage = require("../utilities/usageQuota")
 
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
 
@@ -80,7 +82,6 @@ class Orchestrator {
           return stepFn({
             inputs: step.inputs,
             appId: this._appId,
-            apiKey: automation.apiKey,
             emitter: this._emitter,
             context: this._context,
           })
@@ -95,6 +96,11 @@ class Orchestrator {
         return err
       }
     }
+
+    // Increment quota for automation runs
+    if (!env.SELF_HOSTED && !isDevAppID(this._appId)) {
+      usage.update(usage.Properties.AUTOMATION, 1)
+    }
     return this.executionOutput
   }
 }

packages/server/src/definitions/datasource.ts

@@ -1,4 +1,4 @@
-import {Table} from "./common";
+import { Table } from "./common"
 
 export enum Operation {
   CREATE = "CREATE",
@@ -139,7 +139,7 @@ export interface QueryJson {
   paginate?: PaginationJson
   body?: object
   meta?: {
-    table?: Table,
+    table?: Table
   }
   extra?: {
     idFilter?: SearchFilters

packages/server/src/integrations/mysql.ts

@@ -12,7 +12,11 @@ import { getSqlQuery } from "./utils"
 module MySQLModule {
   const mysql = require("mysql")
   const Sql = require("./base/sql")
-  const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils")
+  const {
+    buildExternalTableId,
+    convertType,
+    copyExistingPropsOver,
+  } = require("./utils")
   const { FieldTypes } = require("../constants")
 
   interface MySQLConfig {
@@ -248,9 +252,9 @@ module MySQLModule {
       json.extra = {
         idFilter: {
           equal: {
-            [primaryKey]: results.insertId
+            [primaryKey]: results.insertId,
+          },
         },
-        }
       }
       return json
     }

packages/server/src/integrations/postgres.ts

@@ -12,7 +12,11 @@ module PostgresModule {
   const { Pool } = require("pg")
   const Sql = require("./base/sql")
   const { FieldTypes } = require("../constants")
-  const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils")
+  const {
+    buildExternalTableId,
+    convertType,
+    copyExistingPropsOver,
+  } = require("./utils")
   const { escapeDangerousCharacters } = require("../utilities")
 
   const JSON_REGEX = /'{.*}'::json/s
@@ -193,10 +197,16 @@ module PostgresModule {
         }
 
         const type: string = convertType(column.data_type, TYPE_MAP)
-        const identity = !!(column.identity_generation || column.identity_start || column.identity_increment)
+        const identity = !!(
-        const hasDefault = typeof column.column_default === "string" &&
+          column.identity_generation ||
+          column.identity_start ||
+          column.identity_increment
+        )
+        const hasDefault =
+          typeof column.column_default === "string" &&
           column.column_default.startsWith("nextval")
-        const isGenerated = column.is_generated && column.is_generated !== "NEVER"
+        const isGenerated =
+          column.is_generated && column.is_generated !== "NEVER"
         const isAuto: boolean = hasDefault || identity || isGenerated
         tables[tableName].schema[columnName] = {
           autocolumn: isAuto,

packages/server/src/integrations/utils.ts

@@ -84,7 +84,11 @@ export function isIsoDateString(str: string) {
 }
 
 // add the existing relationships from the entities if they exist, to prevent them from being overridden
-export function copyExistingPropsOver(tableName: string, tables: { [key: string]: any }, entities: { [key: string]: any }) {
+export function copyExistingPropsOver(
+  tableName: string,
+  tables: { [key: string]: any },
+  entities: { [key: string]: any }
+) {
   if (entities && entities[tableName]) {
     if (entities[tableName].primaryDisplay) {
       tables[tableName].primaryDisplay = entities[tableName].primaryDisplay

packages/server/src/middleware/tests/usageQuota.spec.js

@@ -39,7 +39,7 @@ class TestConfiguration {
     if (bool) {
       env.isDev = () => false
       env.isProd = () => true
-      this.ctx.auth = { apiKey: "test" }
+      this.ctx.user = { tenantId: "test" }
     } else {
       env.isDev = () => true
       env.isProd = () => false
@@ -114,7 +114,7 @@ describe("usageQuota middleware", () => {
 
     await config.executeMiddleware()
 
-    expect(usageQuota.update).toHaveBeenCalledWith("test", "rows", 1)
+    expect(usageQuota.update).toHaveBeenCalledWith("rows", 1)
     expect(config.next).toHaveBeenCalled()
   })
 
@@ -131,7 +131,7 @@ describe("usageQuota middleware", () => {
     ])
     await config.executeMiddleware()
 
-    expect(usageQuota.update).toHaveBeenCalledWith("test", "storage", 10100)
+    expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
     expect(config.next).toHaveBeenCalled()
   })
 })

packages/server/src/middleware/usageQuota.js

@@ -13,6 +13,7 @@ const DOMAIN_MAP = {
   upload: usageQuota.Properties.UPLOAD,
   views: usageQuota.Properties.VIEW,
   users: usageQuota.Properties.USER,
+  applications: usageQuota.Properties.APPS,
   // this will not be updated by endpoint calls
   // instead it will be updated by triggerInfo
   automationRuns: usageQuota.Properties.AUTOMATION,
@@ -57,9 +58,9 @@ module.exports = async (ctx, next) => {
     usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
   try {
-    await usageQuota.update(ctx.auth.apiKey, property, usage)
+    await usageQuota.update(property, usage)
     return next()
   } catch (err) {
-    ctx.throw(403, err)
+    ctx.throw(400, err)
   }
 }

packages/server/src/utilities/usageQuota.js

@@ -1,41 +1,9 @@
 const env = require("../environment")
-const { apiKeyTable } = require("../db/dynamoClient")
+const { getGlobalDB } = require("@budibase/auth/tenancy")
-
+const {
-const DEFAULT_USAGE = {
+  StaticDatabases,
-  rows: 0,
+  generateNewUsageQuotaDoc,
-  storage: 0,
+} = require("@budibase/auth/db")
-  views: 0,
-  automationRuns: 0,
-  users: 0,
-}
-
-const DEFAULT_PLAN = {
-  rows: 1000,
-  // 1 GB
-  storage: 8589934592,
-  views: 10,
-  automationRuns: 100,
-  users: 10000,
-}
-
-function buildUpdateParams(key, property, usage) {
-  return {
-    primary: key,
-    condition:
-      "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now",
-    expression: "ADD #quota.#prop :usage",
-    names: {
-      "#quota": "usageQuota",
-      "#prop": property,
-      "#limits": "usageLimits",
-      "#quotaReset": "quotaReset",
-    },
-    values: {
-      ":usage": usage,
-      ":now": Date.now(),
-    },
-  }
-}
 
 function getNewQuotaReset() {
   return Date.now() + 2592000000
@@ -47,59 +15,59 @@ exports.Properties = {
   VIEW: "views",
   USER: "users",
   AUTOMATION: "automationRuns",
+  APPS: "apps",
+  EMAILS: "emails",
 }
 
-exports.getAPIKey = async appId => {
+async function getUsageQuotaDoc(db) {
-  if (!env.USE_QUOTAS) {
+  let quota
-    return { apiKey: null }
+  try {
+    quota = await db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota)
+  } catch (err) {
+    // doc doesn't exist. Create it
+    quota = await db.post(generateNewUsageQuotaDoc())
   }
-  return apiKeyTable.get({ primary: appId })
+
+  return quota
 }
 
 /**
- * Given a specified API key this will add to the usage object for the specified property.
+ * Given a specified tenantId this will add to the usage object for the specified property.
- * @param {string} apiKey The API key which is to be updated.
  * @param {string} property The property which is to be added to (within the nested usageQuota object).
  * @param {number} usage The amount (this can be negative) to adjust the number by.
  * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
  * also been reset after this call.
  */
-exports.update = async (apiKey, property, usage) => {
+exports.update = async (property, usage) => {
   if (!env.USE_QUOTAS) {
     return
   }
+
   try {
-    await apiKeyTable.update(buildUpdateParams(apiKey, property, usage))
+    const db = getGlobalDB()
+    const quota = await getUsageQuotaDoc(db)
+
+    // Check if the quota needs reset
+    if (Date.now() >= quota.quotaReset) {
+      quota.quotaReset = getNewQuotaReset()
+      for (let prop of Object.keys(quota.usageQuota)) {
+        quota.usageQuota[prop] = 0
+      }
+    }
+
+    // increment the quota
+    quota.usageQuota[property] += usage
+
+    if (quota.usageQuota[property] >= quota.usageLimits[property]) {
+      throw new Error(
+        `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
+      )
+    }
+
+    // update the usage quotas
+    await db.put(quota)
   } catch (err) {
-    // conditional check means the condition failed, need to check why
+    console.error(`Error updating usage quotas for ${property}`, err)
-    if (err.code === "ConditionalCheckFailedException") {
-      // get the API key so we can check it
-      const keyObj = await apiKeyTable.get({ primary: apiKey })
-      // the usage quota or usage limits didn't exist
-      if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) {
-        keyObj.usageQuota =
-          keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota
-        keyObj.usageLimits =
-          keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits
-        keyObj.quotaReset = getNewQuotaReset()
-        await apiKeyTable.put({ item: keyObj })
-        return
-      }
-      // we have in fact breached the reset period
-      else if (keyObj && keyObj.quotaReset <= Date.now()) {
-        // update the quota reset period and reset the values for all properties
-        keyObj.quotaReset = getNewQuotaReset()
-        for (let prop of Object.keys(keyObj.usageQuota)) {
-          if (prop === property) {
-            keyObj.usageQuota[prop] = usage > 0 ? usage : 0
-          } else {
-            keyObj.usageQuota[prop] = 0
-          }
-        }
-        await apiKeyTable.put({ item: keyObj })
-        return
-      }
-    }
     throw err
   }
 }

packages/server/src/utilities/workerRequests.js

@@ -34,7 +34,7 @@ function request(ctx, request) {
 exports.request = request
 
 // have to pass in the tenant ID as this could be coming from an automation
-exports.sendSmtpEmail = async (to, from, subject, contents) => {
+exports.sendSmtpEmail = async (to, from, subject, contents, automation) => {
   // tenant ID will be set in header
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`),
@@ -46,6 +46,7 @@ exports.sendSmtpEmail = async (to, from, subject, contents) => {
         contents,
         subject,
         purpose: "custom",
+        automation,
       },
     })
   )

packages/worker/src/api/controllers/global/email.js

@@ -2,8 +2,16 @@ const { sendEmail } = require("../../../utilities/email")
 const { getGlobalDB } = require("@budibase/auth/tenancy")
 
 exports.sendEmail = async ctx => {
-  let { workspaceId, email, userId, purpose, contents, from, subject } =
+  let {
-    ctx.request.body
+    workspaceId,
+    email,
+    userId,
+    purpose,
+    contents,
+    from,
+    subject,
+    automation,
+  } = ctx.request.body
   let user
   if (userId) {
     const db = getGlobalDB()
@@ -15,6 +23,7 @@ exports.sendEmail = async ctx => {
     contents,
     from,
     subject,
+    automation,
   })
   ctx.body = {
     ...response,

packages/worker/src/api/controllers/global/users.js

@@ -1,8 +1,8 @@
 const {
   generateGlobalUserID,
   getGlobalUserParams,
-
   StaticDatabases,
+  generateNewUsageQuotaDoc,
 } = require("@budibase/auth/db")
 const { hash, getGlobalUserByEmail } = require("@budibase/auth").utils
 const { UserStatus, EmailTemplatePurpose } = require("../../../constants")
@@ -18,6 +18,7 @@ const {
   tryAddTenant,
   updateTenantId,
 } = require("@budibase/auth/tenancy")
+const env = require("../../../environment")
 
 const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
 
@@ -139,6 +140,11 @@ exports.adminUser = async ctx => {
     })
   )
 
+  // write usage quotas for cloud
+  if (!env.SELF_HOSTED) {
+    await db.post(generateNewUsageQuotaDoc())
+  }
+
   if (response.rows.some(row => row.doc.admin)) {
     ctx.throw(
       403,

packages/worker/src/environment.js

@@ -33,6 +33,11 @@ module.exports = {
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
+  SMTP_USER: process.env.SMTP_USER,
+  SMTP_PASSWORD: process.env.SMTP_PASSWORD,
+  SMTP_HOST: process.env.SMTP_HOST,
+  SMTP_PORT: process.env.SMTP_PORT,
+  SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS,
   _set(key, value) {
     process.env[key] = value
     module.exports[key] = value

packages/worker/src/utilities/email.js

@@ -1,4 +1,5 @@
 const nodemailer = require("nodemailer")
+const env = require("../environment")
 const { getScopedConfig } = require("@budibase/auth/db")
 const { EmailTemplatePurpose, TemplateTypes, Configs } = require("../constants")
 const { getTemplateByPurpose } = require("../constants/templates")
@@ -101,16 +102,35 @@ async function buildEmail(purpose, email, context, { user, contents } = {}) {
  * Utility function for finding most valid SMTP configuration.
  * @param {object} db The CouchDB database which is to be looked up within.
  * @param {string|null} workspaceId If using finer grain control of configs a workspace can be used.
+ * @param {boolean|null} automation Whether or not the configuration is being fetched for an email automation.
  * @return {Promise<object|null>} returns the SMTP configuration if it exists
  */
-async function getSmtpConfiguration(db, workspaceId = null) {
+async function getSmtpConfiguration(db, workspaceId = null, automation) {
   const params = {
     type: Configs.SMTP,
   }
   if (workspaceId) {
     params.workspace = workspaceId
   }
-  return getScopedConfig(db, params)
+
+  const customConfig = getScopedConfig(db, params)
+
+  if (customConfig) {
+    return customConfig
+  }
+
+  // Use an SMTP fallback configuration from env variables
+  if (!automation && env.SMTP_FALLBACK_ENABLED) {
+    return {
+      port: env.SMTP_PORT,
+      host: env.SMTP_HOST,
+      secure: false,
+      auth: {
+        user: env.SMTP_USER,
+        pass: env.SMTP_PASSWORD,
+      },
+    }
+  }
 }
 
 /**
@@ -118,8 +138,8 @@ async function getSmtpConfiguration(db, workspaceId = null) {
  * @return {Promise<boolean>} returns true if there is a configuration that can be used.
  */
 exports.isEmailConfigured = async (workspaceId = null) => {
-  // when "testing" simply return true
+  // when "testing" or smtp fallback is enabled simply return true
-  if (TEST_MODE) {
+  if (TEST_MODE || env.SMTP_FALLBACK_ENABLED) {
     return true
   }
   const db = getGlobalDB()
@@ -138,16 +158,17 @@ exports.isEmailConfigured = async (workspaceId = null) => {
  * @param {string|undefined} contents If sending a custom email then can supply contents which will be added to it.
  * @param {string|undefined} subject A custom subject can be specified if the config one is not desired.
  * @param {object|undefined} info Pass in a structure of information to be stored alongside the invitation.
+ * @param {boolean|undefined} disableFallback Prevent email being sent from SMTP fallback to avoid spam.
  * @return {Promise<object>} returns details about the attempt to send email, e.g. if it is successful; based on
  * nodemailer response.
  */
 exports.sendEmail = async (
   email,
   purpose,
-  { workspaceId, user, from, contents, subject, info } = {}
+  { workspaceId, user, from, contents, subject, info, automation } = {}
 ) => {
   const db = getGlobalDB()
-  let config = (await getSmtpConfiguration(db, workspaceId)) || {}
+  let config = (await getSmtpConfiguration(db, workspaceId, automation)) || {}
   if (Object.keys(config).length === 0 && !TEST_MODE) {
     throw "Unable to find SMTP configuration."
   }