Merge remote-tracking branch 'origin/develop' into feature/datasource-wizard

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.140-alpha.8",
+  "version": "0.9.143-alpha.4",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.140-alpha.8",
+  "version": "0.9.143-alpha.4",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",

packages/auth/src/db/constants.js

@@ -12,6 +12,7 @@ exports.StaticDatabases = {
     name: "global-info",
     docs: {
       tenants: "tenants",
+      usageQuota: "usage_quota",
     },
   },
 }

packages/auth/src/db/utils.js

@@ -368,8 +368,33 @@ async function getScopedConfig(db, params) {
   return configDoc && configDoc.config ? configDoc.config : configDoc
 }
 
+function generateNewUsageQuotaDoc() {
+  return {
+    _id: StaticDatabases.PLATFORM_INFO.docs.usageQuota,
+    quotaReset: Date.now() + 2592000000,
+    usageQuota: {
+      automationRuns: 0,
+      rows: 0,
+      storage: 0,
+      apps: 0,
+      users: 0,
+      views: 0,
+      emails: 0,
+    },
+    usageLimits: {
+      automationRuns: 1000,
+      rows: 4000,
+      apps: 4,
+      storage: 1000,
+      users: 10,
+      emails: 50,
+    },
+  }
+}
+
 exports.Replication = Replication
 exports.getScopedConfig = getScopedConfig
 exports.generateConfigID = generateConfigID
 exports.getConfigParams = getConfigParams
 exports.getScopedFullConfig = getScopedFullConfig
+exports.generateNewUsageQuotaDoc = generateNewUsageQuotaDoc

packages/auth/src/security/permissions.js

@@ -139,8 +139,7 @@ exports.doesHaveResourcePermission = (
   // set foundSub to not subResourceId, incase there is no subResource
   let foundMain = false,
     foundSub = false
-  for (let [resource, level] of Object.entries(permissions)) {
+  for (let [resource, levels] of Object.entries(permissions)) {
-    const levels = getAllowedLevels(level)
     if (resource === resourceId && levels.indexOf(permLevel) !== -1) {
       foundMain = true
     }
@@ -177,10 +176,6 @@ exports.doesHaveBasePermission = (permType, permLevel, permissionIds) => {
   return false
 }
 
-exports.higherPermission = (perm1, perm2) => {
-  return levelToNumber(perm1) > levelToNumber(perm2) ? perm1 : perm2
-}
-
 exports.isPermissionLevelHigherThanRead = level => {
   return levelToNumber(level) > 1
 }

packages/auth/src/security/roles.js

@@ -1,6 +1,6 @@
 const { getDB } = require("../db")
 const { cloneDeep } = require("lodash/fp")
-const { BUILTIN_PERMISSION_IDS, higherPermission } = require("./permissions")
+const { BUILTIN_PERMISSION_IDS } = require("./permissions")
 const {
   generateRoleID,
   getRoleParams,
@@ -193,8 +193,17 @@ exports.getUserPermissions = async (appId, userRoleId) => {
   const permissions = {}
   for (let role of rolesHierarchy) {
     if (role.permissions) {
-      for (let [resource, level] of Object.entries(role.permissions)) {
+      for (let [resource, levels] of Object.entries(role.permissions)) {
-        permissions[resource] = higherPermission(permissions[resource], level)
+        if (!permissions[resource]) {
+          permissions[resource] = []
+        }
+        const permsSet = new Set(permissions[resource])
+        if (Array.isArray(levels)) {
+          levels.forEach(level => permsSet.add(level))
+        } else {
+          permsSet.add(levels)
+        }
+        permissions[resource] = [...permsSet]
       }
     }
   }

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.140-alpha.8",
+  "version": "0.9.143-alpha.4",
   "license": "AGPL-3.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.140-alpha.8",
+  "version": "0.9.143-alpha.4",
   "license": "AGPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.140-alpha.8",
+    "@budibase/bbui": "^0.9.143-alpha.4",
-    "@budibase/client": "^0.9.140-alpha.8",
+    "@budibase/client": "^0.9.143-alpha.4",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.140-alpha.8",
+    "@budibase/string-templates": "^0.9.143-alpha.4",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/analytics/index.js

@@ -3,8 +3,6 @@ import PosthogClient from "./PosthogClient"
 import IntercomClient from "./IntercomClient"
 import SentryClient from "./SentryClient"
 import { Events } from "./constants"
-import { auth } from "stores/portal"
-import { get } from "svelte/store"
 
 const posthog = new PosthogClient(
   process.env.POSTHOG_TOKEN,
@@ -19,27 +17,13 @@ class AnalyticsHub {
   }
 
   async activate() {
-    // Setting the analytics env var off in the backend overrides org/tenant settings
     const analyticsStatus = await api.get("/api/analytics")
     const json = await analyticsStatus.json()
 
-    // Multitenancy disabled on the backend
+    // Analytics disabled
     if (!json.enabled) return
 
-    const tenantId = get(auth).tenantId
-
-    if (tenantId) {
-      const res = await api.get(
-        `/api/global/configs/public?tenantId=${tenantId}`
-      )
-      const orgJson = await res.json()
-
-      // analytics opted out for the tenant
-      if (orgJson.config?.analytics === false) return
-    }
-
     this.clients.forEach(client => client.init())
-    this.enabled = true
   }
 
   identify(id, metadata) {

packages/builder/src/builderStore/dataBinding.js

@@ -443,10 +443,9 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
     for (let from of convertFromProps) {
       if (shouldReplaceBinding(newBoundValue, from, convertTo)) {
         const binding = bindableProperties.find(el => el[convertFrom] === from)
-        newBoundValue = newBoundValue.replace(
+        while (newBoundValue.includes(from)) {
-          new RegExp(from, "gi"),
+          newBoundValue = newBoundValue.replace(from, binding[convertTo])
-          binding[convertTo]
+        }
-        )
       }
     }
     result = result.replace(boundValue, newBoundValue)

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -1,8 +1,9 @@
 <script>
   import { onMount } from "svelte"
+  import { get } from "svelte/store"
   import { goto } from "@roxi/routify"
   import { BUDIBASE_INTERNAL_DB } from "constants"
-  import { database, datasources, queries } from "stores/backend"
+  import { database, datasources, queries, tables } from "stores/backend"
   import EditDatasourcePopover from "./popovers/EditDatasourcePopover.svelte"
   import EditQueryPopover from "./popovers/EditQueryPopover.svelte"
   import NavItem from "components/common/NavItem.svelte"
@@ -10,6 +11,13 @@
   import ICONS from "./icons"
 
   let openDataSources = []
+  $: enrichedDataSources = $datasources.list.map(datasource => ({
+    ...datasource,
+    open:
+      openDataSources.includes(datasource._id) ||
+      containsActiveTable(datasource),
+    selected: $datasources.selected === datasource._id,
+  }))
 
   function selectDatasource(datasource) {
     toggleNode(datasource)
@@ -35,16 +43,28 @@
     datasources.fetch()
     queries.fetch()
   })
+
+  const containsActiveTable = datasource => {
+    const activeTableId = get(tables).selected?._id
+    if (!datasource.entities) {
+      return false
+    }
+    let tableOptions = datasource.entities
+    if (!Array.isArray(tableOptions)) {
+      tableOptions = Object.values(tableOptions)
+    }
+    return tableOptions.find(x => x._id === activeTableId) != null
+  }
 </script>
 
 {#if $database?._id}
   <div class="hierarchy-items-container">
-    {#each $datasources.list as datasource, idx}
+    {#each enrichedDataSources as datasource, idx}
       <NavItem
         border={idx > 0}
         text={datasource.name}
-        opened={openDataSources.includes(datasource._id)}
+        opened={datasource.open}
-        selected={$datasources.selected === datasource._id}
+        selected={datasource.selected}
         withArrow={true}
         on:click={() => selectDatasource(datasource)}
         on:iconClick={() => toggleNode(datasource)}
@@ -61,7 +81,7 @@
         {/if}
       </NavItem>
 
-      {#if openDataSources.includes(datasource._id)}
+      {#if datasource.open}
         <TableNavigator sourceId={datasource._id} />
         {#each $queries.list.filter(query => query.datasourceId === datasource._id) as query}
           <NavItem

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/RefreshDataProvider.svelte

@@ -0,0 +1,35 @@
+<script>
+  import { Select, Label } from "@budibase/bbui"
+  import { currentAsset, store } from "builderStore"
+  import { getActionProviderComponents } from "builderStore/dataBinding"
+
+  export let parameters
+
+  $: actionProviders = getActionProviderComponents(
+    $currentAsset,
+    $store.selectedComponentId,
+    "RefreshDatasource"
+  )
+</script>
+
+<div class="root">
+  <Label small>Data Provider</Label>
+  <Select
+    bind:value={parameters.componentId}
+    options={actionProviders}
+    getOptionLabel={x => x._instanceName}
+    getOptionValue={x => x._id}
+  />
+</div>
+
+<style>
+  .root {
+    display: grid;
+    column-gap: var(--spacing-l);
+    row-gap: var(--spacing-s);
+    grid-template-columns: 70px 1fr;
+    align-items: center;
+    max-width: 400px;
+    margin: 0 auto;
+  }
+</style>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/index.js

@@ -12,6 +12,7 @@ import ClearForm from "./ClearForm.svelte"
 import CloseScreenModal from "./CloseScreenModal.svelte"
 import ChangeFormStep from "./ChangeFormStep.svelte"
 import UpdateStateStep from "./UpdateState.svelte"
+import RefreshDataProvider from "./RefreshDataProvider.svelte"
 
 // Defines which actions are available to configure in the front end.
 // Unfortunately the "name" property is used as the identifier so please don't
@@ -62,6 +63,10 @@ export const getAvailableActions = () => {
       name: "Change Form Step",
       component: ChangeFormStep,
     },
+    {
+      name: "Refresh Data Provider",
+      component: RefreshDataProvider,
+    },
   ]
 
   if (get(store).clientFeatures?.state) {

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/_layout.svelte

@@ -0,0 +1,7 @@
+<script>
+  import { datasources } from "stores/backend"
+
+  datasources.select("bb_internal")
+</script>
+
+<slot />

packages/builder/src/pages/builder/portal/settings/organisation.svelte

@@ -7,13 +7,11 @@
     Divider,
     Label,
     Input,
-    Toggle,
     Dropzone,
     notifications,
   } from "@budibase/bbui"
-  import { auth, organisation } from "stores/portal"
+  import { auth, organisation, admin } from "stores/portal"
   import { post } from "builderStore/api"
-  import analytics from "analytics"
   import { writable } from "svelte/store"
   import { redirect } from "@roxi/routify"
 
@@ -25,7 +23,6 @@
   }
 
   const values = writable({
-    analytics: analytics.enabled,
     company: $organisation.company,
     platformUrl: $organisation.platformUrl,
     logo: $organisation.logoUrl
@@ -57,7 +54,6 @@
     const config = {
       company: $values.company ?? "",
       platformUrl: $values.platformUrl ?? "",
-      analytics: $values.analytics,
     }
     // remove logo if required
     if (!$values.logo) {
@@ -112,6 +108,7 @@
         </div>
       </div>
     </div>
+    {#if !$admin.cloud}
       <Divider size="S" />
       <Layout gap="XS" noPadding>
         <Heading size="S">Platform</Heading>
@@ -123,24 +120,11 @@
           <Input thin bind:value={$values.platformUrl} />
         </div>
       </div>
-    <Divider size="S" />
+    {/if}
-    <Layout gap="S" noPadding>
-      <Layout gap="XS" noPadding>
-        <Heading size="S">Analytics</Heading>
-        <Body size="S">
-          If you would like to send analytics that help us make Budibase better,
-          please let us know below.
-        </Body>
-      </Layout>
-      <Toggle
-        text="Send Analytics to Budibase"
-        bind:value={$values.analytics}
-      />
     <div>
       <Button disabled={loading} on:click={saveConfig} cta>Save</Button>
     </div>
   </Layout>
-  </Layout>
 {/if}
 
 <style>

packages/builder/src/stores/backend/datasources.js

@@ -1,4 +1,4 @@
-import { writable } from "svelte/store"
+import { writable, get } from "svelte/store"
 import { queries, tables, views } from "./"
 import api from "../../builderStore/api"
 
@@ -8,7 +8,8 @@ export const INITIAL_DATASOURCE_VALUES = {
 }
 
 export function createDatasourcesStore() {
-  const { subscribe, update, set } = writable(INITIAL_DATASOURCE_VALUES)
+  const store = writable(INITIAL_DATASOURCE_VALUES)
+  const { subscribe, update, set } = store
 
   return {
     subscribe,
@@ -21,7 +22,15 @@ export function createDatasourcesStore() {
     fetch: async () => {
       const response = await api.get(`/api/datasources`)
       const json = await response.json()
-      update(state => ({ ...state, list: json, selected: null }))
+
+      // Clear selected if it no longer exists, otherwise keep it
+      const selected = get(store).selected
+      let nextSelected = null
+      if (selected && json.find(source => source._id === selected)) {
+        nextSelected = selected
+      }
+
+      update(state => ({ ...state, list: json, selected: nextSelected }))
       return json
     },
     select: async datasourceId => {

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "0.9.140-alpha.8",
+  "version": "0.9.143-alpha.4",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/manifest.json

@@ -2389,6 +2389,7 @@
     "icon": "Data",
     "illegalChildren": ["section"],
     "hasChildren": true,
+    "actions": ["RefreshDatasource"],
     "settings": [
       {
         "type": "dataSource",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "0.9.140-alpha.8",
+  "version": "0.9.143-alpha.4",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.140-alpha.8",
+    "@budibase/bbui": "^0.9.143-alpha.4",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^0.9.140-alpha.8",
+    "@budibase/string-templates": "^0.9.143-alpha.4",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/client/src/utils/buttonActions.js

@@ -88,7 +88,7 @@ const validateFormHandler = async (action, context) => {
   )
 }
 
-const refreshDatasourceHandler = async (action, context) => {
+const refreshDataProviderHandler = async (action, context) => {
   return await executeActionHandler(
     context,
     action.parameters.componentId,
@@ -139,7 +139,7 @@ const handlerMap = {
   ["Execute Query"]: queryExecutionHandler,
   ["Trigger Automation"]: triggerAutomationHandler,
   ["Validate Form"]: validateFormHandler,
-  ["Refresh Datasource"]: refreshDatasourceHandler,
+  ["Refresh Data Provider"]: refreshDataProviderHandler,
   ["Log Out"]: logoutHandler,
   ["Clear Form"]: clearFormHandler,
   ["Close Screen Modal"]: closeScreenModalHandler,

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.140-alpha.8",
+  "version": "0.9.143-alpha.4",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -25,7 +25,9 @@
     "lint:fix": "yarn run format && yarn run lint",
     "initialise": "node scripts/initialise.js",
     "multi:enable": "node scripts/multiTenancy.js enable",
-    "multi:disable": "node scripts/multiTenancy.js disable"
+    "multi:disable": "node scripts/multiTenancy.js disable",
+    "selfhost:enable": "node scripts/selfhost.js enable",
+    "selfhost:disable": "node scripts/selfhost.js disable"
   },
   "jest": {
     "preset": "ts-jest",
@@ -62,9 +64,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.140-alpha.8",
+    "@budibase/auth": "^0.9.143-alpha.4",
-    "@budibase/client": "^0.9.140-alpha.8",
+    "@budibase/client": "^0.9.143-alpha.4",
-    "@budibase/string-templates": "^0.9.140-alpha.8",
+    "@budibase/string-templates": "^0.9.143-alpha.4",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",

packages/server/scripts/integrations/pg-json/docker-compose.yml

@@ -0,0 +1,28 @@
+version: "3.8"
+services:
+  db:
+    container_name: postgres-json
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_USER: root
+      POSTGRES_PASSWORD: root
+      POSTGRES_DB: main
+    ports:
+      - "5432:5432"
+    volumes:
+      #- pg_data:/var/lib/postgresql/data/
+      - ./init.sql:/docker-entrypoint-initdb.d/init.sql
+
+  pgadmin:
+    container_name: pgadmin-json
+    image: dpage/pgadmin4
+    restart: always
+    environment:
+      PGADMIN_DEFAULT_EMAIL: root@root.com
+      PGADMIN_DEFAULT_PASSWORD: root
+    ports:
+      - "5050:80"
+
+#volumes:
+#  pg_data:

packages/server/scripts/integrations/pg-json/init.sql

@@ -0,0 +1,22 @@
+SELECT 'CREATE DATABASE main'
+WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec
+CREATE TABLE jsonTable (
+    id character varying(32),
+    data jsonb,
+    text text
+);
+
+INSERT INTO jsonTable (id, data) VALUES ('1', '{"id": 1, "age": 1, "name": "Mike", "newline": "this is text with a\n newline in it"}');
+
+CREATE VIEW jsonView AS SELECT
+	x.id,
+	x.age,
+	x.name,
+	x.newline
+FROM
+	jsonTable c,
+	LATERAL jsonb_to_record(c.data) x (id character varying(32),
+		age BIGINT,
+		name TEXT,
+		newline TEXT
+		);

packages/server/scripts/integrations/pg-json/reset.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+docker-compose down
+docker volume prune -f

packages/server/scripts/integrations/postgres/docker-compose.yml

@@ -15,7 +15,7 @@ services:
       - ./init.sql:/docker-entrypoint-initdb.d/init.sql
 
   pgadmin:
-    container_name: pgadmin
+    container_name: pgadmin-pg
     image: dpage/pgadmin4
     restart: always
     environment:

packages/server/src/api/controllers/application.js

@@ -230,7 +230,12 @@ exports.create = async function (ctx) {
   const response = await db.put(newApplication, { force: true })
   newApplication._rev = response.rev
 
+  // Only create the default home screens and layout if we aren't importing
+  // an app
+  if (useTemplate !== "true") {
     await createEmptyAppPackage(ctx, newApplication)
+  }
+
   /* istanbul ignore next */
   if (!env.isTest()) {
     await createApp(appId)

packages/server/src/api/controllers/permission.js

@@ -1,9 +1,4 @@
-const {
+const { getBuiltinPermissions } = require("@budibase/auth/permissions")
-  getBuiltinPermissions,
-  PermissionLevels,
-  isPermissionLevelHigherThanRead,
-  higherPermission,
-} = require("@budibase/auth/permissions")
 const {
   isBuiltin,
   getDBRoleID,
@@ -16,6 +11,7 @@ const {
   CURRENTLY_SUPPORTED_LEVELS,
   getBasePermissions,
 } = require("../../utilities/security")
+const { removeFromArray } = require("../../utilities")
 
 const PermissionUpdateType = {
   REMOVE: "remove",
@@ -24,22 +20,6 @@ const PermissionUpdateType = {
 
 const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS
 
-// quick function to perform a bit of weird logic, make sure fetch calls
-// always say a write role also has read permission
-function fetchLevelPerms(permissions, level, roleId) {
-  if (!permissions) {
-    permissions = {}
-  }
-  permissions[level] = roleId
-  if (
-    isPermissionLevelHigherThanRead(level) &&
-    !permissions[PermissionLevels.READ]
-  ) {
-    permissions[PermissionLevels.READ] = roleId
-  }
-  return permissions
-}
-
 // utility function to stop this repetition - permissions always stored under roles
 async function getAllDBRoles(db) {
   const body = await db.allDocs(
@@ -74,23 +54,31 @@ async function updatePermissionOnRole(
   for (let role of dbRoles) {
     let updated = false
     const rolePermissions = role.permissions ? role.permissions : {}
+    // make sure its an array, also handle migrating
+    if (
+      !rolePermissions[resourceId] ||
+      !Array.isArray(rolePermissions[resourceId])
+    ) {
+      rolePermissions[resourceId] =
+        typeof rolePermissions[resourceId] === "string"
+          ? [rolePermissions[resourceId]]
+          : []
+    }
     // handle the removal/updating the role which has this permission first
     // the updating (role._id !== dbRoleId) is required because a resource/level can
     // only be permitted in a single role (this reduces hierarchy confusion and simplifies
     // the general UI for this, rather than needing to show everywhere it is used)
     if (
       (role._id !== dbRoleId || remove) &&
-      rolePermissions[resourceId] === level
+      rolePermissions[resourceId].indexOf(level) !== -1
     ) {
-      delete rolePermissions[resourceId]
+      removeFromArray(rolePermissions[resourceId], level)
       updated = true
     }
     // handle the adding, we're on the correct role, at it to this
     if (!remove && role._id === dbRoleId) {
-      rolePermissions[resourceId] = higherPermission(
+      const set = new Set(rolePermissions[resourceId])
-        rolePermissions[resourceId],
+      rolePermissions[resourceId] = [...set.add(level)]
-        level
-      )
       updated = true
     }
     // handle the update, add it to bulk docs to perform at end
@@ -127,12 +115,11 @@ exports.fetch = async function (ctx) {
       continue
     }
     const roleId = getExternalRoleID(role._id)
-    for (let [resource, level] of Object.entries(role.permissions)) {
+    for (let [resource, levelArr] of Object.entries(role.permissions)) {
-      permissions[resource] = fetchLevelPerms(
+      const levels = Array.isArray(levelArr) ? [levelArr] : levelArr
-        permissions[resource],
+      const perms = {}
-        level,
+      levels.forEach(level => (perms[level] = roleId))
-        roleId
+      permissions[resource] = perms
-      )
     }
   }
   // apply the base permissions
@@ -157,12 +144,13 @@ exports.getResourcePerms = async function (ctx) {
   for (let level of SUPPORTED_LEVELS) {
     // update the various roleIds in the resource permissions
     for (let role of roles) {
-      if (role.permissions && role.permissions[resourceId] === level) {
+      const rolePerms = role.permissions
-        permissions = fetchLevelPerms(
+      if (
-          permissions,
+        rolePerms &&
-          level,
+        (rolePerms[resourceId] === level ||
-          getExternalRoleID(role._id)
+          rolePerms[resourceId].indexOf(level) !== -1)
-        )
+      ) {
+        permissions[level] = getExternalRoleID(role._id)
       }
     }
   }

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -544,6 +544,9 @@ module External {
         extra: {
           idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
         },
+        meta: {
+          table,
+        },
       }
       // can't really use response right now
       const response = await makeExternalQuery(appId, json)

packages/server/src/api/routes/application.js

@@ -2,11 +2,12 @@ const Router = require("@koa/router")
 const controller = require("../controllers/application")
 const authorized = require("../../middleware/authorized")
 const { BUILDER } = require("@budibase/auth/permissions")
+const usage = require("../../middleware/usageQuota")
 
 const router = Router()
 
 router
-  .post("/api/applications", authorized(BUILDER), controller.create)
+  .post("/api/applications", authorized(BUILDER), usage, controller.create)
   .get("/api/applications/:appId/definition", controller.fetchAppDefinition)
   .get("/api/applications", controller.fetch)
   .get("/api/applications/:appId/appPackage", controller.fetchAppPackage)
@@ -21,6 +22,11 @@ router
     authorized(BUILDER),
     controller.revertClient
   )
-  .delete("/api/applications/:appId", authorized(BUILDER), controller.delete)
+  .delete(
+    "/api/applications/:appId",
+    authorized(BUILDER),
+    usage,
+    controller.delete
+  )
 
 module.exports = router

packages/server/src/api/routes/tests/datasource.spec.js

@@ -94,7 +94,8 @@ describe("/datasources", () => {
         .expect(200)
       // this is mock data, can't test it
       expect(res.body).toBeDefined()
-      expect(pg.queryMock).toHaveBeenCalledWith(`select "users"."name" as "users.name", "users"."age" as "users.age" from "users" where "users"."name" ilike $1 limit $2`, ["John%", 5000])
+      const expSql = `select "users"."name" as "users.name", "users"."age" as "users.age" from (select * from "users" where "users"."name" ilike $1 limit $2) as "users"`
+      expect(pg.queryMock).toHaveBeenCalledWith(expSql, ["John%", 5000])
     })
   })
 

packages/server/src/api/routes/tests/role.spec.js

@@ -72,7 +72,7 @@ describe("/roles", () => {
         .expect(200)
       expect(res.body.length).toBeGreaterThan(0)
       const power = res.body.find(role => role._id === BUILTIN_ROLE_IDS.POWER)
-      expect(power.permissions[table._id]).toEqual("read")
+      expect(power.permissions[table._id]).toEqual(["read"])
     })
   })
 

packages/server/src/api/routes/user.js

@@ -5,7 +5,6 @@ const {
   PermissionLevels,
   PermissionTypes,
 } = require("@budibase/auth/permissions")
-const usage = require("../../middleware/usageQuota")
 
 const router = Router()
 
@@ -28,13 +27,11 @@ router
   .post(
     "/api/users/metadata/self",
     authorized(PermissionTypes.USER, PermissionLevels.WRITE),
-    usage,
     controller.updateSelfMetadata
   )
   .delete(
     "/api/users/metadata/:id",
     authorized(PermissionTypes.USER, PermissionLevels.WRITE),
-    usage,
     controller.destroyMetadata
   )
 

packages/server/src/api/routes/view.js

@@ -8,7 +8,6 @@ const {
   PermissionTypes,
   PermissionLevels,
 } = require("@budibase/auth/permissions")
-const usage = require("../../middleware/usageQuota")
 
 const router = Router()
 
@@ -25,9 +24,8 @@ router
     "/api/views/:viewName",
     paramResource("viewName"),
     authorized(BUILDER),
-    usage,
     viewController.destroy
   )
-  .post("/api/views", authorized(BUILDER), usage, viewController.save)
+  .post("/api/views", authorized(BUILDER), viewController.save)
 
 module.exports = router

packages/server/src/automations/steps/createRow.js

@@ -60,7 +60,7 @@ exports.definition = {
   },
 }
 
-exports.run = async function ({ inputs, appId, apiKey, emitter }) {
+exports.run = async function ({ inputs, appId, emitter }) {
   if (inputs.row == null || inputs.row.tableId == null) {
     return {
       success: false,
@@ -84,7 +84,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) {
       inputs.row
     )
     if (env.USE_QUOTAS) {
-      await usage.update(apiKey, usage.Properties.ROW, 1)
+      await usage.update(usage.Properties.ROW, 1)
     }
     await rowController.save(ctx)
     return {

packages/server/src/automations/steps/deleteRow.js

@@ -52,7 +52,7 @@ exports.definition = {
   },
 }
 
-exports.run = async function ({ inputs, appId, apiKey, emitter }) {
+exports.run = async function ({ inputs, appId, emitter }) {
   if (inputs.id == null || inputs.revision == null) {
     return {
       success: false,
@@ -74,7 +74,7 @@ exports.run = async function ({ inputs, appId, apiKey, emitter }) {
 
   try {
     if (env.isProd()) {
-      await usage.update(apiKey, usage.Properties.ROW, -1)
+      await usage.update(usage.Properties.ROW, -1)
     }
     await rowController.destroy(ctx)
     return {

packages/server/src/automations/steps/sendSmtpEmail.js

@@ -53,7 +53,7 @@ exports.run = async function ({ inputs }) {
     contents = "<h1>No content</h1>"
   }
   try {
-    let response = await sendSmtpEmail(to, from, subject, contents)
+    let response = await sendSmtpEmail(to, from, subject, contents, true)
     return {
       success: true,
       response,

packages/server/src/automations/tests/automation.spec.js

@@ -13,8 +13,6 @@ const { makePartial } = require("../../tests/utilities")
 const { cleanInputValues } = require("../automationUtils")
 const setup = require("./utilities")
 
-usageQuota.getAPIKey.mockReturnValue({ apiKey: "test" })
-
 describe("Run through some parts of the automations system", () => {
   let config = setup.getConfig()
 

packages/server/src/automations/tests/createRow.spec.js

@@ -46,7 +46,7 @@ describe("test the create row action", () => {
       await setup.runStep(setup.actions.CREATE_ROW.stepId, {
         row
       })
-      expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", 1)
+      expect(usageQuota.update).toHaveBeenCalledWith("rows", 1)
     })
   })
 

packages/server/src/automations/tests/deleteRow.spec.js

@@ -37,7 +37,7 @@ describe("test the delete row action", () => {
   it("check usage quota attempts", async () => {
     await setup.runInProd(async () => {
       await setup.runStep(setup.actions.DELETE_ROW.stepId, inputs)
-      expect(usageQuota.update).toHaveBeenCalledWith(setup.apiKey, "rows", -1)
+      expect(usageQuota.update).toHaveBeenCalledWith("rows", -1)
     })
   })
 

packages/server/src/automations/thread.js

@@ -4,8 +4,10 @@ const AutomationEmitter = require("../events/AutomationEmitter")
 const { processObject } = require("@budibase/string-templates")
 const { DEFAULT_TENANT_ID } = require("@budibase/auth").constants
 const CouchDB = require("../db")
-const { DocumentTypes } = require("../db/utils")
+const { DocumentTypes, isDevAppID } = require("../db/utils")
 const { doInTenant } = require("@budibase/auth/tenancy")
+const env = require("../environment")
+const usage = require("../utilities/usageQuota")
 
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
 
@@ -80,7 +82,6 @@ class Orchestrator {
           return stepFn({
             inputs: step.inputs,
             appId: this._appId,
-            apiKey: automation.apiKey,
             emitter: this._emitter,
             context: this._context,
           })
@@ -95,6 +96,11 @@ class Orchestrator {
         return err
       }
     }
+
+    // Increment quota for automation runs
+    if (!env.SELF_HOSTED && !isDevAppID(this._appId)) {
+      usage.update(usage.Properties.AUTOMATION, 1)
+    }
     return this.executionOutput
   }
 }

packages/server/src/definitions/datasource.ts

@@ -1,3 +1,5 @@
+import { Table } from "./common"
+
 export enum Operation {
   CREATE = "CREATE",
   READ = "READ",
@@ -136,6 +138,9 @@ export interface QueryJson {
   sort?: SortJson
   paginate?: PaginationJson
   body?: object
+  meta?: {
+    table?: Table
+  }
   extra?: {
     idFilter?: SearchFilters
   }

packages/server/src/integrations/base/sql.ts

@@ -1,7 +1,5 @@
 import { Knex, knex } from "knex"
 const BASE_LIMIT = 5000
-// if requesting a single row then need to up the limit for the sake of joins
-const SINGLE_ROW_LIMIT = 100
 import {
   QueryJson,
   SearchFilters,
@@ -146,46 +144,48 @@ function buildCreate(
 function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery {
   let { endpoint, resource, filters, sort, paginate, relationships } = json
   const tableName = endpoint.entityId
-  let query: KnexQuery = knex(tableName)
   // select all if not specified
   if (!resource) {
     resource = { fields: [] }
   }
+  let selectStatement: string | string[] = "*"
   // handle select
   if (resource.fields && resource.fields.length > 0) {
     // select the resources as the format "table.columnName" - this is what is provided
     // by the resource builder further up
-    query = query.select(resource.fields.map(field => `${field} as ${field}`))
+    selectStatement = resource.fields.map(field => `${field} as ${field}`)
-  } else {
+  }
-    query = query.select("*")
+  let foundLimit = limit || BASE_LIMIT
+  // handle pagination
+  let foundOffset: number | null = null
+  if (paginate && paginate.page && paginate.limit) {
+    // @ts-ignore
+    const page = paginate.page <= 1 ? 0 : paginate.page - 1
+    const offset = page * paginate.limit
+    foundLimit = paginate.limit
+    foundOffset = offset
+  } else if (paginate && paginate.limit) {
+    foundLimit = paginate.limit
+  }
+  // start building the query
+  let query: KnexQuery = knex(tableName).limit(foundLimit)
+  if (foundOffset) {
+    query = query.offset(foundOffset)
   }
-  // handle where
-  query = addFilters(tableName, query, filters)
-  // handle join
-  query = addRelationships(query, tableName, relationships)
-  // handle sorting
   if (sort) {
     for (let [key, value] of Object.entries(sort)) {
       const direction = value === SortDirection.ASCENDING ? "asc" : "desc"
       query = query.orderBy(key, direction)
     }
   }
-  let foundLimit = limit || BASE_LIMIT
+  query = addFilters(tableName, query, filters)
-  // handle pagination
-  if (paginate && paginate.page && paginate.limit) {
   // @ts-ignore
-    const page = paginate.page <= 1 ? 0 : paginate.page - 1
+  let preQuery: KnexQuery = knex({
-    const offset = page * paginate.limit
+    // @ts-ignore
-    foundLimit = paginate.limit
+    [tableName]: query,
-    query = query.offset(offset)
+  }).select(selectStatement)
-  } else if (paginate && paginate.limit) {
+  // handle joins
-    foundLimit = paginate.limit
+  return addRelationships(preQuery, tableName, relationships)
-  }
-  if (foundLimit === 1) {
-    foundLimit = SINGLE_ROW_LIMIT
-  }
-  query = query.limit(foundLimit)
-  return query
 }
 
 function buildUpdate(

packages/server/src/integrations/mysql.ts

@@ -12,7 +12,11 @@ import { getSqlQuery } from "./utils"
 module MySQLModule {
   const mysql = require("mysql")
   const Sql = require("./base/sql")
-  const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils")
+  const {
+    buildExternalTableId,
+    convertType,
+    copyExistingPropsOver,
+  } = require("./utils")
   const { FieldTypes } = require("../constants")
 
   interface MySQLConfig {
@@ -104,7 +108,7 @@ module MySQLModule {
     client: any,
     query: SqlQuery,
     connect: boolean = true
-  ): Promise<any[]> {
+  ): Promise<any[] | any> {
     // Node MySQL is callback based, so we must wrap our call in a promise
     return new Promise((resolve, reject) => {
       if (connect) {
@@ -238,6 +242,23 @@ module MySQLModule {
       return internalQuery(this.client, input, false)
     }
 
+    // when creating if an ID has been inserted need to make sure
+    // the id filter is enriched with it before trying to retrieve the row
+    checkLookupKeys(results: any, json: QueryJson) {
+      if (!results?.insertId || !json.meta?.table || !json.meta.table.primary) {
+        return json
+      }
+      const primaryKey = json.meta.table.primary?.[0]
+      json.extra = {
+        idFilter: {
+          equal: {
+            [primaryKey]: results.insertId,
+          },
+        },
+      }
+      return json
+    }
+
     async query(json: QueryJson) {
       const operation = this._operation(json)
       this.client.connect()
@@ -250,7 +271,7 @@ module MySQLModule {
       const results = await internalQuery(this.client, input, false)
       // same as delete, manage returning
       if (operation === Operation.CREATE || operation === Operation.UPDATE) {
-        row = this.getReturningRow(json)
+        row = this.getReturningRow(this.checkLookupKeys(results, json))
       }
       this.client.end()
       if (operation !== Operation.READ) {

packages/server/src/integrations/postgres.ts

@@ -12,7 +12,14 @@ module PostgresModule {
   const { Pool } = require("pg")
   const Sql = require("./base/sql")
   const { FieldTypes } = require("../constants")
-  const { buildExternalTableId, convertType, copyExistingPropsOver } = require("./utils")
+  const {
+    buildExternalTableId,
+    convertType,
+    copyExistingPropsOver,
+  } = require("./utils")
+  const { escapeDangerousCharacters } = require("../utilities")
+
+  const JSON_REGEX = /'{.*}'::json/s
 
   interface PostgresConfig {
     host: string
@@ -94,6 +101,17 @@ module PostgresModule {
   }
 
   async function internalQuery(client: any, query: SqlQuery) {
+    // need to handle a specific issue with json data types in postgres,
+    // new lines inside the JSON data will break it
+    if (query && query.sql) {
+      const matches = query.sql.match(JSON_REGEX)
+      if (matches && matches.length > 0) {
+        for (let match of matches) {
+          const escaped = escapeDangerousCharacters(match)
+          query.sql = query.sql.replace(match, escaped)
+        }
+      }
+    }
     try {
       return await client.query(query.sql, query.bindings || [])
     } catch (err) {
@@ -179,10 +197,16 @@ module PostgresModule {
         }
 
         const type: string = convertType(column.data_type, TYPE_MAP)
-        const identity = !!(column.identity_generation || column.identity_start || column.identity_increment)
+        const identity = !!(
-        const hasDefault = typeof column.column_default === "string" &&
+          column.identity_generation ||
+          column.identity_start ||
+          column.identity_increment
+        )
+        const hasDefault =
+          typeof column.column_default === "string" &&
           column.column_default.startsWith("nextval")
-        const isGenerated = column.is_generated && column.is_generated !== "NEVER"
+        const isGenerated =
+          column.is_generated && column.is_generated !== "NEVER"
         const isAuto: boolean = hasDefault || identity || isGenerated
         tables[tableName].schema[columnName] = {
           autocolumn: isAuto,

packages/server/src/integrations/tests/sql.spec.js

@@ -57,7 +57,7 @@ describe("SQL query builder", () => {
     const query = sql._query(generateReadJson())
     expect(query).toEqual({
       bindings: [limit],
-      sql: `select * from "${TABLE_NAME}" limit $1`
+      sql: `select * from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"`
     })
   })
 
@@ -68,7 +68,7 @@ describe("SQL query builder", () => {
     }))
     expect(query).toEqual({
       bindings: [limit],
-      sql: `select "${TABLE_NAME}"."name" as "${nameProp}", "${TABLE_NAME}"."age" as "${ageProp}" from "${TABLE_NAME}" limit $1`
+      sql: `select "${TABLE_NAME}"."name" as "${nameProp}", "${TABLE_NAME}"."age" as "${ageProp}" from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"`
     })
   })
 
@@ -82,7 +82,7 @@ describe("SQL query builder", () => {
     }))
     expect(query).toEqual({
       bindings: ["John%", limit],
-      sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" ilike $1 limit $2`
+      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" ilike $1 limit $2) as "${TABLE_NAME}"`
     })
   })
 
@@ -99,7 +99,7 @@ describe("SQL query builder", () => {
     }))
     expect(query).toEqual({
       bindings: [2, 10, limit],
-      sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3`
+      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3) as "${TABLE_NAME}"`
     })
   })
 
@@ -115,7 +115,7 @@ describe("SQL query builder", () => {
     }))
     expect(query).toEqual({
       bindings: [10, "John", limit],
-      sql: `select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3`
+      sql: `select * from (select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3) as "${TABLE_NAME}"`
     })
   })
 
@@ -160,7 +160,7 @@ describe("SQL query builder", () => {
     const query = new Sql("mssql", 10)._query(generateReadJson())
     expect(query).toEqual({
       bindings: [10],
-      sql: `select top (@p0) * from [${TABLE_NAME}]`
+      sql: `select * from (select top (@p0) * from [${TABLE_NAME}]) as [${TABLE_NAME}]`
     })
   })
 
@@ -168,7 +168,7 @@ describe("SQL query builder", () => {
     const query = new Sql("mysql", 10)._query(generateReadJson())
     expect(query).toEqual({
       bindings: [10],
-      sql: `select * from \`${TABLE_NAME}\` limit ?`
+      sql: `select * from (select * from \`${TABLE_NAME}\` limit ?) as \`${TABLE_NAME}\``
     })
   })
 })

packages/server/src/integrations/utils.ts

@@ -84,7 +84,11 @@ export function isIsoDateString(str: string) {
 }
 
 // add the existing relationships from the entities if they exist, to prevent them from being overridden
-export function copyExistingPropsOver(tableName: string, tables: { [key: string]: any }, entities: { [key: string]: any }) {
+export function copyExistingPropsOver(
+  tableName: string,
+  tables: { [key: string]: any },
+  entities: { [key: string]: any }
+) {
   if (entities && entities[tableName]) {
     if (entities[tableName].primaryDisplay) {
       tables[tableName].primaryDisplay = entities[tableName].primaryDisplay

packages/server/src/middleware/tests/usageQuota.spec.js

@@ -39,7 +39,7 @@ class TestConfiguration {
     if (bool) {
       env.isDev = () => false
       env.isProd = () => true
-      this.ctx.auth = { apiKey: "test" }
+      this.ctx.user = { tenantId: "test" }
     } else {
       env.isDev = () => true
       env.isProd = () => false
@@ -114,7 +114,7 @@ describe("usageQuota middleware", () => {
 
     await config.executeMiddleware()
 
-    expect(usageQuota.update).toHaveBeenCalledWith("test", "rows", 1)
+    expect(usageQuota.update).toHaveBeenCalledWith("rows", 1)
     expect(config.next).toHaveBeenCalled()
   })
 
@@ -131,7 +131,7 @@ describe("usageQuota middleware", () => {
     ])
     await config.executeMiddleware()
 
-    expect(usageQuota.update).toHaveBeenCalledWith("test", "storage", 10100)
+    expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
     expect(config.next).toHaveBeenCalled()
   })
 })

packages/server/src/middleware/usageQuota.js

@@ -13,6 +13,7 @@ const DOMAIN_MAP = {
   upload: usageQuota.Properties.UPLOAD,
   views: usageQuota.Properties.VIEW,
   users: usageQuota.Properties.USER,
+  applications: usageQuota.Properties.APPS,
   // this will not be updated by endpoint calls
   // instead it will be updated by triggerInfo
   automationRuns: usageQuota.Properties.AUTOMATION,
@@ -57,9 +58,9 @@ module.exports = async (ctx, next) => {
     usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
   try {
-    await usageQuota.update(ctx.auth.apiKey, property, usage)
+    await usageQuota.update(property, usage)
     return next()
   } catch (err) {
-    ctx.throw(403, err)
+    ctx.throw(400, err)
   }
 }

packages/server/src/utilities/index.js

@@ -10,6 +10,14 @@ exports.wait = ms => new Promise(resolve => setTimeout(resolve, ms))
 
 exports.isDev = env.isDev
 
+exports.removeFromArray = (array, element) => {
+  const index = array.indexOf(element)
+  if (index !== -1) {
+    array.splice(index, 1)
+  }
+  return array
+}
+
 /**
  * Makes sure that a URL has the correct number of slashes, while maintaining the
  * http(s):// double slashes.
@@ -106,3 +114,13 @@ exports.deleteEntityMetadata = async (appId, type, entityId) => {
     await db.remove(id, rev)
   }
 }
+
+exports.escapeDangerousCharacters = string => {
+  return string
+    .replace(/[\\]/g, "\\\\")
+    .replace(/[\b]/g, "\\b")
+    .replace(/[\f]/g, "\\f")
+    .replace(/[\n]/g, "\\n")
+    .replace(/[\r]/g, "\\r")
+    .replace(/[\t]/g, "\\t")
+}

packages/server/src/utilities/usageQuota.js

@@ -1,41 +1,9 @@
 const env = require("../environment")
-const { apiKeyTable } = require("../db/dynamoClient")
+const { getGlobalDB } = require("@budibase/auth/tenancy")
-
+const {
-const DEFAULT_USAGE = {
+  StaticDatabases,
-  rows: 0,
+  generateNewUsageQuotaDoc,
-  storage: 0,
+} = require("@budibase/auth/db")
-  views: 0,
-  automationRuns: 0,
-  users: 0,
-}
-
-const DEFAULT_PLAN = {
-  rows: 1000,
-  // 1 GB
-  storage: 8589934592,
-  views: 10,
-  automationRuns: 100,
-  users: 10000,
-}
-
-function buildUpdateParams(key, property, usage) {
-  return {
-    primary: key,
-    condition:
-      "attribute_exists(#quota) AND attribute_exists(#limits) AND #quota.#prop < #limits.#prop AND #quotaReset > :now",
-    expression: "ADD #quota.#prop :usage",
-    names: {
-      "#quota": "usageQuota",
-      "#prop": property,
-      "#limits": "usageLimits",
-      "#quotaReset": "quotaReset",
-    },
-    values: {
-      ":usage": usage,
-      ":now": Date.now(),
-    },
-  }
-}
 
 function getNewQuotaReset() {
   return Date.now() + 2592000000
@@ -47,59 +15,59 @@ exports.Properties = {
   VIEW: "views",
   USER: "users",
   AUTOMATION: "automationRuns",
+  APPS: "apps",
+  EMAILS: "emails",
 }
 
-exports.getAPIKey = async appId => {
+async function getUsageQuotaDoc(db) {
-  if (!env.USE_QUOTAS) {
+  let quota
-    return { apiKey: null }
+  try {
+    quota = await db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota)
+  } catch (err) {
+    // doc doesn't exist. Create it
+    quota = await db.post(generateNewUsageQuotaDoc())
   }
-  return apiKeyTable.get({ primary: appId })
+
+  return quota
 }
 
 /**
- * Given a specified API key this will add to the usage object for the specified property.
+ * Given a specified tenantId this will add to the usage object for the specified property.
- * @param {string} apiKey The API key which is to be updated.
  * @param {string} property The property which is to be added to (within the nested usageQuota object).
  * @param {number} usage The amount (this can be negative) to adjust the number by.
  * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
  * also been reset after this call.
  */
-exports.update = async (apiKey, property, usage) => {
+exports.update = async (property, usage) => {
   if (!env.USE_QUOTAS) {
     return
   }
+
   try {
-    await apiKeyTable.update(buildUpdateParams(apiKey, property, usage))
+    const db = getGlobalDB()
+    const quota = await getUsageQuotaDoc(db)
+
+    // Check if the quota needs reset
+    if (Date.now() >= quota.quotaReset) {
+      quota.quotaReset = getNewQuotaReset()
+      for (let prop of Object.keys(quota.usageQuota)) {
+        quota.usageQuota[prop] = 0
+      }
+    }
+
+    // increment the quota
+    quota.usageQuota[property] += usage
+
+    if (quota.usageQuota[property] >= quota.usageLimits[property]) {
+      throw new Error(
+        `You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
+      )
+    }
+
+    // update the usage quotas
+    await db.put(quota)
   } catch (err) {
-    // conditional check means the condition failed, need to check why
+    console.error(`Error updating usage quotas for ${property}`, err)
-    if (err.code === "ConditionalCheckFailedException") {
-      // get the API key so we can check it
-      const keyObj = await apiKeyTable.get({ primary: apiKey })
-      // the usage quota or usage limits didn't exist
-      if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) {
-        keyObj.usageQuota =
-          keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota
-        keyObj.usageLimits =
-          keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits
-        keyObj.quotaReset = getNewQuotaReset()
-        await apiKeyTable.put({ item: keyObj })
-        return
-      }
-      // we have in fact breached the reset period
-      else if (keyObj && keyObj.quotaReset <= Date.now()) {
-        // update the quota reset period and reset the values for all properties
-        keyObj.quotaReset = getNewQuotaReset()
-        for (let prop of Object.keys(keyObj.usageQuota)) {
-          if (prop === property) {
-            keyObj.usageQuota[prop] = usage > 0 ? usage : 0
-          } else {
-            keyObj.usageQuota[prop] = 0
-          }
-        }
-        await apiKeyTable.put({ item: keyObj })
-        return
-      }
-    }
     throw err
   }
 }

packages/server/src/utilities/workerRequests.js

@@ -34,7 +34,7 @@ function request(ctx, request) {
 exports.request = request
 
 // have to pass in the tenant ID as this could be coming from an automation
-exports.sendSmtpEmail = async (to, from, subject, contents) => {
+exports.sendSmtpEmail = async (to, from, subject, contents, automation) => {
   // tenant ID will be set in header
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`),
@@ -46,6 +46,7 @@ exports.sendSmtpEmail = async (to, from, subject, contents) => {
         contents,
         subject,
         purpose: "custom",
+        automation,
       },
     })
   )

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "0.9.140-alpha.8",
+  "version": "0.9.143-alpha.4",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "0.9.140-alpha.8",
+  "version": "0.9.143-alpha.4",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -25,8 +25,8 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.140-alpha.8",
+    "@budibase/auth": "^0.9.143-alpha.4",
-    "@budibase/string-templates": "^0.9.140-alpha.8",
+    "@budibase/string-templates": "^0.9.143-alpha.4",
     "@koa/router": "^8.0.0",
     "@techpass/passport-openidconnect": "^0.3.0",
     "aws-sdk": "^2.811.0",

packages/worker/src/api/controllers/global/configs.js

@@ -10,6 +10,7 @@ const email = require("../../../utilities/email")
 const { upload, ObjectStoreBuckets } = require("@budibase/auth").objectStore
 const CouchDB = require("../../../db")
 const { getGlobalDB } = require("@budibase/auth/tenancy")
+const env = require("../../../environment")
 
 exports.save = async function (ctx) {
   const db = getGlobalDB()
@@ -174,7 +175,13 @@ exports.upload = async function (ctx) {
   const file = ctx.request.files.file
   const { type, name } = ctx.params
 
-  const bucket = ObjectStoreBuckets.GLOBAL
+  let bucket
+  if (env.SELF_HOSTED) {
+    bucket = ObjectStoreBuckets.GLOBAL
+  } else {
+    bucket = ObjectStoreBuckets.GLOBAL_CLOUD
+  }
+
   const key = `${type}/${name}`
   await upload({
     bucket,

packages/worker/src/api/controllers/global/email.js

@@ -2,8 +2,16 @@ const { sendEmail } = require("../../../utilities/email")
 const { getGlobalDB } = require("@budibase/auth/tenancy")
 
 exports.sendEmail = async ctx => {
-  let { workspaceId, email, userId, purpose, contents, from, subject } =
+  let {
-    ctx.request.body
+    workspaceId,
+    email,
+    userId,
+    purpose,
+    contents,
+    from,
+    subject,
+    automation,
+  } = ctx.request.body
   let user
   if (userId) {
     const db = getGlobalDB()
@@ -15,6 +23,7 @@ exports.sendEmail = async ctx => {
     contents,
     from,
     subject,
+    automation,
   })
   ctx.body = {
     ...response,

packages/worker/src/api/controllers/global/users.js

@@ -1,8 +1,8 @@
 const {
   generateGlobalUserID,
   getGlobalUserParams,
-
   StaticDatabases,
+  generateNewUsageQuotaDoc,
 } = require("@budibase/auth/db")
 const { hash, getGlobalUserByEmail } = require("@budibase/auth").utils
 const { UserStatus, EmailTemplatePurpose } = require("../../../constants")
@@ -18,6 +18,7 @@ const {
   tryAddTenant,
   updateTenantId,
 } = require("@budibase/auth/tenancy")
+const env = require("../../../environment")
 
 const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
 
@@ -139,6 +140,11 @@ exports.adminUser = async ctx => {
     })
   )
 
+  // write usage quotas for cloud
+  if (!env.SELF_HOSTED) {
+    await db.post(generateNewUsageQuotaDoc())
+  }
+
   if (response.rows.some(row => row.doc.admin)) {
     ctx.throw(
       403,

packages/worker/src/environment.js

@@ -33,6 +33,12 @@ module.exports = {
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
+  SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,
+  SMTP_USER: process.env.SMTP_USER,
+  SMTP_PASSWORD: process.env.SMTP_PASSWORD,
+  SMTP_HOST: process.env.SMTP_HOST,
+  SMTP_PORT: process.env.SMTP_PORT,
+  SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS,
   _set(key, value) {
     process.env[key] = value
     module.exports[key] = value

packages/worker/src/utilities/email.js

@@ -1,4 +1,5 @@
 const nodemailer = require("nodemailer")
+const env = require("../environment")
 const { getScopedConfig } = require("@budibase/auth/db")
 const { EmailTemplatePurpose, TemplateTypes, Configs } = require("../constants")
 const { getTemplateByPurpose } = require("../constants/templates")
@@ -101,16 +102,35 @@ async function buildEmail(purpose, email, context, { user, contents } = {}) {
  * Utility function for finding most valid SMTP configuration.
  * @param {object} db The CouchDB database which is to be looked up within.
  * @param {string|null} workspaceId If using finer grain control of configs a workspace can be used.
+ * @param {boolean|null} automation Whether or not the configuration is being fetched for an email automation.
  * @return {Promise<object|null>} returns the SMTP configuration if it exists
  */
-async function getSmtpConfiguration(db, workspaceId = null) {
+async function getSmtpConfiguration(db, workspaceId = null, automation) {
   const params = {
     type: Configs.SMTP,
   }
   if (workspaceId) {
     params.workspace = workspaceId
   }
-  return getScopedConfig(db, params)
+
+  const customConfig = getScopedConfig(db, params)
+
+  if (customConfig) {
+    return customConfig
+  }
+
+  // Use an SMTP fallback configuration from env variables
+  if (!automation && env.SMTP_FALLBACK_ENABLED) {
+    return {
+      port: env.SMTP_PORT,
+      host: env.SMTP_HOST,
+      secure: false,
+      auth: {
+        user: env.SMTP_USER,
+        pass: env.SMTP_PASSWORD,
+      },
+    }
+  }
 }
 
 /**
@@ -118,8 +138,8 @@ async function getSmtpConfiguration(db, workspaceId = null) {
  * @return {Promise<boolean>} returns true if there is a configuration that can be used.
  */
 exports.isEmailConfigured = async (workspaceId = null) => {
-  // when "testing" simply return true
+  // when "testing" or smtp fallback is enabled simply return true
-  if (TEST_MODE) {
+  if (TEST_MODE || env.SMTP_FALLBACK_ENABLED) {
     return true
   }
   const db = getGlobalDB()
@@ -138,16 +158,17 @@ exports.isEmailConfigured = async (workspaceId = null) => {
  * @param {string|undefined} contents If sending a custom email then can supply contents which will be added to it.
  * @param {string|undefined} subject A custom subject can be specified if the config one is not desired.
  * @param {object|undefined} info Pass in a structure of information to be stored alongside the invitation.
+ * @param {boolean|undefined} disableFallback Prevent email being sent from SMTP fallback to avoid spam.
  * @return {Promise<object>} returns details about the attempt to send email, e.g. if it is successful; based on
  * nodemailer response.
  */
 exports.sendEmail = async (
   email,
   purpose,
-  { workspaceId, user, from, contents, subject, info } = {}
+  { workspaceId, user, from, contents, subject, info, automation } = {}
 ) => {
   const db = getGlobalDB()
-  let config = (await getSmtpConfiguration(db, workspaceId)) || {}
+  let config = (await getSmtpConfiguration(db, workspaceId, automation)) || {}
   if (Object.keys(config).length === 0 && !TEST_MODE) {
     throw "Unable to find SMTP configuration."
   }