Merge remote-tracking branch 'origin/develop' into feature/auto-select-dataprovider-source

hosting/nginx.prod.conf.hbs

@@ -124,12 +124,21 @@ http {
     }
 
     location /api/backups/ {
+      # calls to export apps are limited
+      limit_req zone=ratelimit burst=20 nodelay;
+
+      # 1800s timeout for app export requests
       proxy_read_timeout 1800s;
       proxy_connect_timeout 1800s;
       proxy_send_timeout 1800s;
-      proxy_pass      http://app-service;
+
       proxy_http_version 1.1;
-      proxy_set_header Connection "";
+      proxy_set_header    Connection          $connection_upgrade;
+      proxy_set_header    Upgrade             $http_upgrade;
+      proxy_set_header    X-Real-IP           $remote_addr;
+      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+
+      proxy_pass      http://$apps:4002;
     }
 
     location /api/ {

hosting/single/nginx/nginx-default-site.conf

@@ -43,6 +43,24 @@ server {
         rewrite ^/worker/(.*)$ /$1 break;
     }
 
+    location /api/backups/ {
+        # calls to export apps are limited
+        limit_req zone=ratelimit burst=20 nodelay;
+
+        # 1800s timeout for app export requests
+        proxy_read_timeout 1800s;
+        proxy_connect_timeout 1800s;
+        proxy_send_timeout 1800s;
+
+        proxy_http_version 1.1;
+        proxy_set_header    Connection          $connection_upgrade;
+        proxy_set_header    Upgrade             $http_upgrade;
+        proxy_set_header    X-Real-IP           $remote_addr;
+        proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+
+        proxy_pass      http://127.0.0.1:4001;
+    }
+
     location /api/ {
         # calls to the API are rate limited with bursting
         limit_req zone=ratelimit burst=20 nodelay;

hosting/single/runner.sh

@@ -27,12 +27,14 @@ if [[ "${TARGETBUILD}" = "aas" ]]; then
 else
     DATA_DIR=${DATA_DIR:-/data}
 fi
-
+mkdir -p ${DATA_DIR}
 # Mount NFS or GCP Filestore if env vars exist for it
-if [[ -z ${FILESHARE_IP} && -z ${FILESHARE_NAME} ]]; then
+if [[ ! -z ${FILESHARE_IP} && ! -z ${FILESHARE_NAME} ]]; then
+    echo "Mounting NFS share"
+    apt update && apt install -y nfs-common nfs-kernel-server
     echo "Mount file share ${FILESHARE_IP}:/${FILESHARE_NAME} to ${DATA_DIR}"
     mount -o nolock ${FILESHARE_IP}:/${FILESHARE_NAME} ${DATA_DIR}
-    echo "Mounting completed."
+    echo "Mounting result: $?"
 fi
 
 if [ -f "${DATA_DIR}/.env" ]; then
@@ -74,9 +76,9 @@ mkdir -p ${DATA_DIR}/couch/{dbs,views}
 mkdir -p ${DATA_DIR}/minio
 mkdir -p ${DATA_DIR}/search
 chown -R couchdb:couchdb ${DATA_DIR}/couch
-redis-server --requirepass $REDIS_PASSWORD &
+redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
-/opt/clouseau/bin/clouseau &
+/opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
-/minio/minio server ${DATA_DIR}/minio &
+/minio/minio server ${DATA_DIR}/minio > /dev/stdout 2>&1 &
 /docker-entrypoint.sh /opt/couchdb/bin/couchdb &
 /etc/init.d/nginx restart
 if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then
@@ -85,16 +87,18 @@ if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then
     chmod +x /etc/cron.d/certificate-renew
     # Request the certbot certificate
     /app/letsencrypt/certificate-request.sh ${CUSTOM_DOMAIN}
+    /etc/init.d/nginx restart
 fi
 
-/etc/init.d/nginx restart
 pushd app
-pm2 start --name app "yarn run:docker"
+pm2 start -l /dev/stdout --name app "yarn run:docker"
 popd
 pushd worker
-pm2 start --name worker "yarn run:docker"
+pm2 start -l /dev/stdout --name worker "yarn run:docker"
 popd
 sleep 10
+echo "curl to couchdb endpoints"
 curl -X PUT ${COUCH_DB_URL}/_users
 curl -X PUT ${COUCH_DB_URL}/_replicator
+echo "end of runner.sh, sleeping ..."
 sleep infinity

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.1.22-alpha.6",
+  "version": "2.1.32-alpha.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/db.js

@@ -1,7 +1 @@
-module.exports = {
+module.exports = require("./src/db")
-  ...require("./src/db/utils"),
-  ...require("./src/db/constants"),
-  ...require("./src/db"),
-  ...require("./src/db/views"),
-  ...require("./src/db/pouch"),
-}

packages/backend-core/jest.config.ts

@@ -12,6 +12,7 @@ if (!process.env.CI) {
   // use sources when not in CI
   config.moduleNameMapper = {
     "@budibase/types": "<rootDir>/../types/src",
+    "^axios.*$": "<rootDir>/node_modules/axios/lib/axios.js",
   }
 } else {
   console.log("Running tests with compiled dependency sources")

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.1.22-alpha.6",
+  "version": "2.1.32-alpha.1",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -20,7 +20,7 @@
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
-    "@budibase/types": "2.1.22-alpha.6",
+    "@budibase/types": "2.1.32-alpha.1",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-sdk": "2.1030.0",
@@ -35,6 +35,7 @@
     "koa-passport": "4.1.4",
     "lodash": "4.17.21",
     "lodash.isarguments": "3.1.0",
+    "nano": "^10.1.0",
     "node-fetch": "2.6.7",
     "passport-google-auth": "1.0.2",
     "passport-google-oauth": "2.0.0",

packages/backend-core/src/auth.ts

@@ -3,7 +3,7 @@ const LocalStrategy = require("passport-local").Strategy
 const JwtStrategy = require("passport-jwt").Strategy
 import { getGlobalDB } from "./tenancy"
 const refresh = require("passport-oauth2-refresh")
-import { Configs } from "./constants"
+import { Config } from "./constants"
 import { getScopedConfig } from "./db/utils"
 import {
   jwt,
@@ -76,7 +76,7 @@ async function refreshOIDCAccessToken(
 
   return new Promise(resolve => {
     refresh.requestNewAccessToken(
-      Configs.OIDC,
+      Config.OIDC,
       refreshToken,
       (err: any, accessToken: string, refreshToken: any, params: any) => {
         resolve({ err, accessToken, refreshToken, params })
@@ -106,7 +106,7 @@ async function refreshGoogleAccessToken(
 
   return new Promise(resolve => {
     refresh.requestNewAccessToken(
-      Configs.GOOGLE,
+      Config.GOOGLE,
       refreshToken,
       (err: any, accessToken: string, refreshToken: string, params: any) => {
         resolve({ err, accessToken, refreshToken, params })
@@ -129,7 +129,7 @@ async function refreshOAuthToken(
 
   let chosenConfig = {}
   let refreshResponse
-  if (configType === Configs.OIDC) {
+  if (configType === Config.OIDC) {
     // configId - retrieved from cookie.
     chosenConfig = config.configs.filter((c: any) => c.uuid === configId)[0]
     if (!chosenConfig) {

packages/backend-core/src/cache/tests/writethrough.spec.js

@@ -1,6 +1,6 @@
 require("../../../tests")
 const { Writethrough } = require("../writethrough")
-const { dangerousGetDB } = require("../../db")
+const { getDB } = require("../../db")
 const tk = require("timekeeper")
 
 const START_DATE = Date.now()
@@ -8,8 +8,8 @@ tk.freeze(START_DATE)
 
 const DELAY = 5000
 
-const db = dangerousGetDB("test")
+const db = getDB("test")
-const db2 = dangerousGetDB("test2")
+const db2 = getDB("test2")
 const writethrough = new Writethrough(db, DELAY), writethrough2 = new Writethrough(db2, DELAY)
 
 describe("writethrough", () => {

packages/backend-core/src/cache/writethrough.ts

@@ -1,7 +1,7 @@
 import BaseCache from "./base"
 import { getWritethroughClient } from "../redis/init"
 import { logWarn } from "../logging"
-import PouchDB from "pouchdb"
+import { Database } from "@budibase/types"
 
 const DEFAULT_WRITE_RATE_MS = 10000
 let CACHE: BaseCache | null = null
@@ -19,7 +19,7 @@ async function getCache() {
   return CACHE
 }
 
-function makeCacheKey(db: PouchDB.Database, key: string) {
+function makeCacheKey(db: Database, key: string) {
   return db.name + key
 }
 
@@ -28,7 +28,7 @@ function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem {
 }
 
 export async function put(
-  db: PouchDB.Database,
+  db: Database,
   doc: any,
   writeRateMs: number = DEFAULT_WRITE_RATE_MS
 ) {
@@ -64,7 +64,7 @@ export async function put(
   return { ok: true, id: output._id, rev: output._rev }
 }
 
-export async function get(db: PouchDB.Database, id: string): Promise<any> {
+export async function get(db: Database, id: string): Promise<any> {
   const cache = await getCache()
   const cacheKey = makeCacheKey(db, id)
   let cacheItem: CacheItem = await cache.get(cacheKey)
@@ -77,7 +77,7 @@ export async function get(db: PouchDB.Database, id: string): Promise<any> {
 }
 
 export async function remove(
-  db: PouchDB.Database,
+  db: Database,
   docOrId: any,
   rev?: any
 ): Promise<void> {
@@ -95,13 +95,10 @@ export async function remove(
 }
 
 export class Writethrough {
-  db: PouchDB.Database
+  db: Database
   writeRateMs: number
 
-  constructor(
+  constructor(db: Database, writeRateMs: number = DEFAULT_WRITE_RATE_MS) {
-    db: PouchDB.Database,
-    writeRateMs: number = DEFAULT_WRITE_RATE_MS
-  ) {
     this.db = db
     this.writeRateMs = writeRateMs
   }

packages/backend-core/src/cloud/accounts.ts

@@ -1,6 +1,6 @@
 import API from "./api"
 import env from "../environment"
-import { Headers } from "../constants"
+import { Header } from "../constants"
 import { CloudAccount } from "@budibase/types"
 
 const api = new API(env.ACCOUNT_PORTAL_URL)
@@ -14,7 +14,7 @@ export const getAccount = async (
   const response = await api.post(`/api/accounts/search`, {
     body: payload,
     headers: {
-      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+      [Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
     },
   })
 
@@ -35,7 +35,7 @@ export const getAccountByTenantId = async (
   const response = await api.post(`/api/accounts/search`, {
     body: payload,
     headers: {
-      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+      [Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
     },
   })
 
@@ -50,7 +50,7 @@ export const getAccountByTenantId = async (
 export const getStatus = async () => {
   const response = await api.get(`/api/status`, {
     headers: {
-      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+      [Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
     },
   })
   const json = await response.json()

packages/backend-core/src/clshooked/index.js

@@ -1,650 +0,0 @@
-const util = require("util")
-const assert = require("assert")
-const wrapEmitter = require("emitter-listener")
-const async_hooks = require("async_hooks")
-
-const CONTEXTS_SYMBOL = "cls@contexts"
-const ERROR_SYMBOL = "error@context"
-
-const DEBUG_CLS_HOOKED = process.env.DEBUG_CLS_HOOKED
-
-let currentUid = -1
-
-module.exports = {
-  getNamespace: getNamespace,
-  createNamespace: createNamespace,
-  destroyNamespace: destroyNamespace,
-  reset: reset,
-  ERROR_SYMBOL: ERROR_SYMBOL,
-}
-
-function Namespace(name) {
-  this.name = name
-  // changed in 2.7: no default context
-  this.active = null
-  this._set = []
-  this.id = null
-  this._contexts = new Map()
-  this._indent = 0
-  this._hook = null
-}
-
-Namespace.prototype.set = function set(key, value) {
-  if (!this.active) {
-    throw new Error(
-      "No context available. ns.run() or ns.bind() must be called first."
-    )
-  }
-
-  this.active[key] = value
-
-  if (DEBUG_CLS_HOOKED) {
-    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
-    debug2(
-      indentStr +
-        "CONTEXT-SET KEY:" +
-        key +
-        "=" +
-        value +
-        " in ns:" +
-        this.name +
-        " currentUid:" +
-        currentUid +
-        " active:" +
-        util.inspect(this.active, { showHidden: true, depth: 2, colors: true })
-    )
-  }
-
-  return value
-}
-
-Namespace.prototype.get = function get(key) {
-  if (!this.active) {
-    if (DEBUG_CLS_HOOKED) {
-      const asyncHooksCurrentId = async_hooks.currentId()
-      const triggerId = async_hooks.triggerAsyncId()
-      const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
-      debug2(
-        `${indentStr}CONTEXT-GETTING KEY NO ACTIVE NS: (${this.name}) ${key}=undefined currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${this._set.length}`
-      )
-    }
-    return undefined
-  }
-  if (DEBUG_CLS_HOOKED) {
-    const asyncHooksCurrentId = async_hooks.executionAsyncId()
-    const triggerId = async_hooks.triggerAsyncId()
-    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
-    debug2(
-      indentStr +
-        "CONTEXT-GETTING KEY:" +
-        key +
-        "=" +
-        this.active[key] +
-        " (" +
-        this.name +
-        ") currentUid:" +
-        currentUid +
-        " active:" +
-        util.inspect(this.active, { showHidden: true, depth: 2, colors: true })
-    )
-    debug2(
-      `${indentStr}CONTEXT-GETTING KEY: (${this.name}) ${key}=${
-        this.active[key]
-      } currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${
-        this._set.length
-      } active:${util.inspect(this.active)}`
-    )
-  }
-  return this.active[key]
-}
-
-Namespace.prototype.createContext = function createContext() {
-  // Prototype inherit existing context if created a new child context within existing context.
-  let context = Object.create(this.active ? this.active : Object.prototype)
-  context._ns_name = this.name
-  context.id = currentUid
-
-  if (DEBUG_CLS_HOOKED) {
-    const asyncHooksCurrentId = async_hooks.executionAsyncId()
-    const triggerId = async_hooks.triggerAsyncId()
-    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
-    debug2(
-      `${indentStr}CONTEXT-CREATED Context: (${
-        this.name
-      }) currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${
-        this._set.length
-      } context:${util.inspect(context, {
-        showHidden: true,
-        depth: 2,
-        colors: true,
-      })}`
-    )
-  }
-
-  return context
-}
-
-Namespace.prototype.run = function run(fn) {
-  let context = this.createContext()
-  this.enter(context)
-
-  try {
-    if (DEBUG_CLS_HOOKED) {
-      const triggerId = async_hooks.triggerAsyncId()
-      const asyncHooksCurrentId = async_hooks.executionAsyncId()
-      const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
-      debug2(
-        `${indentStr}CONTEXT-RUN BEGIN: (${
-          this.name
-        }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
-          this._set.length
-        } context:${util.inspect(context)}`
-      )
-    }
-    fn(context)
-    return context
-  } catch (exception) {
-    if (exception) {
-      exception[ERROR_SYMBOL] = context
-    }
-    throw exception
-  } finally {
-    if (DEBUG_CLS_HOOKED) {
-      const triggerId = async_hooks.triggerAsyncId()
-      const asyncHooksCurrentId = async_hooks.executionAsyncId()
-      const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
-      debug2(
-        `${indentStr}CONTEXT-RUN END: (${
-          this.name
-        }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
-          this._set.length
-        } ${util.inspect(context)}`
-      )
-    }
-    this.exit(context)
-  }
-}
-
-Namespace.prototype.runAndReturn = function runAndReturn(fn) {
-  let value
-  this.run(function (context) {
-    value = fn(context)
-  })
-  return value
-}
-
-/**
- * Uses global Promise and assumes Promise is cls friendly or wrapped already.
- * @param {function} fn
- * @returns {*}
- */
-Namespace.prototype.runPromise = function runPromise(fn) {
-  let context = this.createContext()
-  this.enter(context)
-
-  let promise = fn(context)
-  if (!promise || !promise.then || !promise.catch) {
-    throw new Error("fn must return a promise.")
-  }
-
-  if (DEBUG_CLS_HOOKED) {
-    debug2(
-      "CONTEXT-runPromise BEFORE: (" +
-        this.name +
-        ") currentUid:" +
-        currentUid +
-        " len:" +
-        this._set.length +
-        " " +
-        util.inspect(context)
-    )
-  }
-
-  return promise
-    .then(result => {
-      if (DEBUG_CLS_HOOKED) {
-        debug2(
-          "CONTEXT-runPromise AFTER then: (" +
-            this.name +
-            ") currentUid:" +
-            currentUid +
-            " len:" +
-            this._set.length +
-            " " +
-            util.inspect(context)
-        )
-      }
-      this.exit(context)
-      return result
-    })
-    .catch(err => {
-      err[ERROR_SYMBOL] = context
-      if (DEBUG_CLS_HOOKED) {
-        debug2(
-          "CONTEXT-runPromise AFTER catch: (" +
-            this.name +
-            ") currentUid:" +
-            currentUid +
-            " len:" +
-            this._set.length +
-            " " +
-            util.inspect(context)
-        )
-      }
-      this.exit(context)
-      throw err
-    })
-}
-
-Namespace.prototype.bind = function bindFactory(fn, context) {
-  if (!context) {
-    if (!this.active) {
-      context = this.createContext()
-    } else {
-      context = this.active
-    }
-  }
-
-  let self = this
-  return function clsBind() {
-    self.enter(context)
-    try {
-      return fn.apply(this, arguments)
-    } catch (exception) {
-      if (exception) {
-        exception[ERROR_SYMBOL] = context
-      }
-      throw exception
-    } finally {
-      self.exit(context)
-    }
-  }
-}
-
-Namespace.prototype.enter = function enter(context) {
-  assert.ok(context, "context must be provided for entering")
-  if (DEBUG_CLS_HOOKED) {
-    const asyncHooksCurrentId = async_hooks.executionAsyncId()
-    const triggerId = async_hooks.triggerAsyncId()
-    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
-    debug2(
-      `${indentStr}CONTEXT-ENTER: (${
-        this.name
-      }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
-        this._set.length
-      } ${util.inspect(context)}`
-    )
-  }
-
-  this._set.push(this.active)
-  this.active = context
-}
-
-Namespace.prototype.exit = function exit(context) {
-  assert.ok(context, "context must be provided for exiting")
-  if (DEBUG_CLS_HOOKED) {
-    const asyncHooksCurrentId = async_hooks.executionAsyncId()
-    const triggerId = async_hooks.triggerAsyncId()
-    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
-    debug2(
-      `${indentStr}CONTEXT-EXIT: (${
-        this.name
-      }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
-        this._set.length
-      } ${util.inspect(context)}`
-    )
-  }
-
-  // Fast path for most exits that are at the top of the stack
-  if (this.active === context) {
-    assert.ok(this._set.length, "can't remove top context")
-    this.active = this._set.pop()
-    return
-  }
-
-  // Fast search in the stack using lastIndexOf
-  let index = this._set.lastIndexOf(context)
-
-  if (index < 0) {
-    if (DEBUG_CLS_HOOKED) {
-      debug2(
-        "??ERROR?? context exiting but not entered - ignoring: " +
-          util.inspect(context)
-      )
-    }
-    assert.ok(
-      index >= 0,
-      "context not currently entered; can't exit. \n" +
-        util.inspect(this) +
-        "\n" +
-        util.inspect(context)
-    )
-  } else {
-    assert.ok(index, "can't remove top context")
-    this._set.splice(index, 1)
-  }
-}
-
-Namespace.prototype.bindEmitter = function bindEmitter(emitter) {
-  assert.ok(
-    emitter.on && emitter.addListener && emitter.emit,
-    "can only bind real EEs"
-  )
-
-  let namespace = this
-  let thisSymbol = "context@" + this.name
-
-  // Capture the context active at the time the emitter is bound.
-  function attach(listener) {
-    if (!listener) {
-      return
-    }
-    if (!listener[CONTEXTS_SYMBOL]) {
-      listener[CONTEXTS_SYMBOL] = Object.create(null)
-    }
-
-    listener[CONTEXTS_SYMBOL][thisSymbol] = {
-      namespace: namespace,
-      context: namespace.active,
-    }
-  }
-
-  // At emit time, bind the listener within the correct context.
-  function bind(unwrapped) {
-    if (!(unwrapped && unwrapped[CONTEXTS_SYMBOL])) {
-      return unwrapped
-    }
-
-    let wrapped = unwrapped
-    let unwrappedContexts = unwrapped[CONTEXTS_SYMBOL]
-    Object.keys(unwrappedContexts).forEach(function (name) {
-      let thunk = unwrappedContexts[name]
-      wrapped = thunk.namespace.bind(wrapped, thunk.context)
-    })
-    return wrapped
-  }
-
-  wrapEmitter(emitter, attach, bind)
-}
-
-/**
- * If an error comes out of a namespace, it will have a context attached to it.
- * This function knows how to find it.
- *
- * @param {Error} exception Possibly annotated error.
- */
-Namespace.prototype.fromException = function fromException(exception) {
-  return exception[ERROR_SYMBOL]
-}
-
-function getNamespace(name) {
-  return process.namespaces[name]
-}
-
-function createNamespace(name) {
-  assert.ok(name, "namespace must be given a name.")
-
-  if (DEBUG_CLS_HOOKED) {
-    debug2(`NS-CREATING NAMESPACE (${name})`)
-  }
-  let namespace = new Namespace(name)
-  namespace.id = currentUid
-
-  const hook = async_hooks.createHook({
-    init(asyncId, type, triggerId, resource) {
-      currentUid = async_hooks.executionAsyncId()
-
-      //CHAIN Parent's Context onto child if none exists. This is needed to pass net-events.spec
-      // let initContext = namespace.active;
-      // if(!initContext && triggerId) {
-      //   let parentContext = namespace._contexts.get(triggerId);
-      //   if (parentContext) {
-      //     namespace.active = parentContext;
-      //     namespace._contexts.set(currentUid, parentContext);
-      //     if (DEBUG_CLS_HOOKED) {
-      //       const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent);
-      //       debug2(`${indentStr}INIT [${type}] (${name}) WITH PARENT CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`);
-      //     }
-      //   } else if (DEBUG_CLS_HOOKED) {
-      //       const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent);
-      //       debug2(`${indentStr}INIT [${type}] (${name}) MISSING CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`);
-      //     }
-      // }else {
-      //   namespace._contexts.set(currentUid, namespace.active);
-      //   if (DEBUG_CLS_HOOKED) {
-      //     const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent);
-      //     debug2(`${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`);
-      //   }
-      // }
-      if (namespace.active) {
-        namespace._contexts.set(asyncId, namespace.active)
-
-        if (DEBUG_CLS_HOOKED) {
-          const indentStr = " ".repeat(
-            namespace._indent < 0 ? 0 : namespace._indent
-          )
-          debug2(
-            `${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
-              namespace.active,
-              { showHidden: true, depth: 2, colors: true }
-            )} resource:${resource}`
-          )
-        }
-      } else if (currentUid === 0) {
-        // CurrentId will be 0 when triggered from C++. Promise events
-        // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid
-        const triggerId = async_hooks.triggerAsyncId()
-        const triggerIdContext = namespace._contexts.get(triggerId)
-        if (triggerIdContext) {
-          namespace._contexts.set(asyncId, triggerIdContext)
-          if (DEBUG_CLS_HOOKED) {
-            const indentStr = " ".repeat(
-              namespace._indent < 0 ? 0 : namespace._indent
-            )
-            debug2(
-              `${indentStr}INIT USING CONTEXT FROM TRIGGERID [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
-                namespace.active,
-                { showHidden: true, depth: 2, colors: true }
-              )} resource:${resource}`
-            )
-          }
-        } else if (DEBUG_CLS_HOOKED) {
-          const indentStr = " ".repeat(
-            namespace._indent < 0 ? 0 : namespace._indent
-          )
-          debug2(
-            `${indentStr}INIT MISSING CONTEXT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
-              namespace.active,
-              { showHidden: true, depth: 2, colors: true }
-            )} resource:${resource}`
-          )
-        }
-      }
-
-      if (DEBUG_CLS_HOOKED && type === "PROMISE") {
-        debug2(util.inspect(resource, { showHidden: true }))
-        const parentId = resource.parentId
-        const indentStr = " ".repeat(
-          namespace._indent < 0 ? 0 : namespace._indent
-        )
-        debug2(
-          `${indentStr}INIT RESOURCE-PROMISE [${type}] (${name}) parentId:${parentId} asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
-            namespace.active,
-            { showHidden: true, depth: 2, colors: true }
-          )} resource:${resource}`
-        )
-      }
-    },
-    before(asyncId) {
-      currentUid = async_hooks.executionAsyncId()
-      let context
-
-      /*
-      if(currentUid === 0){
-        // CurrentId will be 0 when triggered from C++. Promise events
-        // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid
-        //const triggerId = async_hooks.triggerAsyncId();
-        context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId);
-      }else{
-        context = namespace._contexts.get(currentUid);
-      }
-      */
-
-      //HACK to work with promises until they are fixed in node > 8.1.1
-      context =
-        namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid)
-
-      if (context) {
-        if (DEBUG_CLS_HOOKED) {
-          const triggerId = async_hooks.triggerAsyncId()
-          const indentStr = " ".repeat(
-            namespace._indent < 0 ? 0 : namespace._indent
-          )
-          debug2(
-            `${indentStr}BEFORE (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
-              namespace.active,
-              { showHidden: true, depth: 2, colors: true }
-            )} context:${util.inspect(context)}`
-          )
-          namespace._indent += 2
-        }
-
-        namespace.enter(context)
-      } else if (DEBUG_CLS_HOOKED) {
-        const triggerId = async_hooks.triggerAsyncId()
-        const indentStr = " ".repeat(
-          namespace._indent < 0 ? 0 : namespace._indent
-        )
-        debug2(
-          `${indentStr}BEFORE MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
-            namespace.active,
-            { showHidden: true, depth: 2, colors: true }
-          )} namespace._contexts:${util.inspect(namespace._contexts, {
-            showHidden: true,
-            depth: 2,
-            colors: true,
-          })}`
-        )
-        namespace._indent += 2
-      }
-    },
-    after(asyncId) {
-      currentUid = async_hooks.executionAsyncId()
-      let context // = namespace._contexts.get(currentUid);
-      /*
-      if(currentUid === 0){
-        // CurrentId will be 0 when triggered from C++. Promise events
-        // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid
-        //const triggerId = async_hooks.triggerAsyncId();
-        context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId);
-      }else{
-        context = namespace._contexts.get(currentUid);
-      }
-      */
-      //HACK to work with promises until they are fixed in node > 8.1.1
-      context =
-        namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid)
-
-      if (context) {
-        if (DEBUG_CLS_HOOKED) {
-          const triggerId = async_hooks.triggerAsyncId()
-          namespace._indent -= 2
-          const indentStr = " ".repeat(
-            namespace._indent < 0 ? 0 : namespace._indent
-          )
-          debug2(
-            `${indentStr}AFTER (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
-              namespace.active,
-              { showHidden: true, depth: 2, colors: true }
-            )} context:${util.inspect(context)}`
-          )
-        }
-
-        namespace.exit(context)
-      } else if (DEBUG_CLS_HOOKED) {
-        const triggerId = async_hooks.triggerAsyncId()
-        namespace._indent -= 2
-        const indentStr = " ".repeat(
-          namespace._indent < 0 ? 0 : namespace._indent
-        )
-        debug2(
-          `${indentStr}AFTER MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
-            namespace.active,
-            { showHidden: true, depth: 2, colors: true }
-          )} context:${util.inspect(context)}`
-        )
-      }
-    },
-    destroy(asyncId) {
-      currentUid = async_hooks.executionAsyncId()
-      if (DEBUG_CLS_HOOKED) {
-        const triggerId = async_hooks.triggerAsyncId()
-        const indentStr = " ".repeat(
-          namespace._indent < 0 ? 0 : namespace._indent
-        )
-        debug2(
-          `${indentStr}DESTROY (${name}) currentUid:${currentUid} asyncId:${asyncId} triggerId:${triggerId} active:${util.inspect(
-            namespace.active,
-            { showHidden: true, depth: 2, colors: true }
-          )} context:${util.inspect(namespace._contexts.get(currentUid))}`
-        )
-      }
-
-      namespace._contexts.delete(asyncId)
-    },
-  })
-
-  hook.enable()
-  namespace._hook = hook
-
-  process.namespaces[name] = namespace
-  return namespace
-}
-
-function destroyNamespace(name) {
-  let namespace = getNamespace(name)
-
-  assert.ok(namespace, "can't delete nonexistent namespace! \"" + name + '"')
-  assert.ok(
-    namespace.id,
-    "don't assign to process.namespaces directly! " + util.inspect(namespace)
-  )
-
-  namespace._hook.disable()
-  namespace._contexts = null
-  process.namespaces[name] = null
-}
-
-function reset() {
-  // must unregister async listeners
-  if (process.namespaces) {
-    Object.keys(process.namespaces).forEach(function (name) {
-      destroyNamespace(name)
-    })
-  }
-  process.namespaces = Object.create(null)
-}
-
-process.namespaces = process.namespaces || {}
-
-//const fs = require('fs');
-function debug2(...args) {
-  if (DEBUG_CLS_HOOKED) {
-    //fs.writeSync(1, `${util.format(...args)}\n`);
-    process._rawDebug(`${util.format(...args)}`)
-  }
-}
-
-/*function getFunctionName(fn) {
-  if (!fn) {
-    return fn;
-  }
-  if (typeof fn === 'function') {
-    if (fn.name) {
-      return fn.name;
-    }
-    return (fn.toString().trim().match(/^function\s*([^\s(]+)/) || [])[1];
-  } else if (fn.constructor && fn.constructor.name) {
-    return fn.constructor.name;
-  }
-}*/

packages/backend-core/src/constants.js

@@ -1,44 +0,0 @@
-exports.UserStatus = {
-  ACTIVE: "active",
-  INACTIVE: "inactive",
-}
-
-exports.Cookies = {
-  CurrentApp: "budibase:currentapp",
-  Auth: "budibase:auth",
-  Init: "budibase:init",
-  ACCOUNT_RETURN_URL: "budibase:account:returnurl",
-  DatasourceAuth: "budibase:datasourceauth",
-  OIDC_CONFIG: "budibase:oidc:config",
-}
-
-exports.Headers = {
-  API_KEY: "x-budibase-api-key",
-  LICENSE_KEY: "x-budibase-license-key",
-  API_VER: "x-budibase-api-version",
-  APP_ID: "x-budibase-app-id",
-  TYPE: "x-budibase-type",
-  PREVIEW_ROLE: "x-budibase-role",
-  TENANT_ID: "x-budibase-tenant-id",
-  TOKEN: "x-budibase-token",
-  CSRF_TOKEN: "x-csrf-token",
-}
-
-exports.GlobalRoles = {
-  OWNER: "owner",
-  ADMIN: "admin",
-  BUILDER: "builder",
-  WORKSPACE_MANAGER: "workspace_manager",
-}
-
-exports.Configs = {
-  SETTINGS: "settings",
-  ACCOUNT: "account",
-  SMTP: "smtp",
-  GOOGLE: "google",
-  OIDC: "oidc",
-  OIDC_LOGOS: "logos_oidc",
-}
-
-exports.MAX_VALID_DATE = new Date(2147483647000)
-exports.DEFAULT_TENANT_ID = "default"

packages/backend-core/src/constants.ts

@@ -0,0 +1,44 @@
+export enum UserStatus {
+  ACTIVE = "active",
+  INACTIVE = "inactive",
+}
+
+export enum Cookie {
+  CurrentApp = "budibase:currentapp",
+  Auth = "budibase:auth",
+  Init = "budibase:init",
+  ACCOUNT_RETURN_URL = "budibase:account:returnurl",
+  DatasourceAuth = "budibase:datasourceauth",
+  OIDC_CONFIG = "budibase:oidc:config",
+}
+
+export enum Header {
+  API_KEY = "x-budibase-api-key",
+  LICENSE_KEY = "x-budibase-license-key",
+  API_VER = "x-budibase-api-version",
+  APP_ID = "x-budibase-app-id",
+  TYPE = "x-budibase-type",
+  PREVIEW_ROLE = "x-budibase-role",
+  TENANT_ID = "x-budibase-tenant-id",
+  TOKEN = "x-budibase-token",
+  CSRF_TOKEN = "x-csrf-token",
+}
+
+export enum GlobalRole {
+  OWNER = "owner",
+  ADMIN = "admin",
+  BUILDER = "builder",
+  WORKSPACE_MANAGER = "workspace_manager",
+}
+
+export enum Config {
+  SETTINGS = "settings",
+  ACCOUNT = "account",
+  SMTP = "smtp",
+  GOOGLE = "google",
+  OIDC = "oidc",
+  OIDC_LOGOS = "logos_oidc",
+}
+
+export const MAX_VALID_DATE = new Date(2147483647000)
+export const DEFAULT_TENANT_ID = "default"

packages/backend-core/src/context/Context.ts

@@ -0,0 +1,18 @@
+import { AsyncLocalStorage } from "async_hooks"
+import { ContextMap } from "./constants"
+
+export default class Context {
+  static storage = new AsyncLocalStorage<ContextMap>()
+
+  static run(context: ContextMap, func: any) {
+    return Context.storage.run(context, () => func())
+  }
+
+  static get(): ContextMap {
+    return Context.storage.getStore() as ContextMap
+  }
+
+  static set(context: ContextMap) {
+    Context.storage.enterWith(context)
+  }
+}

packages/backend-core/src/context/FunctionContext.js

@@ -1,47 +0,0 @@
-const cls = require("../clshooked")
-const { newid } = require("../hashing")
-
-const REQUEST_ID_KEY = "requestId"
-const MAIN_CTX = cls.createNamespace("main")
-
-function getContextStorage(namespace) {
-  if (namespace && namespace.active) {
-    let contextData = namespace.active
-    delete contextData.id
-    delete contextData._ns_name
-    return contextData
-  }
-  return {}
-}
-
-class FunctionContext {
-  static run(callback) {
-    return MAIN_CTX.runAndReturn(async () => {
-      const namespaceId = newid()
-      MAIN_CTX.set(REQUEST_ID_KEY, namespaceId)
-      const namespace = cls.createNamespace(namespaceId)
-      let response = await namespace.runAndReturn(callback)
-      cls.destroyNamespace(namespaceId)
-      return response
-    })
-  }
-
-  static setOnContext(key, value) {
-    const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY)
-    const namespace = cls.getNamespace(namespaceId)
-    namespace.set(key, value)
-  }
-
-  static getFromContext(key) {
-    const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY)
-    const namespace = cls.getNamespace(namespaceId)
-    const context = getContextStorage(namespace)
-    if (context) {
-      return context[key]
-    } else {
-      return null
-    }
-  }
-}
-
-module.exports = FunctionContext

packages/backend-core/src/context/constants.ts

@@ -1,17 +1,7 @@
-export enum ContextKey {
+import { IdentityContext } from "@budibase/types"
-  TENANT_ID = "tenantId",
+
-  GLOBAL_DB = "globalDb",
+export type ContextMap = {
-  APP_ID = "appId",
+  tenantId?: string
-  IDENTITY = "identity",
+  appId?: string
-  // whatever the request app DB was
+  identity?: IdentityContext
-  CURRENT_DB = "currentDb",
-  // get the prod app DB from the request
-  PROD_DB = "prodDb",
-  // get the dev app DB from the request
-  DEV_DB = "devDb",
-  DB_OPTS = "dbOpts",
-  // check if something else is using the context, don't close DB
-  TENANCY_IN_USE = "tenancyInUse",
-  APP_IN_USE = "appInUse",
-  IDENTITY_IN_USE = "identityInUse",
 }

packages/backend-core/src/context/deprovision.ts

@@ -1,15 +1,18 @@
-import { getGlobalUserParams, getAllApps } from "../db/utils"
+import {
-import { doWithDB } from "../db"
+  getGlobalUserParams,
+  getAllApps,
+  doWithDB,
+  StaticDatabases,
+} from "../db"
 import { doWithGlobalDB } from "../tenancy"
-import { StaticDatabases } from "../db/constants"
+import { App, Tenants, User, Database } from "@budibase/types"
-import { App, Tenants, User } from "@budibase/types"
 
 const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
 const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
 
-const removeTenantFromInfoDB = async (tenantId: string) => {
+async function removeTenantFromInfoDB(tenantId: string) {
   try {
-    await doWithDB(PLATFORM_INFO_DB, async (infoDb: any) => {
+    await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => {
       const tenants = (await infoDb.get(TENANT_DOC)) as Tenants
       tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)
 
@@ -21,9 +24,9 @@ const removeTenantFromInfoDB = async (tenantId: string) => {
   }
 }
 
-export const removeUserFromInfoDB = async (dbUser: User) => {
+export async function removeUserFromInfoDB(dbUser: User) {
-  await doWithDB(PLATFORM_INFO_DB, async (infoDb: any) => {
+  await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => {
-    const keys = [dbUser._id, dbUser.email]
+    const keys = [dbUser._id!, dbUser.email]
     const userDocs = await infoDb.allDocs({
       keys,
       include_docs: true,
@@ -38,7 +41,7 @@ export const removeUserFromInfoDB = async (dbUser: User) => {
   })
 }
 
-const removeUsersFromInfoDB = async (tenantId: string) => {
+async function removeUsersFromInfoDB(tenantId: string) {
   return doWithGlobalDB(tenantId, async (db: any) => {
     try {
       const allUsers = await db.allDocs(
@@ -72,8 +75,8 @@ const removeUsersFromInfoDB = async (tenantId: string) => {
   })
 }
 
-const removeGlobalDB = async (tenantId: string) => {
+async function removeGlobalDB(tenantId: string) {
-  return doWithGlobalDB(tenantId, async (db: any) => {
+  return doWithGlobalDB(tenantId, async (db: Database) => {
     try {
       await db.destroy()
     } catch (err) {
@@ -83,11 +86,11 @@ const removeGlobalDB = async (tenantId: string) => {
   })
 }
 
-const removeTenantApps = async (tenantId: string) => {
+async function removeTenantApps(tenantId: string) {
   try {
     const apps = (await getAllApps({ all: true })) as App[]
     const destroyPromises = apps.map(app =>
-      doWithDB(app.appId, (db: any) => db.destroy())
+      doWithDB(app.appId, (db: Database) => db.destroy())
     )
     await Promise.allSettled(destroyPromises)
   } catch (err) {
@@ -97,7 +100,7 @@ const removeTenantApps = async (tenantId: string) => {
 }
 
 // can't live in tenancy package due to circular dependency on db/utils
-export const deleteTenant = async (tenantId: string) => {
+export async function deleteTenant(tenantId: string) {
   await removeTenantFromInfoDB(tenantId)
   await removeUsersFromInfoDB(tenantId)
   await removeGlobalDB(tenantId)

packages/backend-core/src/context/index.ts

@@ -1,47 +1,32 @@
 import env from "../environment"
-import { SEPARATOR, DocumentType } from "../db/constants"
-import cls from "./FunctionContext"
-import { dangerousGetDB, closeDB } from "../db"
-import { baseGlobalDBName } from "../db/tenancy"
-import { IdentityContext } from "@budibase/types"
-import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants"
-import { ContextKey } from "./constants"
-import PouchDB from "pouchdb"
 import {
-  updateUsing,
+  SEPARATOR,
-  closeWithUsing,
+  DocumentType,
-  setAppTenantId,
+  getDevelopmentAppID,
-  setIdentity,
+  getProdAppID,
-  closeAppDBs,
+  baseGlobalDBName,
-  getContextDB,
+  getDB,
-} from "./utils"
+} from "../db"
-
+import Context from "./Context"
+import { IdentityContext, Database } from "@budibase/types"
+import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants"
+import { ContextMap } from "./constants"
 export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID
 
 // some test cases call functions directly, need to
 // store an app ID to pretend there is a context
 let TEST_APP_ID: string | null = null
 
-export const closeTenancy = async () => {
+export function isMultiTenant() {
-  try {
+  return env.MULTI_TENANCY
-    if (env.USE_COUCH) {
-      const db = getGlobalDB()
-      await closeDB(db)
-    }
-  } catch (err) {
-    // no DB found - skip closing
-    return
-  }
-  // clear from context now that database is closed/task is finished
-  cls.setOnContext(ContextKey.TENANT_ID, null)
-  cls.setOnContext(ContextKey.GLOBAL_DB, null)
 }
 
-// export const isDefaultTenant = () => {
+export function isTenantIdSet() {
-//   return getTenantId() === DEFAULT_TENANT_ID
+  const context = Context.get()
-// }
+  return !!context?.tenantId
+}
 
-export const isMultiTenant = () => {
+export function isTenancyEnabled() {
   return env.MULTI_TENANCY
 }
 
@@ -49,9 +34,9 @@ export const isMultiTenant = () => {
  * Given an app ID this will attempt to retrieve the tenant ID from it.
  * @return {null|string} The tenant ID found within the app ID.
  */
-export const getTenantIDFromAppID = (appId: string) => {
+export function getTenantIDFromAppID(appId: string) {
   if (!appId) {
-    return null
+    return undefined
   }
   if (!isMultiTenant()) {
     return DEFAULT_TENANT_ID
@@ -59,7 +44,7 @@ export const getTenantIDFromAppID = (appId: string) => {
   const split = appId.split(SEPARATOR)
   const hasDev = split[1] === DocumentType.DEV
   if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) {
-    return null
+    return undefined
   }
   if (hasDev) {
     return split[2]
@@ -68,127 +53,125 @@ export const getTenantIDFromAppID = (appId: string) => {
   }
 }
 
-export const doInContext = async (appId: string, task: any) => {
+function updateContext(updates: ContextMap) {
-  // gets the tenant ID from the app ID
+  let context: ContextMap
-  const tenantId = getTenantIDFromAppID(appId)
+  try {
-  return doInTenant(tenantId, async () => {
+    context = Context.get()
-    return doInAppContext(appId, async () => {
+  } catch (err) {
-      return task()
+    // no context, start empty
-    })
+    context = {}
-  })
+  }
+  context = {
+    ...context,
+    ...updates,
+  }
+  return context
 }
 
-export const doInTenant = (tenantId: string | null, task: any) => {
+async function newContext(updates: ContextMap, task: any) {
+  // see if there already is a context setup
+  let context: ContextMap = updateContext(updates)
+  return Context.run(context, task)
+}
+
+export async function doInContext(appId: string, task: any): Promise<any> {
+  const tenantId = getTenantIDFromAppID(appId)
+  return newContext(
+    {
+      tenantId,
+      appId,
+    },
+    task
+  )
+}
+
+export async function doInTenant(
+  tenantId: string | null,
+  task: any
+): Promise<any> {
   // make sure default always selected in single tenancy
   if (!env.MULTI_TENANCY) {
     tenantId = tenantId || DEFAULT_TENANT_ID
   }
-  // the internal function is so that we can re-use an existing
-  // context - don't want to close DB on a parent context
-  async function internal(opts = { existing: false }) {
-    // set the tenant id + global db if this is a new context
-    if (!opts.existing) {
-      updateTenantId(tenantId)
-    }
 
-    try {
+  const updates = tenantId ? { tenantId } : {}
-      // invoke the task
+  return newContext(updates, task)
-      return await task()
-    } finally {
-      await closeWithUsing(ContextKey.TENANCY_IN_USE, () => {
-        return closeTenancy()
-      })
-    }
-  }
-
-  const existing = cls.getFromContext(ContextKey.TENANT_ID) === tenantId
-  return updateUsing(ContextKey.TENANCY_IN_USE, existing, internal)
 }
 
-export const doInAppContext = (appId: string, task: any) => {
+export async function doInAppContext(appId: string, task: any): Promise<any> {
   if (!appId) {
     throw new Error("appId is required")
   }
 
-  const identity = getIdentity()
+  const tenantId = getTenantIDFromAppID(appId)
-
+  const updates: ContextMap = { appId }
-  // the internal function is so that we can re-use an existing
+  if (tenantId) {
-  // context - don't want to close DB on a parent context
+    updates.tenantId = tenantId
-  async function internal(opts = { existing: false }) {
-    // set the app tenant id
-    if (!opts.existing) {
-      setAppTenantId(appId)
   }
-    // set the app ID
+  return newContext(updates, task)
-    cls.setOnContext(ContextKey.APP_ID, appId)
-
-    // preserve the identity
-    if (identity) {
-      setIdentity(identity)
-    }
-    try {
-      // invoke the task
-      return await task()
-    } finally {
-      await closeWithUsing(ContextKey.APP_IN_USE, async () => {
-        await closeAppDBs()
-        await closeTenancy()
-      })
-    }
-  }
-  const existing = cls.getFromContext(ContextKey.APP_ID) === appId
-  return updateUsing(ContextKey.APP_IN_USE, existing, internal)
 }
 
-export const doInIdentityContext = (identity: IdentityContext, task: any) => {
+export async function doInIdentityContext(
+  identity: IdentityContext,
+  task: any
+): Promise<any> {
   if (!identity) {
     throw new Error("identity is required")
   }
 
-  async function internal(opts = { existing: false }) {
+  const context: ContextMap = {
-    if (!opts.existing) {
+    identity,
-      cls.setOnContext(ContextKey.IDENTITY, identity)
+  }
-      // set the tenant so that doInTenant will preserve identity
   if (identity.tenantId) {
-        updateTenantId(identity.tenantId)
+    context.tenantId = identity.tenantId
   }
-    }
+  return newContext(context, task)
-
-    try {
-      // invoke the task
-      return await task()
-    } finally {
-      await closeWithUsing(ContextKey.IDENTITY_IN_USE, async () => {
-        setIdentity(null)
-        await closeTenancy()
-      })
-    }
-  }
-
-  const existing = cls.getFromContext(ContextKey.IDENTITY)
-  return updateUsing(ContextKey.IDENTITY_IN_USE, existing, internal)
 }
 
-export const getIdentity = (): IdentityContext | undefined => {
+export function getIdentity(): IdentityContext | undefined {
   try {
-    return cls.getFromContext(ContextKey.IDENTITY)
+    const context = Context.get()
+    return context?.identity
   } catch (e) {
     // do nothing - identity is not in context
   }
 }
 
-export const updateTenantId = (tenantId: string | null) => {
+export function getTenantId(): string {
-  cls.setOnContext(ContextKey.TENANT_ID, tenantId)
+  if (!isMultiTenant()) {
-  if (env.USE_COUCH) {
+    return DEFAULT_TENANT_ID
-    setGlobalDB(tenantId)
+  }
+  const context = Context.get()
+  const tenantId = context?.tenantId
+  if (!tenantId) {
+    throw new Error("Tenant id not found")
+  }
+  return tenantId
+}
+
+export function getAppId(): string | undefined {
+  const context = Context.get()
+  const foundId = context?.appId
+  if (!foundId && env.isTest() && TEST_APP_ID) {
+    return TEST_APP_ID
+  } else {
+    return foundId
   }
 }
 
-export const updateAppId = async (appId: string) => {
+export function updateTenantId(tenantId?: string) {
+  let context: ContextMap = updateContext({
+    tenantId,
+  })
+  Context.set(context)
+}
+
+export function updateAppId(appId: string) {
+  let context: ContextMap = updateContext({
+    appId,
+  })
   try {
-    // have to close first, before removing the databases from context
+    Context.set(context)
-    await closeAppDBs()
-    cls.setOnContext(ContextKey.APP_ID, appId)
   } catch (err) {
     if (env.isTest()) {
       TEST_APP_ID = appId
@@ -198,70 +181,43 @@ export const updateAppId = async (appId: string) => {
   }
 }
 
-export const setGlobalDB = (tenantId: string | null) => {
+export function getGlobalDB(): Database {
-  const dbName = baseGlobalDBName(tenantId)
+  const context = Context.get()
-  const db = dangerousGetDB(dbName)
+  if (!context || (env.MULTI_TENANCY && !context.tenantId)) {
-  cls.setOnContext(ContextKey.GLOBAL_DB, db)
-  return db
-}
-
-export const getGlobalDB = () => {
-  const db = cls.getFromContext(ContextKey.GLOBAL_DB)
-  if (!db) {
     throw new Error("Global DB not found")
   }
-  return db
+  return getDB(baseGlobalDBName(context?.tenantId))
-}
-
-export const isTenantIdSet = () => {
-  const tenantId = cls.getFromContext(ContextKey.TENANT_ID)
-  return !!tenantId
-}
-
-export const getTenantId = () => {
-  if (!isMultiTenant()) {
-    return DEFAULT_TENANT_ID
-  }
-  const tenantId = cls.getFromContext(ContextKey.TENANT_ID)
-  if (!tenantId) {
-    throw new Error("Tenant id not found")
-  }
-  return tenantId
-}
-
-export const getAppId = () => {
-  const foundId = cls.getFromContext(ContextKey.APP_ID)
-  if (!foundId && env.isTest() && TEST_APP_ID) {
-    return TEST_APP_ID
-  } else {
-    return foundId
-  }
-}
-
-export const isTenancyEnabled = () => {
-  return env.MULTI_TENANCY
 }
 
 /**
- * Opens the app database based on whatever the request
+ * Gets the app database based on whatever the request
  * contained, dev or prod.
  */
-export const getAppDB = (opts?: any) => {
+export function getAppDB(opts?: any): Database {
-  return getContextDB(ContextKey.CURRENT_DB, opts)
+  const appId = getAppId()
+  return getDB(appId, opts)
 }
 
 /**
  * This specifically gets the prod app ID, if the request
- * contained a development app ID, this will open the prod one.
+ * contained a development app ID, this will get the prod one.
  */
-export const getProdAppDB = (opts?: any) => {
+export function getProdAppDB(opts?: any): Database {
-  return getContextDB(ContextKey.PROD_DB, opts)
+  const appId = getAppId()
+  if (!appId) {
+    throw new Error("Unable to retrieve prod DB - no app ID.")
+  }
+  return getDB(getProdAppID(appId), opts)
 }
 
 /**
  * This specifically gets the dev app ID, if the request
- * contained a prod app ID, this will open the dev one.
+ * contained a prod app ID, this will get the dev one.
  */
-export const getDevAppDB = (opts?: any) => {
+export function getDevAppDB(opts?: any): Database {
-  return getContextDB(ContextKey.DEV_DB, opts)
+  const appId = getAppId()
+  if (!appId) {
+    throw new Error("Unable to retrieve dev DB - no app ID.")
+  }
+  return getDB(getDevelopmentAppID(appId), opts)
 }

packages/backend-core/src/context/tests/index.spec.js

@@ -1,18 +1,9 @@
-import "../../../tests"
+require("../../../tests")
-import * as context from ".."
+const context = require("../")
-import { DEFAULT_TENANT_ID } from "../../constants"
+const { DEFAULT_TENANT_ID } = require("../../constants")
-import env from "../../environment"
+const env = require("../../environment")
-
-// must use require to spy index file exports due to known issue in jest
-const dbUtils = require("../../db")
-jest.spyOn(dbUtils, "closeDB")
-jest.spyOn(dbUtils, "dangerousGetDB")
 
 describe("context", () => {
-  beforeEach(() => {
-    jest.clearAllMocks()
-  })
-
   describe("doInTenant", () => {
     describe("single-tenancy", () => {
       it("defaults to the default tenant", () => {
@@ -25,8 +16,6 @@ describe("context", () => {
           const db = context.getGlobalDB()
           expect(db.name).toBe("global-db")
         })
-        expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
-        expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
       })
     })
 
@@ -40,7 +29,7 @@ describe("context", () => {
           let error
           try {
             context.getTenantId()
-          } catch (e: any) {
+          } catch (e) {
             error = e
           }
           expect(error.message).toBe("Tenant id not found")
@@ -59,7 +48,7 @@ describe("context", () => {
           let error
           try {
             context.getGlobalDB()
-          } catch (e: any) {
+          } catch (e) {
             error = e
           }
           expect(error.message).toBe("Global DB not found")
@@ -85,8 +74,6 @@ describe("context", () => {
           const db = context.getGlobalDB()
           expect(db.name).toBe("test_global-db")
         })
-        expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
-        expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
       })
 
       it("sets the tenant id when nested with same tenant id", async () => {
@@ -121,10 +108,6 @@ describe("context", () => {
             })
           })
         })
-
-        // only 1 db is opened and closed
-        expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
-        expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
       })
 
       it("sets different tenant id inside another context", () => {

packages/backend-core/src/context/utils.ts

@@ -1,109 +0,0 @@
-import {
-  DEFAULT_TENANT_ID,
-  getAppId,
-  getTenantIDFromAppID,
-  updateTenantId,
-} from "./index"
-import cls from "./FunctionContext"
-import { IdentityContext } from "@budibase/types"
-import { ContextKey } from "./constants"
-import { dangerousGetDB, closeDB } from "../db"
-import { isEqual } from "lodash"
-import { getDevelopmentAppID, getProdAppID } from "../db/conversions"
-import env from "../environment"
-
-export async function updateUsing(
-  usingKey: string,
-  existing: boolean,
-  internal: (opts: { existing: boolean }) => Promise<any>
-) {
-  const using = cls.getFromContext(usingKey)
-  if (using && existing) {
-    cls.setOnContext(usingKey, using + 1)
-    return internal({ existing: true })
-  } else {
-    return cls.run(async () => {
-      cls.setOnContext(usingKey, 1)
-      return internal({ existing: false })
-    })
-  }
-}
-
-export async function closeWithUsing(
-  usingKey: string,
-  closeFn: () => Promise<any>
-) {
-  const using = cls.getFromContext(usingKey)
-  if (!using || using <= 1) {
-    await closeFn()
-  } else {
-    cls.setOnContext(usingKey, using - 1)
-  }
-}
-
-export const setAppTenantId = (appId: string) => {
-  const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID
-  updateTenantId(appTenantId)
-}
-
-export const setIdentity = (identity: IdentityContext | null) => {
-  cls.setOnContext(ContextKey.IDENTITY, identity)
-}
-
-// this function makes sure the PouchDB objects are closed and
-// fully deleted when finished - this protects against memory leaks
-export async function closeAppDBs() {
-  const dbKeys = [ContextKey.CURRENT_DB, ContextKey.PROD_DB, ContextKey.DEV_DB]
-  for (let dbKey of dbKeys) {
-    const db = cls.getFromContext(dbKey)
-    if (!db) {
-      continue
-    }
-    await closeDB(db)
-    // clear the DB from context, incase someone tries to use it again
-    cls.setOnContext(dbKey, null)
-  }
-  // clear the app ID now that the databases are closed
-  if (cls.getFromContext(ContextKey.APP_ID)) {
-    cls.setOnContext(ContextKey.APP_ID, null)
-  }
-  if (cls.getFromContext(ContextKey.DB_OPTS)) {
-    cls.setOnContext(ContextKey.DB_OPTS, null)
-  }
-}
-
-export function getContextDB(key: string, opts: any) {
-  const dbOptsKey = `${key}${ContextKey.DB_OPTS}`
-  let storedOpts = cls.getFromContext(dbOptsKey)
-  let db = cls.getFromContext(key)
-  if (db && isEqual(opts, storedOpts)) {
-    return db
-  }
-
-  const appId = getAppId()
-  let toUseAppId
-
-  switch (key) {
-    case ContextKey.CURRENT_DB:
-      toUseAppId = appId
-      break
-    case ContextKey.PROD_DB:
-      toUseAppId = getProdAppID(appId)
-      break
-    case ContextKey.DEV_DB:
-      toUseAppId = getDevelopmentAppID(appId)
-      break
-  }
-  db = dangerousGetDB(toUseAppId, opts)
-  try {
-    cls.setOnContext(key, db)
-    if (opts) {
-      cls.setOnContext(dbOptsKey, opts)
-    }
-  } catch (err) {
-    if (!env.isTest()) {
-      throw err
-    }
-  }
-  return db
-}

packages/backend-core/src/db/Replication.ts

@@ -1,4 +1,4 @@
-import { dangerousGetDB, closeDB } from "."
+import { getPouchDB, closePouchDB } from "./couch/pouchDB"
 import { DocumentType } from "./constants"
 
 class Replication {
@@ -12,12 +12,12 @@ class Replication {
    * @param {String} target - the DB you want to replicate to, or rollback from
    */
   constructor({ source, target }: any) {
-    this.source = dangerousGetDB(source)
+    this.source = getPouchDB(source)
-    this.target = dangerousGetDB(target)
+    this.target = getPouchDB(target)
   }
 
   close() {
-    return Promise.all([closeDB(this.source), closeDB(this.target)])
+    return Promise.all([closePouchDB(this.source), closePouchDB(this.target)])
   }
 
   promisify(operation: any, opts = {}) {
@@ -68,7 +68,7 @@ class Replication {
   async rollback() {
     await this.target.destroy()
     // Recreate the DB again
-    this.target = dangerousGetDB(this.target.name)
+    this.target = getPouchDB(this.target.name)
     // take the opportunity to remove deleted tombstones
     await this.replicate()
   }

packages/backend-core/src/db/conversions.ts

@@ -1,32 +1,33 @@
+import { APP_DEV_PREFIX, APP_PREFIX } from "./constants"
+import { App } from "@budibase/types"
 const NO_APP_ERROR = "No app provided"
-const { APP_DEV_PREFIX, APP_PREFIX } = require("./constants")
 
-exports.isDevAppID = appId => {
+export function isDevAppID(appId?: string) {
   if (!appId) {
     throw NO_APP_ERROR
   }
   return appId.startsWith(APP_DEV_PREFIX)
 }
 
-exports.isProdAppID = appId => {
+export function isProdAppID(appId?: string) {
   if (!appId) {
     throw NO_APP_ERROR
   }
-  return appId.startsWith(APP_PREFIX) && !exports.isDevAppID(appId)
+  return appId.startsWith(APP_PREFIX) && !isDevAppID(appId)
 }
 
-exports.isDevApp = app => {
+export function isDevApp(app: App) {
   if (!app) {
     throw NO_APP_ERROR
   }
-  return exports.isDevAppID(app.appId)
+  return isDevAppID(app.appId)
 }
 
 /**
  * Generates a development app ID from a real app ID.
  * @returns {string} the dev app ID which can be used for dev database.
  */
-exports.getDevelopmentAppID = appId => {
+export function getDevelopmentAppID(appId: string) {
   if (!appId || appId.startsWith(APP_DEV_PREFIX)) {
     return appId
   }
@@ -36,12 +37,12 @@ exports.getDevelopmentAppID = appId => {
   const rest = split.join(APP_PREFIX)
   return `${APP_DEV_PREFIX}${rest}`
 }
-exports.getDevAppID = exports.getDevelopmentAppID
+export const getDevAppID = getDevelopmentAppID
 
 /**
  * Convert a development app ID to a deployed app ID.
  */
-exports.getProdAppID = appId => {
+export function getProdAppID(appId: string) {
   if (!appId || !appId.startsWith(APP_DEV_PREFIX)) {
     return appId
   }
@@ -52,7 +53,7 @@ exports.getProdAppID = appId => {
   return `${APP_PREFIX}${rest}`
 }
 
-exports.extractAppUUID = id => {
+export function extractAppUUID(id: string) {
   const split = id?.split("_") || []
   return split.length ? split[split.length - 1] : null
 }

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -0,0 +1,193 @@
+import Nano from "nano"
+import {
+  AllDocsResponse,
+  AnyDocument,
+  Database,
+  DatabaseOpts,
+  DatabaseQueryOpts,
+  DatabasePutOpts,
+  DatabaseCreateIndexOpts,
+  DatabaseDeleteIndexOpts,
+  Document,
+  isDocument,
+} from "@budibase/types"
+import { getCouchInfo } from "./connections"
+import { directCouchCall } from "./utils"
+import { getPouchDB } from "./pouchDB"
+import { WriteStream, ReadStream } from "fs"
+
+export class DatabaseImpl implements Database {
+  public readonly name: string
+  private static nano: Nano.ServerScope
+  private readonly pouchOpts: DatabaseOpts
+
+  constructor(dbName?: string, opts?: DatabaseOpts) {
+    if (dbName == null) {
+      throw new Error("Database name cannot be undefined.")
+    }
+    this.name = dbName
+    this.pouchOpts = opts || {}
+    if (!DatabaseImpl.nano) {
+      DatabaseImpl.init()
+    }
+  }
+
+  static init() {
+    const couchInfo = getCouchInfo()
+    DatabaseImpl.nano = Nano({
+      url: couchInfo.url,
+      requestDefaults: {
+        headers: {
+          Authorization: couchInfo.cookie,
+        },
+      },
+      parseUrl: false,
+    })
+  }
+
+  async exists() {
+    let response = await directCouchCall(`/${this.name}`, "HEAD")
+    return response.status === 200
+  }
+
+  async checkSetup() {
+    let shouldCreate = !this.pouchOpts?.skip_setup
+    // check exists in a lightweight fashion
+    let exists = await this.exists()
+    if (!shouldCreate && !exists) {
+      throw new Error("DB does not exist")
+    }
+    if (!exists) {
+      await DatabaseImpl.nano.db.create(this.name)
+    }
+    return DatabaseImpl.nano.db.use(this.name)
+  }
+
+  private async updateOutput(fnc: any) {
+    try {
+      return await fnc()
+    } catch (err: any) {
+      if (err.statusCode) {
+        err.status = err.statusCode
+      }
+      throw err
+    }
+  }
+
+  async get<T>(id?: string): Promise<T | any> {
+    const db = await this.checkSetup()
+    if (!id) {
+      throw new Error("Unable to get doc without a valid _id.")
+    }
+    return this.updateOutput(() => db.get(id))
+  }
+
+  async remove(idOrDoc: string | Document, rev?: string) {
+    const db = await this.checkSetup()
+    let _id: string
+    let _rev: string
+
+    if (isDocument(idOrDoc)) {
+      _id = idOrDoc._id!
+      _rev = idOrDoc._rev!
+    } else {
+      _id = idOrDoc
+      _rev = rev!
+    }
+
+    if (!_id || !_rev) {
+      throw new Error("Unable to remove doc without a valid _id and _rev.")
+    }
+    return this.updateOutput(() => db.destroy(_id, _rev))
+  }
+
+  async put(document: AnyDocument, opts?: DatabasePutOpts) {
+    if (!document._id) {
+      throw new Error("Cannot store document without _id field.")
+    }
+    const db = await this.checkSetup()
+    if (!document.createdAt) {
+      document.createdAt = new Date().toISOString()
+    }
+    document.updatedAt = new Date().toISOString()
+    if (opts?.force && document._id) {
+      try {
+        const existing = await this.get(document._id)
+        if (existing) {
+          document._rev = existing._rev
+        }
+      } catch (err: any) {
+        if (err.status !== 404) {
+          throw err
+        }
+      }
+    }
+    return this.updateOutput(() => db.insert(document))
+  }
+
+  async bulkDocs(documents: AnyDocument[]) {
+    const db = await this.checkSetup()
+    return this.updateOutput(() => db.bulk({ docs: documents }))
+  }
+
+  async allDocs<T>(params: DatabaseQueryOpts): Promise<AllDocsResponse<T>> {
+    const db = await this.checkSetup()
+    return this.updateOutput(() => db.list(params))
+  }
+
+  async query<T>(
+    viewName: string,
+    params: DatabaseQueryOpts
+  ): Promise<AllDocsResponse<T>> {
+    const db = await this.checkSetup()
+    const [database, view] = viewName.split("/")
+    return this.updateOutput(() => db.view(database, view, params))
+  }
+
+  async destroy() {
+    try {
+      await DatabaseImpl.nano.db.destroy(this.name)
+    } catch (err: any) {
+      // didn't exist, don't worry
+      if (err.statusCode === 404) {
+        return
+      } else {
+        throw { ...err, status: err.statusCode }
+      }
+    }
+  }
+
+  async compact() {
+    const db = await this.checkSetup()
+    return this.updateOutput(() => db.compact())
+  }
+
+  // All below functions are in-frequently called, just utilise PouchDB
+  // for them as it implements them better than we can
+  async dump(stream: WriteStream, opts?: { filter?: any }) {
+    const pouch = getPouchDB(this.name)
+    // @ts-ignore
+    return pouch.dump(stream, opts)
+  }
+
+  async load(stream: ReadStream) {
+    const pouch = getPouchDB(this.name)
+    // @ts-ignore
+    return pouch.load(stream)
+  }
+
+  async createIndex(opts: DatabaseCreateIndexOpts) {
+    const pouch = getPouchDB(this.name)
+    return pouch.createIndex(opts)
+  }
+
+  async deleteIndex(opts: DatabaseDeleteIndexOpts) {
+    const pouch = getPouchDB(this.name)
+    return pouch.deleteIndex(opts)
+  }
+
+  async getIndexes() {
+    const pouch = getPouchDB(this.name)
+    return pouch.getIndexes()
+  }
+}

packages/backend-core/src/db/couch/connections.ts

@@ -1,6 +1,37 @@
-import PouchDB from "pouchdb"
+import env from "../../environment"
-import env from "../environment"
+
-import { PouchOptions } from "@budibase/types"
+export const getCouchInfo = () => {
+  const urlInfo = getUrlInfo()
+  let username
+  let password
+  if (env.COUCH_DB_USERNAME) {
+    // set from env
+    username = env.COUCH_DB_USERNAME
+  } else if (urlInfo.auth.username) {
+    // set from url
+    username = urlInfo.auth.username
+  } else if (!env.isTest()) {
+    throw new Error("CouchDB username not set")
+  }
+  if (env.COUCH_DB_PASSWORD) {
+    // set from env
+    password = env.COUCH_DB_PASSWORD
+  } else if (urlInfo.auth.password) {
+    // set from url
+    password = urlInfo.auth.password
+  } else if (!env.isTest()) {
+    throw new Error("CouchDB password not set")
+  }
+  const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
+  return {
+    url: urlInfo.url!,
+    auth: {
+      username: username,
+      password: password,
+    },
+    cookie: `Basic ${authCookie}`,
+  }
+}
 
 export const getUrlInfo = (url = env.COUCH_DB_URL) => {
   let cleanUrl, username, password, host
@@ -44,85 +75,3 @@ export const getUrlInfo = (url = env.COUCH_DB_URL) => {
     },
   }
 }
-
-export const getCouchInfo = () => {
-  const urlInfo = getUrlInfo()
-  let username
-  let password
-  if (env.COUCH_DB_USERNAME) {
-    // set from env
-    username = env.COUCH_DB_USERNAME
-  } else if (urlInfo.auth.username) {
-    // set from url
-    username = urlInfo.auth.username
-  } else if (!env.isTest()) {
-    throw new Error("CouchDB username not set")
-  }
-  if (env.COUCH_DB_PASSWORD) {
-    // set from env
-    password = env.COUCH_DB_PASSWORD
-  } else if (urlInfo.auth.password) {
-    // set from url
-    password = urlInfo.auth.password
-  } else if (!env.isTest()) {
-    throw new Error("CouchDB password not set")
-  }
-  const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
-  return {
-    url: urlInfo.url,
-    auth: {
-      username: username,
-      password: password,
-    },
-    cookie: `Basic ${authCookie}`,
-  }
-}
-
-/**
- * Return a constructor for PouchDB.
- * This should be rarely used outside of the main application config.
- * Exposed for exceptional cases such as in-memory views.
- */
-export const getPouch = (opts: PouchOptions = {}) => {
-  let { url, cookie } = getCouchInfo()
-  let POUCH_DB_DEFAULTS = {
-    prefix: url,
-    fetch: (url: string, opts: any) => {
-      // use a specific authorization cookie - be very explicit about how we authenticate
-      opts.headers.set("Authorization", cookie)
-      return PouchDB.fetch(url, opts)
-    },
-  }
-
-  if (opts.inMemory) {
-    const inMemory = require("pouchdb-adapter-memory")
-    PouchDB.plugin(inMemory)
-    POUCH_DB_DEFAULTS = {
-      prefix: undefined,
-      // @ts-ignore
-      adapter: "memory",
-    }
-  }
-
-  if (opts.onDisk) {
-    POUCH_DB_DEFAULTS = {
-      prefix: undefined,
-      // @ts-ignore
-      adapter: "leveldb",
-    }
-  }
-
-  if (opts.replication) {
-    const replicationStream = require("pouchdb-replication-stream")
-    PouchDB.plugin(replicationStream.plugin)
-    // @ts-ignore
-    PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
-  }
-
-  if (opts.find) {
-    const find = require("pouchdb-find")
-    PouchDB.plugin(find)
-  }
-
-  return PouchDB.defaults(POUCH_DB_DEFAULTS)
-}

packages/backend-core/src/db/couch/index.ts

@@ -0,0 +1,4 @@
+export * from "./connections"
+export * from "./DatabaseImpl"
+export * from "./utils"
+export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB"

packages/backend-core/src/db/couch/pouchDB.ts

@@ -0,0 +1,97 @@
+import PouchDB from "pouchdb"
+import env from "../../environment"
+import { PouchOptions } from "@budibase/types"
+import { getCouchInfo } from "./connections"
+
+let Pouch: any
+let initialised = false
+
+/**
+ * Return a constructor for PouchDB.
+ * This should be rarely used outside of the main application config.
+ * Exposed for exceptional cases such as in-memory views.
+ */
+export const getPouch = (opts: PouchOptions = {}) => {
+  let { url, cookie } = getCouchInfo()
+  let POUCH_DB_DEFAULTS = {
+    prefix: url,
+    fetch: (url: string, opts: any) => {
+      // use a specific authorization cookie - be very explicit about how we authenticate
+      opts.headers.set("Authorization", cookie)
+      return PouchDB.fetch(url, opts)
+    },
+  }
+
+  if (opts.inMemory) {
+    const inMemory = require("pouchdb-adapter-memory")
+    PouchDB.plugin(inMemory)
+    POUCH_DB_DEFAULTS = {
+      // @ts-ignore
+      adapter: "memory",
+    }
+  }
+
+  if (opts.onDisk) {
+    POUCH_DB_DEFAULTS = {
+      // @ts-ignore
+      adapter: "leveldb",
+    }
+  }
+
+  if (opts.replication) {
+    const replicationStream = require("pouchdb-replication-stream")
+    PouchDB.plugin(replicationStream.plugin)
+    // @ts-ignore
+    PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
+  }
+
+  if (opts.find) {
+    const find = require("pouchdb-find")
+    PouchDB.plugin(find)
+  }
+
+  return PouchDB.defaults(POUCH_DB_DEFAULTS)
+}
+
+export function init(opts?: PouchOptions) {
+  Pouch = getPouch(opts)
+  initialised = true
+}
+
+const checkInitialised = () => {
+  if (!initialised) {
+    throw new Error("init has not been called")
+  }
+}
+
+export function getPouchDB(dbName: string, opts?: any): PouchDB.Database {
+  checkInitialised()
+  const db = new Pouch(dbName, opts)
+  const dbPut = db.put
+  db.put = async (doc: any, options = {}) => {
+    if (!doc.createdAt) {
+      doc.createdAt = new Date().toISOString()
+    }
+    doc.updatedAt = new Date().toISOString()
+    return dbPut(doc, options)
+  }
+  db.exists = async () => {
+    const info = await db.info()
+    return !info.error
+  }
+  return db
+}
+
+// use this function if you have called getPouchDB - close
+// the databases you've opened once finished
+export async function closePouchDB(db: PouchDB.Database) {
+  if (!db || env.isTest()) {
+    return
+  }
+  try {
+    // specifically await so that if there is an error, it can be ignored
+    return await db.close()
+  } catch (err) {
+    // ignore error, already closed
+  }
+}

packages/backend-core/src/db/couch/utils.ts

@@ -0,0 +1,36 @@
+import { getCouchInfo } from "./connections"
+import fetch from "node-fetch"
+import { checkSlashesInUrl } from "../../helpers"
+
+export async function directCouchCall(
+  path: string,
+  method: string = "GET",
+  body?: any
+) {
+  let { url, cookie } = getCouchInfo()
+  const couchUrl = `${url}/${path}`
+  const params: any = {
+    method: method,
+    headers: {
+      Authorization: cookie,
+    },
+  }
+  if (body && method !== "GET") {
+    params.body = JSON.stringify(body)
+    params.headers["Content-Type"] = "application/json"
+  }
+  return await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params)
+}
+
+export async function directCouchQuery(
+  path: string,
+  method: string = "GET",
+  body?: any
+) {
+  const response = await directCouchCall(path, method, body)
+  if (response.status < 300) {
+    return await response.json()
+  } else {
+    throw "Cannot connect to CouchDB instance"
+  }
+}

packages/backend-core/src/db/db.ts

@@ -0,0 +1,46 @@
+import env from "../environment"
+import { directCouchQuery, getPouchDB } from "./couch"
+import { CouchFindOptions, Database } from "@budibase/types"
+import { DatabaseImpl } from "../db"
+
+const dbList = new Set()
+
+export function getDB(dbName?: string, opts?: any): Database {
+  // TODO: once using the test image, need to remove this
+  if (env.isTest()) {
+    dbList.add(dbName)
+    // @ts-ignore
+    return getPouchDB(dbName, opts)
+  }
+  return new DatabaseImpl(dbName, opts)
+}
+
+// we have to use a callback for this so that we can close
+// the DB when we're done, without this manual requests would
+// need to close the database when done with it to avoid memory leaks
+export async function doWithDB(dbName: string, cb: any, opts = {}) {
+  const db = getDB(dbName, opts)
+  // need this to be async so that we can correctly close DB after all
+  // async operations have been completed
+  return await cb(db)
+}
+
+export function allDbs() {
+  if (!env.isTest()) {
+    throw new Error("Cannot be used outside test environment.")
+  }
+  return [...dbList]
+}
+
+export async function directCouchAllDbs(queryString?: string) {
+  let couchPath = "/_all_dbs"
+  if (queryString) {
+    couchPath += `?${queryString}`
+  }
+  return await directCouchQuery(couchPath)
+}
+
+export async function directCouchFind(dbName: string, opts: CouchFindOptions) {
+  const json = await directCouchQuery(`${dbName}/_find`, "POST", opts)
+  return { rows: json.docs, bookmark: json.bookmark }
+}

packages/backend-core/src/db/index.ts

@@ -1,133 +1,7 @@
-import * as pouch from "./pouch"
+export * from "./couch"
-import env from "../environment"
+export * from "./db"
-import { checkSlashesInUrl } from "../helpers"
+export * from "./utils"
-import fetch from "node-fetch"
+export * from "./views"
-import { PouchOptions, CouchFindOptions } from "@budibase/types"
+export * from "./constants"
-import PouchDB from "pouchdb"
+export * from "./conversions"
-
+export * from "./tenancy"
-const openDbs: string[] = []
-let Pouch: any
-let initialised = false
-const dbList = new Set()
-
-if (env.MEMORY_LEAK_CHECK) {
-  setInterval(() => {
-    console.log("--- OPEN DBS ---")
-    console.log(openDbs)
-  }, 5000)
-}
-
-const put =
-  (dbPut: any) =>
-  async (doc: any, options = {}) => {
-    if (!doc.createdAt) {
-      doc.createdAt = new Date().toISOString()
-    }
-    doc.updatedAt = new Date().toISOString()
-    return dbPut(doc, options)
-  }
-
-const checkInitialised = () => {
-  if (!initialised) {
-    throw new Error("init has not been called")
-  }
-}
-
-export function init(opts?: PouchOptions) {
-  Pouch = pouch.getPouch(opts)
-  initialised = true
-}
-
-// NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION
-// this function is prone to leaks, should only be used
-// in situations that using the function doWithDB does not work
-export function dangerousGetDB(dbName: string, opts?: any): PouchDB.Database {
-  checkInitialised()
-  if (env.isTest()) {
-    dbList.add(dbName)
-  }
-  const db = new Pouch(dbName, opts)
-  if (env.MEMORY_LEAK_CHECK) {
-    openDbs.push(db.name)
-  }
-  const dbPut = db.put
-  db.put = put(dbPut)
-  return db
-}
-
-// use this function if you have called dangerousGetDB - close
-// the databases you've opened once finished
-export async function closeDB(db: PouchDB.Database) {
-  if (!db || env.isTest()) {
-    return
-  }
-  if (env.MEMORY_LEAK_CHECK) {
-    openDbs.splice(openDbs.indexOf(db.name), 1)
-  }
-  try {
-    // specifically await so that if there is an error, it can be ignored
-    return await db.close()
-  } catch (err) {
-    // ignore error, already closed
-  }
-}
-
-// we have to use a callback for this so that we can close
-// the DB when we're done, without this manual requests would
-// need to close the database when done with it to avoid memory leaks
-export async function doWithDB(dbName: string, cb: any, opts = {}) {
-  const db = dangerousGetDB(dbName, opts)
-  // need this to be async so that we can correctly close DB after all
-  // async operations have been completed
-  try {
-    return await cb(db)
-  } finally {
-    await closeDB(db)
-  }
-}
-
-export function allDbs() {
-  if (!env.isTest()) {
-    throw new Error("Cannot be used outside test environment.")
-  }
-  checkInitialised()
-  return [...dbList]
-}
-
-export async function directCouchQuery(
-  path: string,
-  method: string = "GET",
-  body?: any
-) {
-  let { url, cookie } = pouch.getCouchInfo()
-  const couchUrl = `${url}/${path}`
-  const params: any = {
-    method: method,
-    headers: {
-      Authorization: cookie,
-    },
-  }
-  if (body && method !== "GET") {
-    params.body = JSON.stringify(body)
-    params.headers["Content-Type"] = "application/json"
-  }
-  const response = await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params)
-  if (response.status < 300) {
-    return await response.json()
-  } else {
-    throw "Cannot connect to CouchDB instance"
-  }
-}
-
-export async function directCouchAllDbs(queryString?: string) {
-  let couchPath = "/_all_dbs"
-  if (queryString) {
-    couchPath += `?${queryString}`
-  }
-  return await directCouchQuery(couchPath)
-}
-
-export async function directCouchFind(dbName: string, opts: CouchFindOptions) {
-  const json = await directCouchQuery(`${dbName}/_find`, "POST", opts)
-  return { rows: json.docs, bookmark: json.bookmark }
-}

packages/backend-core/src/db/tests/index.spec.js

@@ -1,11 +1,11 @@
 require("../../../tests")
-const { dangerousGetDB } = require("../")
+const { getDB } = require("../")
 
 describe("db", () => { 
   
   describe("getDB", () => {
     it("returns a db", async () => {
-      const db = dangerousGetDB("test")
+      const db = getDB("test")
       expect(db).toBeDefined()
       expect(db._adapter).toBe("memory")
       expect(db.prefix).toBe("_pouch_")
@@ -13,7 +13,7 @@ describe("db", () => {
     })
 
     it("uses the custom put function", async () => {
-      const db = dangerousGetDB("test")
+      const db = getDB("test")
       let doc = { _id: "test" }
       await db.put(doc)
       doc = await db.get(doc._id)

packages/backend-core/src/db/tests/pouch.spec.js

@@ -1,5 +1,5 @@
 require("../../../tests")
-const getUrlInfo = require("../pouch").getUrlInfo
+const getUrlInfo = require("../couch").getUrlInfo
 
 describe("pouch", () => {
   describe("Couch DB URL parsing", () => {

packages/backend-core/src/db/tests/utils.spec.js

@@ -1,4 +1,4 @@
-require("../../../tests");
+require("../../../tests")
 const {
   generateAppID,
   getDevelopmentAppID,
@@ -8,8 +8,8 @@ const {
   getPlatformUrl,
   getScopedConfig
 } = require("../utils")
-const tenancy = require("../../tenancy");
+const tenancy = require("../../tenancy")
-const { Configs, DEFAULT_TENANT_ID } = require("../../constants");
+const { Config, DEFAULT_TENANT_ID } = require("../../constants")
 const env = require("../../environment")
 
 describe("utils", () => {
@@ -77,7 +77,7 @@ const setDbPlatformUrl = async () => {
   const db = tenancy.getGlobalDB()
   db.put({
     _id: "config_settings",
-    type: Configs.SETTINGS,
+    type: Config.SETTINGS,
     config: {
       platformUrl: DB_URL
     }
@@ -178,7 +178,7 @@ describe("getScopedConfig", () => {
       await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
         await setDbPlatformUrl()
         const db = tenancy.getGlobalDB()
-        const config = await getScopedConfig(db, { type: Configs.SETTINGS })
+        const config = await getScopedConfig(db, { type: Config.SETTINGS })
         expect(config.platformUrl).toBe(DB_URL)
       })
     })
@@ -186,7 +186,7 @@ describe("getScopedConfig", () => {
     it("returns the platform url without an existing config", async () => {
       await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
         const db = tenancy.getGlobalDB()
-        const config = await getScopedConfig(db, { type: Configs.SETTINGS })
+        const config = await getScopedConfig(db, { type: Config.SETTINGS })
         expect(config.platformUrl).toBe(DEFAULT_URL)
       })
     })

packages/backend-core/src/db/utils.ts

@@ -1,5 +1,5 @@
 import { newid } from "../hashing"
-import { DEFAULT_TENANT_ID, Configs } from "../constants"
+import { DEFAULT_TENANT_ID, Config } from "../constants"
 import env from "../environment"
 import {
   SEPARATOR,
@@ -10,12 +10,12 @@ import {
 } from "./constants"
 import { getTenantId, getGlobalDB } from "../context"
 import { getGlobalDBName } from "./tenancy"
-import { doWithDB, allDbs, directCouchAllDbs } from "./index"
+import { doWithDB, allDbs, directCouchAllDbs } from "./db"
 import { getAppMetadata } from "../cache/appMetadata"
 import { isDevApp, isDevAppID, getProdAppID } from "./conversions"
 import { APP_PREFIX } from "./constants"
 import * as events from "../events"
-import { App } from "@budibase/types"
+import { App, Database } from "@budibase/types"
 
 export * from "./constants"
 export * from "./conversions"
@@ -26,7 +26,7 @@ export * from "./tenancy"
  * Generates a new app ID.
  * @returns {string} The new app ID which the app doc can be stored under.
  */
-export const generateAppID = (tenantId = null) => {
+export const generateAppID = (tenantId?: string | null) => {
   let id = APP_PREFIX
   if (tenantId) {
     id += `${tenantId}${SEPARATOR}`
@@ -251,11 +251,11 @@ export function generateRoleID(id: any) {
 /**
  * Gets parameters for retrieving a role, this is a utility function for the getDocParams function.
  */
-export function getRoleParams(roleId = null, otherProps = {}) {
+export function getRoleParams(roleId?: string | null, otherProps = {}) {
   return getDocParams(DocumentType.ROLE, roleId, otherProps)
 }
 
-export function getStartEndKeyURL(baseKey: any, tenantId = null) {
+export function getStartEndKeyURL(baseKey: any, tenantId?: string) {
   const tenancy = tenantId ? `${SEPARATOR}${tenantId}` : ""
   return `startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"`
 }
@@ -392,20 +392,10 @@ export async function getDevAppIDs() {
 }
 
 export async function dbExists(dbName: any) {
-  let exists = false
   return doWithDB(
     dbName,
-    async (db: any) => {
+    async (db: Database) => {
-      try {
+      return await db.exists()
-        // check if database exists
-        const info = await db.info()
-        if (info && !info.error) {
-          exists = true
-        }
-      } catch (err) {
-        exists = false
-      }
-      return exists
     },
     { skip_setup: true }
   )
@@ -504,7 +494,7 @@ export const getScopedFullConfig = async function (
   )[0]
 
   // custom logic for settings doc
-  if (type === Configs.SETTINGS) {
+  if (type === Config.SETTINGS) {
     if (scopedConfig && scopedConfig.doc) {
       // overrides affected by environment variables
       scopedConfig.doc.config.platformUrl = await getPlatformUrl({
@@ -543,7 +533,7 @@ export const getPlatformUrl = async (opts = { tenantAware: true }) => {
     // get the doc directly instead of with getScopedConfig to prevent loop
     let settings
     try {
-      settings = await db.get(generateConfigID({ type: Configs.SETTINGS }))
+      settings = await db.get(generateConfigID({ type: Config.SETTINGS }))
     } catch (e: any) {
       if (e.status !== 404) {
         throw e

packages/backend-core/src/db/views.ts

@@ -1,8 +1,8 @@
 import { DocumentType, ViewName, DeprecatedViews, SEPARATOR } from "./utils"
 import { getGlobalDB } from "../context"
-import PouchDB from "pouchdb"
 import { StaticDatabases } from "./constants"
 import { doWithDB } from "./"
+import { Database, DatabaseQueryOpts } from "@budibase/types"
 
 const DESIGN_DB = "_design/database"
 
@@ -19,7 +19,7 @@ interface DesignDocument {
   views: any
 }
 
-async function removeDeprecated(db: PouchDB.Database, viewName: ViewName) {
+async function removeDeprecated(db: Database, viewName: ViewName) {
   // @ts-ignore
   if (!DeprecatedViews[viewName]) {
     return
@@ -70,16 +70,13 @@ export const createAccountEmailView = async () => {
       emit(doc.email.toLowerCase(), doc._id)
     }
   }`
-  await doWithDB(
+  await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
-    StaticDatabases.PLATFORM_INFO.name,
-    async (db: PouchDB.Database) => {
     await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL)
-    }
+  })
-  )
 }
 
 export const createUserAppView = async () => {
-  const db = getGlobalDB() as PouchDB.Database
+  const db = getGlobalDB()
   const viewJs = `function(doc) {
     if (doc._id.startsWith("${DocumentType.USER}${SEPARATOR}") && doc.roles) {
       for (let prodAppId of Object.keys(doc.roles)) {
@@ -117,12 +114,9 @@ export const createPlatformUserView = async () => {
       emit(doc._id.toLowerCase(), doc._id)
     }
   }`
-  await doWithDB(
+  await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
-    StaticDatabases.PLATFORM_INFO.name,
-    async (db: PouchDB.Database) => {
     await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
-    }
+  })
-  )
 }
 
 export interface QueryViewOptions {
@@ -131,22 +125,24 @@ export interface QueryViewOptions {
 
 export const queryView = async <T>(
   viewName: ViewName,
-  params: PouchDB.Query.Options<T, T>,
+  params: DatabaseQueryOpts,
-  db: PouchDB.Database,
+  db: Database,
   createFunc: any,
   opts?: QueryViewOptions
 ): Promise<T[] | T | undefined> => {
   try {
-    let response = await db.query<T, T>(`database/${viewName}`, params)
+    let response = await db.query<T>(`database/${viewName}`, params)
     const rows = response.rows
-    const docs = rows.map(row => (params.include_docs ? row.doc : row.value))
+    const docs = rows.map((row: any) =>
+      params.include_docs ? row.doc : row.value
+    )
 
     // if arrayResponse has been requested, always return array regardless of length
     if (opts?.arrayResponse) {
-      return docs
+      return docs as T[]
     } else {
       // return the single document if there is only one
-      return docs.length <= 1 ? docs[0] : docs
+      return docs.length <= 1 ? (docs[0] as T) : (docs as T[])
     }
   } catch (err: any) {
     if (err != null && err.name === "not_found") {
@@ -161,7 +157,7 @@ export const queryView = async <T>(
 
 export const queryPlatformView = async <T>(
   viewName: ViewName,
-  params: PouchDB.Query.Options<T, T>,
+  params: DatabaseQueryOpts,
   opts?: QueryViewOptions
 ): Promise<T[] | T | undefined> => {
   const CreateFuncByName: any = {
@@ -169,19 +165,16 @@ export const queryPlatformView = async <T>(
     [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView,
   }
 
-  return doWithDB(
+  return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
-    StaticDatabases.PLATFORM_INFO.name,
-    async (db: PouchDB.Database) => {
     const createFn = CreateFuncByName[viewName]
     return queryView(viewName, params, db, createFn, opts)
-    }
+  })
-  )
 }
 
 export const queryGlobalView = async <T>(
   viewName: ViewName,
-  params: PouchDB.Query.Options<T, T>,
+  params: DatabaseQueryOpts,
-  db?: PouchDB.Database,
+  db?: Database,
   opts?: QueryViewOptions
 ): Promise<T[] | T | undefined> => {
   const CreateFuncByName: any = {
@@ -192,8 +185,8 @@ export const queryGlobalView = async <T>(
   }
   // can pass DB in if working with something specific
   if (!db) {
-    db = getGlobalDB() as PouchDB.Database
+    db = getGlobalDB()
   }
   const createFn = CreateFuncByName[viewName]
-  return queryView(viewName, params, db, createFn, opts)
+  return queryView(viewName, params, db!, createFn, opts)
 }

packages/backend-core/src/environment.ts

@@ -69,7 +69,6 @@ const env = {
   DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
   DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,
   SERVICE: process.env.SERVICE || "budibase",
-  MEMORY_LEAK_CHECK: process.env.MEMORY_LEAK_CHECK || false,
   LOG_LEVEL: process.env.LOG_LEVEL,
   SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD,
   DEPLOYMENT_ENVIRONMENT:

packages/backend-core/src/events/analytics.ts

@@ -1,7 +1,7 @@
 import env from "../environment"
 import tenancy from "../tenancy"
 import * as dbUtils from "../db/utils"
-import { Configs } from "../constants"
+import { Config } from "../constants"
 import { withCache, TTL, CacheKeys } from "../cache/generic"
 
 export const enabled = async () => {
@@ -45,9 +45,7 @@ const getSettingsDoc = async () => {
   const db = tenancy.getGlobalDB()
   let settings
   try {
-    settings = await db.get(
+    settings = await db.get(dbUtils.generateConfigID({ type: Config.SETTINGS }))
-      dbUtils.generateConfigID({ type: Configs.SETTINGS })
-    )
   } catch (e: any) {
     if (e.status !== 404) {
       throw e

packages/backend-core/src/events/identification.ts

@@ -19,7 +19,7 @@ import {
 } from "@budibase/types"
 import { processors } from "./processors"
 import * as dbUtils from "../db/utils"
-import { Configs } from "../constants"
+import { Config } from "../constants"
 import * as hashing from "../hashing"
 import * as installation from "../installation"
 import { withCache, TTL, CacheKeys } from "../cache/generic"
@@ -273,7 +273,7 @@ const getUniqueTenantId = async (tenantId: string): Promise<string> => {
     return withCache(CacheKeys.UNIQUE_TENANT_ID, TTL.ONE_DAY, async () => {
       const db = context.getGlobalDB()
       const config: SettingsConfig = await dbUtils.getScopedFullConfig(db, {
-        type: Configs.SETTINGS,
+        type: Config.SETTINGS,
       })
 
       let uniqueTenantId: string

packages/backend-core/src/events/publishers/backup.ts

@@ -1,12 +1,34 @@
-import { AppBackup, AppBackupRestoreEvent, Event } from "@budibase/types"
+import {
+  AppBackup,
+  AppBackupRestoreEvent,
+  AppBackupTriggeredEvent,
+  AppBackupTrigger,
+  AppBackupType,
+  Event,
+} from "@budibase/types"
 import { publishEvent } from "../events"
 
 export async function appBackupRestored(backup: AppBackup) {
   const properties: AppBackupRestoreEvent = {
     appId: backup.appId,
-    backupName: backup.name!,
+    restoreId: backup._id!,
     backupCreatedAt: backup.timestamp,
   }
 
   await publishEvent(Event.APP_BACKUP_RESTORED, properties)
 }
+
+export async function appBackupTriggered(
+  appId: string,
+  backupId: string,
+  type: AppBackupType,
+  trigger: AppBackupTrigger
+) {
+  const properties: AppBackupTriggeredEvent = {
+    appId: appId,
+    backupId,
+    type,
+    trigger,
+  }
+  await publishEvent(Event.APP_BACKUP_TRIGGERED, properties)
+}

packages/backend-core/src/index.ts

@@ -13,7 +13,7 @@ import featureFlags from "./featureFlags"
 import * as sessions from "./security/sessions"
 import * as deprovisioning from "./context/deprovision"
 import auth from "./auth"
-import constants from "./constants"
+import * as constants from "./constants"
 import * as dbConstants from "./db/constants"
 import * as logging from "./logging"
 import pino from "./pino"
@@ -21,9 +21,9 @@ import * as middleware from "./middleware"
 import plugins from "./plugin"
 import encryption from "./security/encryption"
 import * as queue from "./queue"
+import * as db from "./db"
 
 // mimic the outer package exports
-import * as db from "./pkg/db"
 import * as objectStore from "./pkg/objectStore"
 import * as utils from "./pkg/utils"
 import redis from "./pkg/redis"

packages/backend-core/src/middleware/authenticated.ts

@@ -1,11 +1,9 @@
-import { Cookies, Headers } from "../constants"
+import { Cookie, Header } from "../constants"
 import { getCookie, clearCookie, openJwt } from "../utils"
 import { getUser } from "../cache/user"
 import { getSession, updateSessionTTL } from "../security/sessions"
 import { buildMatcherRegex, matches } from "./matchers"
-import { SEPARATOR } from "../db/constants"
+import { SEPARATOR, queryGlobalView, ViewName } from "../db"
-import { ViewName } from "../db/utils"
-import { queryGlobalView } from "../db/views"
 import { getGlobalDB, doInTenant } from "../tenancy"
 import { decrypt } from "../security/encryption"
 const identity = require("../context/identity")
@@ -74,7 +72,7 @@ export = (
   const noAuthOptions = noAuthPatterns ? buildMatcherRegex(noAuthPatterns) : []
   return async (ctx: any, next: any) => {
     let publicEndpoint = false
-    const version = ctx.request.headers[Headers.API_VER]
+    const version = ctx.request.headers[Header.API_VER]
     // the path is not authenticated
     const found = matches(ctx, noAuthOptions)
     if (found) {
@@ -82,10 +80,10 @@ export = (
     }
     try {
       // check the actual user is authenticated first, try header or cookie
-      const headerToken = ctx.request.headers[Headers.TOKEN]
+      const headerToken = ctx.request.headers[Header.TOKEN]
-      const authCookie = getCookie(ctx, Cookies.Auth) || openJwt(headerToken)
+      const authCookie = getCookie(ctx, Cookie.Auth) || openJwt(headerToken)
-      const apiKey = ctx.request.headers[Headers.API_KEY]
+      const apiKey = ctx.request.headers[Header.API_KEY]
-      const tenantId = ctx.request.headers[Headers.TENANT_ID]
+      const tenantId = ctx.request.headers[Header.TENANT_ID]
       let authenticated = false,
         user = null,
         internal = false
@@ -116,7 +114,7 @@ export = (
           authenticated = false
           console.error("Auth Error", err?.message || err)
           // remove the cookie as the user does not exist anymore
-          clearCookie(ctx, Cookies.Auth)
+          clearCookie(ctx, Cookie.Auth)
         }
       }
       // this is an internal request, no user made it
@@ -140,7 +138,7 @@ export = (
         delete user.password
       }
       // be explicit
-      if (authenticated !== true) {
+      if (!authenticated) {
         authenticated = false
       }
       // isAuthenticated is a function, so use a variable to be able to check authed state
@@ -155,7 +153,7 @@ export = (
       console.error("Auth Error", err?.message || err)
       // invalid token, clear the cookie
       if (err && err.name === "JsonWebTokenError") {
-        clearCookie(ctx, Cookies.Auth)
+        clearCookie(ctx, Cookie.Auth)
       }
       // allow configuring for public access
       if ((opts && opts.publicAllowed) || publicEndpoint) {

packages/backend-core/src/middleware/csrf.js

@@ -1,4 +1,4 @@
-const { Headers } = require("../constants")
+const { Header } = require("../constants")
 const { buildMatcherRegex, matches } = require("./matchers")
 
 /**
@@ -68,7 +68,7 @@ module.exports = (opts = { noCsrfPatterns: [] }) => {
     }
 
     // reject if no token in request or mismatch
-    const requestToken = ctx.get(Headers.CSRF_TOKEN)
+    const requestToken = ctx.get(Header.CSRF_TOKEN)
     if (!requestToken || requestToken !== userToken) {
       ctx.throw(403, "Invalid CSRF token")
     }

packages/backend-core/src/middleware/internalApi.js

@@ -1,11 +1,11 @@
 const env = require("../environment")
-const { Headers } = require("../constants")
+const { Header } = require("../constants")
 
 /**
  * API Key only endpoint.
  */
 module.exports = async (ctx, next) => {
-  const apiKey = ctx.request.headers[Headers.API_KEY]
+  const apiKey = ctx.request.headers[Header.API_KEY]
   if (apiKey !== env.INTERNAL_API_KEY) {
     ctx.throw(403, "Unauthorized")
   }

packages/backend-core/src/middleware/passport/datasource/google.js

@@ -1,6 +1,6 @@
 const google = require("../google")
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
-const { Cookies, Configs } = require("../../../constants")
+const { Cookie, Config } = require("../../../constants")
 const { clearCookie, getCookie } = require("../../../utils")
 const { getScopedConfig, getPlatformUrl } = require("../../../db/utils")
 const { doWithDB } = require("../../../db")
@@ -11,7 +11,7 @@ async function fetchGoogleCreds() {
   // try and get the config from the tenant
   const db = getGlobalDB()
   const googleConfig = await getScopedConfig(db, {
-    type: Configs.GOOGLE,
+    type: Config.GOOGLE,
   })
   // or fall back to env variables
   return (
@@ -47,7 +47,7 @@ async function postAuth(passport, ctx, next) {
   const platformUrl = await getPlatformUrl({ tenantAware: false })
 
   let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback`
-  const authStateCookie = getCookie(ctx, Cookies.DatasourceAuth)
+  const authStateCookie = getCookie(ctx, Cookie.DatasourceAuth)
 
   return passport.authenticate(
     new GoogleStrategy(
@@ -57,7 +57,7 @@ async function postAuth(passport, ctx, next) {
         callbackURL: callbackUrl,
       },
       (accessToken, refreshToken, profile, done) => {
-        clearCookie(ctx, Cookies.DatasourceAuth)
+        clearCookie(ctx, Cookie.DatasourceAuth)
         done(null, { accessToken, refreshToken })
       }
     ),

packages/backend-core/src/middleware/passport/google.js

@@ -1,7 +1,7 @@
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
 const { ssoCallbackUrl } = require("./utils")
 const { authenticateThirdParty } = require("./third-party-common")
-const { Configs } = require("../../../constants")
+const { Config } = require("../../../constants")
 
 const buildVerifyFn = saveUserFn => {
   return (accessToken, refreshToken, profile, done) => {
@@ -60,7 +60,7 @@ exports.strategyFactory = async function (config, callbackUrl, saveUserFn) {
 }
 
 exports.getCallbackUrl = async function (db, config) {
-  return ssoCallbackUrl(db, config, Configs.GOOGLE)
+  return ssoCallbackUrl(db, config, Config.GOOGLE)
 }
 
 // expose for testing

packages/backend-core/src/middleware/passport/jwt.js

@@ -1,11 +1,11 @@
-const { Cookies } = require("../../constants")
+const { Cookie } = require("../../constants")
 const env = require("../../environment")
 const { authError } = require("./utils")
 
 exports.options = {
   secretOrKey: env.JWT_SECRET,
   jwtFromRequest: function (ctx) {
-    return ctx.cookies.get(Cookies.Auth)
+    return ctx.cookies.get(Cookie.Auth)
   },
 }
 

packages/backend-core/src/middleware/passport/oidc.js

@@ -2,7 +2,7 @@ const fetch = require("node-fetch")
 const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy
 const { authenticateThirdParty } = require("./third-party-common")
 const { ssoCallbackUrl } = require("./utils")
-const { Configs } = require("../../../constants")
+const { Config } = require("../../../constants")
 
 const buildVerifyFn = saveUserFn => {
   /**
@@ -140,7 +140,7 @@ exports.fetchStrategyConfig = async function (enrichedConfig, callbackUrl) {
 }
 
 exports.getCallbackUrl = async function (db, config) {
-  return ssoCallbackUrl(db, config, Configs.OIDC)
+  return ssoCallbackUrl(db, config, Config.OIDC)
 }
 
 // expose for testing

packages/backend-core/src/middleware/passport/utils.js

@@ -1,6 +1,6 @@
 const { isMultiTenant, getTenantId } = require("../../tenancy")
 const { getScopedConfig } = require("../../db/utils")
-const { Configs } = require("../../constants")
+const { Config } = require("../../constants")
 
 /**
  * Utility to handle authentication errors.
@@ -24,7 +24,7 @@ exports.ssoCallbackUrl = async (db, config, type) => {
     return config.callbackURL
   }
   const publicConfig = await getScopedConfig(db, {
-    type: Configs.SETTINGS,
+    type: Config.SETTINGS,
   })
 
   let callbackUrl = `/api/global/auth`

packages/backend-core/src/middleware/tenancy.ts

@@ -1,6 +1,6 @@
 import { doInTenant, getTenantIDFromCtx } from "../tenancy"
 import { buildMatcherRegex, matches } from "./matchers"
-import { Headers } from "../constants"
+import { Header } from "../constants"
 import {
   BBContext,
   EndpointMatcher,
@@ -29,7 +29,7 @@ const tenancy = (
     }
 
     const tenantId = getTenantIDFromCtx(ctx, tenantOpts)
-    ctx.set(Headers.TENANT_ID, tenantId as string)
+    ctx.set(Header.TENANT_ID, tenantId as string)
     return doInTenant(tenantId, next)
   }
 }

packages/backend-core/src/migrations/migrations.ts

@@ -42,7 +42,7 @@ export const runMigration = async (
   options: MigrationOptions = {}
 ) => {
   const migrationType = migration.type
-  let tenantId: string
+  let tenantId: string | undefined
   if (migrationType !== MigrationType.INSTALLATION) {
     tenantId = getTenantId()
   }

packages/backend-core/src/migrations/tests/index.spec.js

@@ -1,6 +1,6 @@
 require("../../../tests")
 const { runMigrations, getMigrationsDoc } = require("../index")
-const { dangerousGetDB } = require("../../db")
+const { getDB } = require("../../db")
 const {
   StaticDatabases,
 } = require("../../db/utils")
@@ -18,7 +18,7 @@ describe("migrations", () => {
   }]
 
   beforeEach(() => {
-    db = dangerousGetDB(StaticDatabases.GLOBAL.name)
+    db = getDB(StaticDatabases.GLOBAL.name)
   })
 
   afterEach(async () => {

packages/backend-core/src/pkg/cache.ts

@@ -3,9 +3,11 @@
 import * as generic from "../cache/generic"
 import * as user from "../cache/user"
 import * as app from "../cache/appMetadata"
+import * as writethrough from "../cache/writethrough"
 
 export = {
   app,
   user,
+  writethrough,
   ...generic,
 }

packages/backend-core/src/pkg/db.ts

@@ -1,7 +0,0 @@
-// Mimic the outer package export for usage in index.ts
-// The outer exports can't be used as they now reference dist directly
-export * from "../db"
-export * from "../db/utils"
-export * from "../db/views"
-export * from "../db/pouch"
-export * from "../db/constants"

packages/backend-core/src/plugin/utils.js

@@ -51,6 +51,7 @@ function validateDatasource(schema) {
   const queryValidator = joi
     .object({
       type: joi.string().allow(...Object.values(QueryType)),
+      readable: joi.boolean(),
       fields: joi.object().pattern(joi.string(), fieldValidator),
     })
     .required()

packages/backend-core/src/security/permissions.js

@@ -1,164 +0,0 @@
-const { flatten } = require("lodash")
-const { cloneDeep } = require("lodash/fp")
-
-const PermissionLevels = {
-  READ: "read",
-  WRITE: "write",
-  EXECUTE: "execute",
-  ADMIN: "admin",
-}
-
-// these are the global types, that govern the underlying default behaviour
-const PermissionTypes = {
-  APP: "app",
-  TABLE: "table",
-  USER: "user",
-  AUTOMATION: "automation",
-  WEBHOOK: "webhook",
-  BUILDER: "builder",
-  VIEW: "view",
-  QUERY: "query",
-}
-
-function Permission(type, level) {
-  this.level = level
-  this.type = type
-}
-
-function levelToNumber(perm) {
-  switch (perm) {
-    // not everything has execute privileges
-    case PermissionLevels.EXECUTE:
-      return 0
-    case PermissionLevels.READ:
-      return 1
-    case PermissionLevels.WRITE:
-      return 2
-    case PermissionLevels.ADMIN:
-      return 3
-    default:
-      return -1
-  }
-}
-
-/**
- * Given the specified permission level for the user return the levels they are allowed to carry out.
- * @param {string} userPermLevel The permission level of the user.
- * @return {string[]} All the permission levels this user is allowed to carry out.
- */
-function getAllowedLevels(userPermLevel) {
-  switch (userPermLevel) {
-    case PermissionLevels.EXECUTE:
-      return [PermissionLevels.EXECUTE]
-    case PermissionLevels.READ:
-      return [PermissionLevels.EXECUTE, PermissionLevels.READ]
-    case PermissionLevels.WRITE:
-    case PermissionLevels.ADMIN:
-      return [
-        PermissionLevels.READ,
-        PermissionLevels.WRITE,
-        PermissionLevels.EXECUTE,
-      ]
-    default:
-      return []
-  }
-}
-
-exports.BUILTIN_PERMISSION_IDS = {
-  PUBLIC: "public",
-  READ_ONLY: "read_only",
-  WRITE: "write",
-  ADMIN: "admin",
-  POWER: "power",
-}
-
-const BUILTIN_PERMISSIONS = {
-  PUBLIC: {
-    _id: exports.BUILTIN_PERMISSION_IDS.PUBLIC,
-    name: "Public",
-    permissions: [
-      new Permission(PermissionTypes.WEBHOOK, PermissionLevels.EXECUTE),
-    ],
-  },
-  READ_ONLY: {
-    _id: exports.BUILTIN_PERMISSION_IDS.READ_ONLY,
-    name: "Read only",
-    permissions: [
-      new Permission(PermissionTypes.QUERY, PermissionLevels.READ),
-      new Permission(PermissionTypes.TABLE, PermissionLevels.READ),
-      new Permission(PermissionTypes.VIEW, PermissionLevels.READ),
-    ],
-  },
-  WRITE: {
-    _id: exports.BUILTIN_PERMISSION_IDS.WRITE,
-    name: "Read/Write",
-    permissions: [
-      new Permission(PermissionTypes.QUERY, PermissionLevels.WRITE),
-      new Permission(PermissionTypes.TABLE, PermissionLevels.WRITE),
-      new Permission(PermissionTypes.VIEW, PermissionLevels.READ),
-      new Permission(PermissionTypes.AUTOMATION, PermissionLevels.EXECUTE),
-    ],
-  },
-  POWER: {
-    _id: exports.BUILTIN_PERMISSION_IDS.POWER,
-    name: "Power",
-    permissions: [
-      new Permission(PermissionTypes.TABLE, PermissionLevels.WRITE),
-      new Permission(PermissionTypes.USER, PermissionLevels.READ),
-      new Permission(PermissionTypes.AUTOMATION, PermissionLevels.EXECUTE),
-      new Permission(PermissionTypes.VIEW, PermissionLevels.READ),
-      new Permission(PermissionTypes.WEBHOOK, PermissionLevels.READ),
-    ],
-  },
-  ADMIN: {
-    _id: exports.BUILTIN_PERMISSION_IDS.ADMIN,
-    name: "Admin",
-    permissions: [
-      new Permission(PermissionTypes.TABLE, PermissionLevels.ADMIN),
-      new Permission(PermissionTypes.USER, PermissionLevels.ADMIN),
-      new Permission(PermissionTypes.AUTOMATION, PermissionLevels.ADMIN),
-      new Permission(PermissionTypes.VIEW, PermissionLevels.ADMIN),
-      new Permission(PermissionTypes.WEBHOOK, PermissionLevels.READ),
-      new Permission(PermissionTypes.QUERY, PermissionLevels.ADMIN),
-    ],
-  },
-}
-
-exports.getBuiltinPermissions = () => {
-  return cloneDeep(BUILTIN_PERMISSIONS)
-}
-
-exports.getBuiltinPermissionByID = id => {
-  const perms = Object.values(BUILTIN_PERMISSIONS)
-  return perms.find(perm => perm._id === id)
-}
-
-exports.doesHaveBasePermission = (permType, permLevel, rolesHierarchy) => {
-  const basePermissions = [
-    ...new Set(rolesHierarchy.map(role => role.permissionId)),
-  ]
-  const builtins = Object.values(BUILTIN_PERMISSIONS)
-  let permissions = flatten(
-    builtins
-      .filter(builtin => basePermissions.indexOf(builtin._id) !== -1)
-      .map(builtin => builtin.permissions)
-  )
-  for (let permission of permissions) {
-    if (
-      permission.type === permType &&
-      getAllowedLevels(permission.level).indexOf(permLevel) !== -1
-    ) {
-      return true
-    }
-  }
-  return false
-}
-
-exports.isPermissionLevelHigherThanRead = level => {
-  return levelToNumber(level) > 1
-}
-
-// utility as a lot of things need simply the builder permission
-exports.BUILDER = PermissionTypes.BUILDER
-exports.PermissionTypes = PermissionTypes
-exports.PermissionLevels = PermissionLevels

packages/backend-core/src/security/permissions.ts

@@ -0,0 +1,175 @@
+const { flatten } = require("lodash")
+const { cloneDeep } = require("lodash/fp")
+
+export type RoleHierarchy = {
+  permissionId: string
+}[]
+
+export enum PermissionLevel {
+  READ = "read",
+  WRITE = "write",
+  EXECUTE = "execute",
+  ADMIN = "admin",
+}
+
+// these are the global types, that govern the underlying default behaviour
+export enum PermissionType {
+  APP = "app",
+  TABLE = "table",
+  USER = "user",
+  AUTOMATION = "automation",
+  WEBHOOK = "webhook",
+  BUILDER = "builder",
+  VIEW = "view",
+  QUERY = "query",
+}
+
+class Permission {
+  type: PermissionType
+  level: PermissionLevel
+
+  constructor(type: PermissionType, level: PermissionLevel) {
+    this.type = type
+    this.level = level
+  }
+}
+
+function levelToNumber(perm: PermissionLevel) {
+  switch (perm) {
+    // not everything has execute privileges
+    case PermissionLevel.EXECUTE:
+      return 0
+    case PermissionLevel.READ:
+      return 1
+    case PermissionLevel.WRITE:
+      return 2
+    case PermissionLevel.ADMIN:
+      return 3
+    default:
+      return -1
+  }
+}
+
+/**
+ * Given the specified permission level for the user return the levels they are allowed to carry out.
+ * @param {string} userPermLevel The permission level of the user.
+ * @return {string[]} All the permission levels this user is allowed to carry out.
+ */
+function getAllowedLevels(userPermLevel: PermissionLevel) {
+  switch (userPermLevel) {
+    case PermissionLevel.EXECUTE:
+      return [PermissionLevel.EXECUTE]
+    case PermissionLevel.READ:
+      return [PermissionLevel.EXECUTE, PermissionLevel.READ]
+    case PermissionLevel.WRITE:
+    case PermissionLevel.ADMIN:
+      return [
+        PermissionLevel.READ,
+        PermissionLevel.WRITE,
+        PermissionLevel.EXECUTE,
+      ]
+    default:
+      return []
+  }
+}
+
+export enum BuiltinPermissionID {
+  PUBLIC = "public",
+  READ_ONLY = "read_only",
+  WRITE = "write",
+  ADMIN = "admin",
+  POWER = "power",
+}
+
+const BUILTIN_PERMISSIONS = {
+  PUBLIC: {
+    _id: BuiltinPermissionID.PUBLIC,
+    name: "Public",
+    permissions: [
+      new Permission(PermissionType.WEBHOOK, PermissionLevel.EXECUTE),
+    ],
+  },
+  READ_ONLY: {
+    _id: BuiltinPermissionID.READ_ONLY,
+    name: "Read only",
+    permissions: [
+      new Permission(PermissionType.QUERY, PermissionLevel.READ),
+      new Permission(PermissionType.TABLE, PermissionLevel.READ),
+      new Permission(PermissionType.VIEW, PermissionLevel.READ),
+    ],
+  },
+  WRITE: {
+    _id: BuiltinPermissionID.WRITE,
+    name: "Read/Write",
+    permissions: [
+      new Permission(PermissionType.QUERY, PermissionLevel.WRITE),
+      new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
+      new Permission(PermissionType.VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
+    ],
+  },
+  POWER: {
+    _id: BuiltinPermissionID.POWER,
+    name: "Power",
+    permissions: [
+      new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
+      new Permission(PermissionType.USER, PermissionLevel.READ),
+      new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
+      new Permission(PermissionType.VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
+    ],
+  },
+  ADMIN: {
+    _id: BuiltinPermissionID.ADMIN,
+    name: "Admin",
+    permissions: [
+      new Permission(PermissionType.TABLE, PermissionLevel.ADMIN),
+      new Permission(PermissionType.USER, PermissionLevel.ADMIN),
+      new Permission(PermissionType.AUTOMATION, PermissionLevel.ADMIN),
+      new Permission(PermissionType.VIEW, PermissionLevel.ADMIN),
+      new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
+      new Permission(PermissionType.QUERY, PermissionLevel.ADMIN),
+    ],
+  },
+}
+
+export function getBuiltinPermissions() {
+  return cloneDeep(BUILTIN_PERMISSIONS)
+}
+
+export function getBuiltinPermissionByID(id: string) {
+  const perms = Object.values(BUILTIN_PERMISSIONS)
+  return perms.find(perm => perm._id === id)
+}
+
+export function doesHaveBasePermission(
+  permType: PermissionType,
+  permLevel: PermissionLevel,
+  rolesHierarchy: RoleHierarchy
+) {
+  const basePermissions = [
+    ...new Set(rolesHierarchy.map(role => role.permissionId)),
+  ]
+  const builtins = Object.values(BUILTIN_PERMISSIONS)
+  let permissions = flatten(
+    builtins
+      .filter(builtin => basePermissions.indexOf(builtin._id) !== -1)
+      .map(builtin => builtin.permissions)
+  )
+  for (let permission of permissions) {
+    if (
+      permission.type === permType &&
+      getAllowedLevels(permission.level).indexOf(permLevel) !== -1
+    ) {
+      return true
+    }
+  }
+  return false
+}
+
+export function isPermissionLevelHigherThanRead(level: PermissionLevel) {
+  return levelToNumber(level) > 1
+}
+
+// utility as a lot of things need simply the builder permission
+export const BUILDER = PermissionType.BUILDER

packages/backend-core/src/security/roles.ts

@@ -1,4 +1,4 @@
-import { BUILTIN_PERMISSION_IDS, PermissionLevels } from "./permissions"
+import { BuiltinPermissionID, PermissionLevel } from "./permissions"
 import {
   generateRoleID,
   getRoleParams,
@@ -54,19 +54,19 @@ export class Role {
 
 const BUILTIN_ROLES = {
   ADMIN: new Role(BUILTIN_IDS.ADMIN, "Admin")
-    .addPermission(BUILTIN_PERMISSION_IDS.ADMIN)
+    .addPermission(BuiltinPermissionID.ADMIN)
     .addInheritance(BUILTIN_IDS.POWER),
   POWER: new Role(BUILTIN_IDS.POWER, "Power")
-    .addPermission(BUILTIN_PERMISSION_IDS.POWER)
+    .addPermission(BuiltinPermissionID.POWER)
     .addInheritance(BUILTIN_IDS.BASIC),
   BASIC: new Role(BUILTIN_IDS.BASIC, "Basic")
-    .addPermission(BUILTIN_PERMISSION_IDS.WRITE)
+    .addPermission(BuiltinPermissionID.WRITE)
     .addInheritance(BUILTIN_IDS.PUBLIC),
   PUBLIC: new Role(BUILTIN_IDS.PUBLIC, "Public").addPermission(
-    BUILTIN_PERMISSION_IDS.PUBLIC
+    BuiltinPermissionID.PUBLIC
   ),
   BUILDER: new Role(BUILTIN_IDS.BUILDER, "Builder").addPermission(
-    BUILTIN_PERMISSION_IDS.ADMIN
+    BuiltinPermissionID.ADMIN
   ),
 }
 
@@ -147,9 +147,9 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string) {
  * @param {string|null} roleId The level ID to lookup.
  * @returns {Promise<Role|object|null>} The role object, which may contain an "inherits" property.
  */
-export async function getRole(roleId?: string) {
+export async function getRole(roleId?: string): Promise<RoleDoc | undefined> {
   if (!roleId) {
-    return null
+    return undefined
   }
   let role: any = {}
   // built in roles mostly come from the in-code implementation,
@@ -193,8 +193,10 @@ async function getAllUserRoles(userRoleId?: string): Promise<RoleDoc[]> {
   ) {
     roleIds.push(currentRole.inherits)
     currentRole = await getRole(currentRole.inherits)
+    if (currentRole) {
       roles.push(currentRole)
     }
+  }
   return roles
 }
 
@@ -225,8 +227,8 @@ export function checkForRoleResourceArray(
   if (rolePerms && !Array.isArray(rolePerms[resourceId])) {
     const permLevel = rolePerms[resourceId] as any
     rolePerms[resourceId] = [permLevel]
-    if (permLevel === PermissionLevels.WRITE) {
+    if (permLevel === PermissionLevel.WRITE) {
-      rolePerms[resourceId].push(PermissionLevels.READ)
+      rolePerms[resourceId].push(PermissionLevel.READ)
     }
   }
   return rolePerms

packages/backend-core/src/tenancy/tenancy.ts

@@ -1,12 +1,15 @@
-import { doWithDB } from "../db"
-import { queryPlatformView } from "../db/views"
-import { StaticDatabases, ViewName } from "../db/constants"
-import { getGlobalDBName } from "../db/tenancy"
 import {
-  getTenantId,
+  doWithDB,
+  queryPlatformView,
+  StaticDatabases,
+  getGlobalDBName,
+  ViewName,
+} from "../db"
+import {
   DEFAULT_TENANT_ID,
-  isMultiTenant,
+  getTenantId,
   getTenantIDFromAppID,
+  isMultiTenant,
 } from "../context"
 import env from "../environment"
 import {
@@ -15,12 +18,12 @@ import {
   TenantResolutionStrategy,
   GetTenantIdOptions,
 } from "@budibase/types"
-import { Headers } from "../constants"
+import { Header } from "../constants"
 
 const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
 const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
 
-export const addTenantToUrl = (url: string) => {
+export function addTenantToUrl(url: string) {
   const tenantId = getTenantId()
 
   if (isMultiTenant()) {
@@ -31,7 +34,7 @@ export const addTenantToUrl = (url: string) => {
   return url
 }
 
-export const doesTenantExist = async (tenantId: string) => {
+export async function doesTenantExist(tenantId: string) {
   return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
     let tenants
     try {
@@ -48,12 +51,12 @@ export const doesTenantExist = async (tenantId: string) => {
   })
 }
 
-export const tryAddTenant = async (
+export async function tryAddTenant(
   tenantId: string,
   userId: string,
   email: string,
   afterCreateTenant: () => Promise<void>
-) => {
+) {
   return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
     const getDoc = async (id: string) => {
       if (!id) {
@@ -95,11 +98,11 @@ export const tryAddTenant = async (
   })
 }
 
-export const doWithGlobalDB = (tenantId: string, cb: any) => {
+export function doWithGlobalDB(tenantId: string, cb: any) {
   return doWithDB(getGlobalDBName(tenantId), cb)
 }
 
-export const lookupTenantId = async (userId: string) => {
+export async function lookupTenantId(userId: string) {
   return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: any) => {
     let tenantId = env.MULTI_TENANCY ? DEFAULT_TENANT_ID : null
     try {
@@ -115,19 +118,26 @@ export const lookupTenantId = async (userId: string) => {
 }
 
 // lookup, could be email or userId, either will return a doc
-export const getTenantUser = async (
+export async function getTenantUser(
   identifier: string
-): Promise<PlatformUser | null> => {
+): Promise<PlatformUser | undefined> {
   // use the view here and allow to find anyone regardless of casing
-  // Use lowercase to ensure email login is case insensitive
+  // Use lowercase to ensure email login is case-insensitive
-  const response = queryPlatformView(ViewName.PLATFORM_USERS_LOWERCASE, {
+  const users = await queryPlatformView<PlatformUser>(
+    ViewName.PLATFORM_USERS_LOWERCASE,
+    {
       keys: [identifier.toLowerCase()],
       include_docs: true,
-  }) as Promise<PlatformUser>
+    }
-  return response
+  )
+  if (Array.isArray(users)) {
+    return users[0]
+  } else {
+    return users
+  }
 }
 
-export const isUserInAppTenant = (appId: string, user?: any) => {
+export function isUserInAppTenant(appId: string, user?: any) {
   let userTenantId
   if (user) {
     userTenantId = user.tenantId || DEFAULT_TENANT_ID
@@ -138,7 +148,7 @@ export const isUserInAppTenant = (appId: string, user?: any) => {
   return tenantId === userTenantId
 }
 
-export const getTenantIds = async () => {
+export async function getTenantIds() {
   return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
     let tenants
     try {
@@ -193,7 +203,7 @@ export const getTenantIDFromCtx = (
 
   // header
   if (isAllowed(TenantResolutionStrategy.HEADER)) {
-    const headerTenantId = ctx.request.headers[Headers.TENANT_ID]
+    const headerTenantId = ctx.request.headers[Header.TENANT_ID]
     if (headerTenantId) {
       return headerTenantId as string
     }

packages/backend-core/src/users.ts

@@ -3,15 +3,14 @@ import {
   getUsersByAppParams,
   getProdAppID,
   generateAppUserID,
-} from "./db/utils"
+  queryGlobalView,
-import { queryGlobalView } from "./db/views"
+  UNICODE_MAX,
-import { UNICODE_MAX } from "./db/constants"
+} from "./db"
 import { BulkDocsResponse, User } from "@budibase/types"
 import { getGlobalDB } from "./context"
-import PouchDB from "pouchdb"
 
 export const bulkGetGlobalUsersById = async (userIds: string[]) => {
-  const db = getGlobalDB() as PouchDB.Database
+  const db = getGlobalDB()
   return (
     await db.allDocs({
       keys: userIds,
@@ -21,7 +20,7 @@ export const bulkGetGlobalUsersById = async (userIds: string[]) => {
 }
 
 export const bulkUpdateGlobalUsers = async (users: User[]) => {
-  const db = getGlobalDB() as PouchDB.Database
+  const db = getGlobalDB()
   return (await db.bulkDocs(users)) as BulkDocsResponse
 }
 
@@ -69,7 +68,7 @@ export const getGlobalUserByAppPage = (appId: string, user: User) => {
   if (!user) {
     return
   }
-  return generateAppUserID(getProdAppID(appId), user._id!)
+  return generateAppUserID(getProdAppID(appId)!, user._id!)
 }
 
 /**

packages/backend-core/src/utils.ts

@@ -1,8 +1,12 @@
-import { DocumentType, SEPARATOR, ViewName, getAllApps } from "./db/utils"
+import {
-const jwt = require("jsonwebtoken")
+  DocumentType,
+  SEPARATOR,
+  ViewName,
+  getAllApps,
+  queryGlobalView,
+} from "./db"
 import { options } from "./middleware/passport/jwt"
-import { queryGlobalView } from "./db/views"
+import { Header, Cookie, MAX_VALID_DATE } from "./constants"
-import { Headers, Cookies, MAX_VALID_DATE } from "./constants"
 import env from "./environment"
 import userCache from "./cache/user"
 import { getSessionsForUser, invalidateSessions } from "./security/sessions"
@@ -15,6 +19,7 @@ import {
   TenantResolutionStrategy,
 } from "@budibase/types"
 import { SetOption } from "cookies"
+const jwt = require("jsonwebtoken")
 
 const APP_PREFIX = DocumentType.APP + SEPARATOR
 const PROD_APP_PREFIX = "/app/"
@@ -29,7 +34,7 @@ async function resolveAppUrl(ctx: BBContext) {
   const appUrl = ctx.path.split("/")[2]
   let possibleAppUrl = `/${appUrl.toLowerCase()}`
 
-  let tenantId = tenancy.getTenantId()
+  let tenantId: string | null = tenancy.getTenantId()
   if (env.MULTI_TENANCY) {
     // always use the tenant id from the subdomain in multi tenancy
     // this ensures the logged-in user tenant id doesn't overwrite
@@ -50,7 +55,7 @@ async function resolveAppUrl(ctx: BBContext) {
   return app && app.appId ? app.appId : undefined
 }
 
-export const isServingApp = (ctx: BBContext) => {
+export function isServingApp(ctx: BBContext) {
   // dev app
   if (ctx.path.startsWith(`/${APP_PREFIX}`)) {
     return true
@@ -67,9 +72,9 @@ export const isServingApp = (ctx: BBContext) => {
  * @param {object} ctx The main request body to look through.
  * @returns {string|undefined} If an appId was found it will be returned.
  */
-export const getAppIdFromCtx = async (ctx: BBContext) => {
+export async function getAppIdFromCtx(ctx: BBContext) {
   // look in headers
-  const options = [ctx.headers[Headers.APP_ID]]
+  const options = [ctx.headers[Header.APP_ID]]
   let appId
   for (let option of options) {
     appId = confirmAppId(option as string)
@@ -103,7 +108,7 @@ export const getAppIdFromCtx = async (ctx: BBContext) => {
  * opens the contents of the specified encrypted JWT.
  * @return {object} the contents of the token.
  */
-export const openJwt = (token: string) => {
+export function openJwt(token: string) {
   if (!token) {
     return token
   }
@@ -115,7 +120,7 @@ export const openJwt = (token: string) => {
  * @param {object} ctx The request which is to be manipulated.
  * @param {string} name The name of the cookie to get.
  */
-export const getCookie = (ctx: BBContext, name: string) => {
+export function getCookie(ctx: BBContext, name: string) {
   const cookie = ctx.cookies.get(name)
 
   if (!cookie) {
@@ -132,12 +137,12 @@ export const getCookie = (ctx: BBContext, name: string) => {
  * @param {string|object} value The value of cookie which will be set.
  * @param {object} opts options like whether to sign.
  */
-export const setCookie = (
+export function setCookie(
   ctx: BBContext,
   value: any,
   name = "builder",
   opts = { sign: true }
-) => {
+) {
   if (value && opts && opts.sign) {
     value = jwt.sign(value, options.secretOrKey)
   }
@@ -159,7 +164,7 @@ export const setCookie = (
 /**
  * Utility function, simply calls setCookie with an empty string for value
  */
-export const clearCookie = (ctx: BBContext, name: string) => {
+export function clearCookie(ctx: BBContext, name: string) {
   setCookie(ctx, null, name)
 }
 
@@ -169,11 +174,11 @@ export const clearCookie = (ctx: BBContext, name: string) => {
  * @param {object} ctx The koa context object to be tested.
  * @return {boolean} returns true if the call is from the client lib (a built app rather than the builder).
  */
-export const isClient = (ctx: BBContext) => {
+export function isClient(ctx: BBContext) {
-  return ctx.headers[Headers.TYPE] === "client"
+  return ctx.headers[Header.TYPE] === "client"
 }
 
-const getBuilders = async () => {
+async function getBuilders() {
   const builders = await queryGlobalView(ViewName.USER_BY_BUILDERS, {
     include_docs: false,
   })
@@ -189,7 +194,7 @@ const getBuilders = async () => {
   }
 }
 
-export const getBuildersCount = async () => {
+export async function getBuildersCount() {
   const builders = await getBuilders()
   return builders.length
 }
@@ -197,14 +202,14 @@ export const getBuildersCount = async () => {
 /**
  * Logs a user out from budibase. Re-used across account portal and builder.
  */
-export const platformLogout = async (opts: PlatformLogoutOpts) => {
+export async function platformLogout(opts: PlatformLogoutOpts) {
   const ctx = opts.ctx
   const userId = opts.userId
   const keepActiveSession = opts.keepActiveSession
 
   if (!ctx) throw new Error("Koa context must be supplied to logout.")
 
-  const currentSession = getCookie(ctx, Cookies.Auth)
+  const currentSession = getCookie(ctx, Cookie.Auth)
   let sessions = await getSessionsForUser(userId)
 
   if (keepActiveSession) {
@@ -213,8 +218,8 @@ export const platformLogout = async (opts: PlatformLogoutOpts) => {
     )
   } else {
     // clear cookies
-    clearCookie(ctx, Cookies.Auth)
+    clearCookie(ctx, Cookie.Auth)
-    clearCookie(ctx, Cookies.CurrentApp)
+    clearCookie(ctx, Cookie.CurrentApp)
   }
 
   const sessionIds = sessions.map(({ sessionId }) => sessionId)
@@ -223,6 +228,6 @@ export const platformLogout = async (opts: PlatformLogoutOpts) => {
   await userCache.invalidateUser(userId)
 }
 
-export const timeout = (timeMs: number) => {
+export function timeout(timeMs: number) {
   return new Promise(resolve => setTimeout(resolve, timeMs))
 }

packages/backend-core/tsconfig.build.json

@@ -12,7 +12,7 @@
     "sourceMap": true,
     "declaration": true,
     "types": [ "node", "jest" ],
-    "outDir": "dist"
+    "outDir": "dist",
   },
   "include": [
     "**/*.js",

packages/backend-core/yarn.lock

@@ -1335,6 +1335,11 @@
   dependencies:
     "@types/node" "*"
 
+"@types/tough-cookie@^4.0.2":
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.2.tgz#6286b4c7228d58ab7866d19716f3696e03a09397"
+  integrity sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw==
+
 "@types/uuid@8.3.4":
   version "8.3.4"
   resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-8.3.4.tgz#bd86a43617df0594787d38b735f55c805becf1bc"
@@ -1411,7 +1416,7 @@ acorn@^8.4.1:
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.1.tgz#0a3f9cbecc4ec3bea6f0a80b66ae8dd2da250b73"
   integrity sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==
 
-agent-base@6:
+agent-base@6, agent-base@^6.0.2:
   version "6.0.2"
   resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77"
   integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==
@@ -1568,6 +1573,15 @@ axios@0.24.0:
   dependencies:
     follow-redirects "^1.14.4"
 
+axios@^1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35"
+  integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==
+  dependencies:
+    follow-redirects "^1.15.0"
+    form-data "^4.0.0"
+    proxy-from-env "^1.1.0"
+
 babel-jest@^28.1.3:
   version "28.1.3"
   resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-28.1.3.tgz#c1187258197c099072156a0a121c11ee1e3917d5"
@@ -1814,6 +1828,14 @@ cacheable-request@^6.0.0:
     normalize-url "^4.1.0"
     responselike "^1.0.2"
 
+call-bind@^1.0.0:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c"
+  integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==
+  dependencies:
+    function-bind "^1.1.1"
+    get-intrinsic "^1.0.2"
+
 callsites@^3.0.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73"
@@ -2472,6 +2494,11 @@ follow-redirects@^1.14.4:
   resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.1.tgz#0ca6a452306c9b276e4d3127483e29575e207ad5"
   integrity sha512-yLAMQs+k0b2m7cVxpS1VKJVvoz7SS9Td1zss3XRwXj+ZDH00RJgnuLx7E44wx02kQLrdM3aOOy+FpzS7+8OizA==
 
+follow-redirects@^1.15.0:
+  version "1.15.2"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13"
+  integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==
+
 forever-agent@~0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
@@ -2486,6 +2513,15 @@ form-data@^3.0.0:
     combined-stream "^1.0.8"
     mime-types "^2.1.12"
 
+form-data@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452"
+  integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==
+  dependencies:
+    asynckit "^0.4.0"
+    combined-stream "^1.0.8"
+    mime-types "^2.1.12"
+
 form-data@~2.3.2:
   version "2.3.3"
   resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6"
@@ -2557,6 +2593,15 @@ get-caller-file@^2.0.5:
   resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e"
   integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==
 
+get-intrinsic@^1.0.2:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.3.tgz#063c84329ad93e83893c7f4f243ef63ffa351385"
+  integrity sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==
+  dependencies:
+    function-bind "^1.1.1"
+    has "^1.0.3"
+    has-symbols "^1.0.3"
+
 get-package-type@^0.1.0:
   version "0.1.0"
   resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a"
@@ -2705,7 +2750,7 @@ has-flag@^4.0.0:
   resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b"
   integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==
 
-has-symbols@^1.0.2:
+has-symbols@^1.0.2, has-symbols@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8"
   integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==
@@ -2752,6 +2797,13 @@ http-cache-semantics@^4.0.0:
   resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz#49e91c5cbf36c9b94bcfcd71c23d5249ec74e390"
   integrity sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==
 
+http-cookie-agent@^4.0.2:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/http-cookie-agent/-/http-cookie-agent-4.0.2.tgz#dcdaae18ed1f7452d81ae4d5cd80b227d6831b69"
+  integrity sha512-noTmxdH5CuytTnLj/Qv3Z84e/YFq8yLXAw3pqIYZ25Edhb9pQErIAC+ednw40Cic6Le/h9ryph5/TqsvkOaUCw==
+  dependencies:
+    agent-base "^6.0.2"
+
 http-errors@^1.6.3, http-errors@~1.8.0:
   version "1.8.1"
   resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.8.1.tgz#7c3f28577cbc8a207388455dbd62295ed07bd68c"
@@ -4018,6 +4070,18 @@ msgpackr@^1.5.2:
   optionalDependencies:
     msgpackr-extract "^2.1.2"
 
+nano@^10.1.0:
+  version "10.1.0"
+  resolved "https://registry.yarnpkg.com/nano/-/nano-10.1.0.tgz#afdd5a7440e62f09a8e23f41fcea328d27383922"
+  integrity sha512-COeN2TpLcHuSN44QLnPmfZCoCsKAg8/aelPOVqqm/2/MvRHDEA11/Kld5C4sLzDlWlhFZ3SO2WGJGevCsvcEzQ==
+  dependencies:
+    "@types/tough-cookie" "^4.0.2"
+    axios "^1.1.3"
+    http-cookie-agent "^4.0.2"
+    node-abort-controller "^3.0.1"
+    qs "^6.11.0"
+    tough-cookie "^4.1.2"
+
 napi-macros@~2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/napi-macros/-/napi-macros-2.0.0.tgz#2b6bae421e7b96eb687aa6c77a7858640670001b"
@@ -4043,6 +4107,11 @@ negotiator@0.6.3:
   resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd"
   integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==
 
+node-abort-controller@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.0.1.tgz#f91fa50b1dee3f909afabb7e261b1e1d6b0cb74e"
+  integrity sha512-/ujIVxthRs+7q6hsdjHMaj8hRG9NuWmwrz+JdRwZ14jdFoKSkm+vDsCbF9PLpnSqjaWQJuTmVtcWHNLr+vrOFw==
+
 node-addon-api@^3.1.0:
   version "3.2.1"
   resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161"
@@ -4178,6 +4247,11 @@ object-assign@^4.1.1:
   resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
   integrity sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==
 
+object-inspect@^1.9.0:
+  version "1.12.2"
+  resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.2.tgz#c0641f26394532f28ab8d796ab954e43c009a8ea"
+  integrity sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==
+
 on-finished@^2.3.0:
   version "2.4.1"
   resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.4.1.tgz#58c8c44116e54845ad57f14ab10b03533184ac3f"
@@ -4675,6 +4749,11 @@ prompts@^2.0.1:
     kleur "^3.0.3"
     sisteransi "^1.0.5"
 
+proxy-from-env@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2"
+  integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==
+
 prr@~1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476"
@@ -4715,6 +4794,13 @@ pupa@^2.1.1:
   dependencies:
     escape-goat "^2.0.0"
 
+qs@^6.11.0:
+  version "6.11.0"
+  resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a"
+  integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==
+  dependencies:
+    side-channel "^1.0.4"
+
 qs@~6.5.2:
   version "6.5.3"
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.3.tgz#3aeeffc91967ef6e35c0e488ef46fb296ab76aad"
@@ -4725,6 +4811,11 @@ querystring@0.2.0:
   resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
   integrity sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==
 
+querystringify@^2.1.1:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.2.0.tgz#3345941b4153cb9d082d8eee4cda2016a9aef7f6"
+  integrity sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==
+
 range-parser@^1.2.0:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031"
@@ -4878,6 +4969,11 @@ require-directory@^2.1.1:
   resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
   integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==
 
+requires-port@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff"
+  integrity sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==
+
 resolve-cwd@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-3.0.0.tgz#0f0075f1bb2544766cf73ba6a6e2adfebcb13f2d"
@@ -4999,6 +5095,15 @@ shimmer@^1.2.0:
   resolved "https://registry.yarnpkg.com/shimmer/-/shimmer-1.2.1.tgz#610859f7de327b587efebf501fb43117f9aff337"
   integrity sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==
 
+side-channel@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf"
+  integrity sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==
+  dependencies:
+    call-bind "^1.0.0"
+    get-intrinsic "^1.0.2"
+    object-inspect "^1.9.0"
+
 signal-exit@^3.0.0, signal-exit@^3.0.2, signal-exit@^3.0.3, signal-exit@^3.0.7:
   version "3.0.7"
   resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9"
@@ -5329,6 +5434,16 @@ touch@^3.1.0:
     punycode "^2.1.1"
     universalify "^0.1.2"
 
+tough-cookie@^4.1.2:
+  version "4.1.2"
+  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.2.tgz#e53e84b85f24e0b65dd526f46628db6c85f6b874"
+  integrity sha512-G9fqXWoYFZgTc2z8Q5zaHy/vJMjm+WV0AkAeHxVCQiEB1b+dGvWzFW6QV07cY5jQ5gRkeid2qIkzkxUnmoQZUQ==
+  dependencies:
+    psl "^1.1.33"
+    punycode "^2.1.1"
+    universalify "^0.2.0"
+    url-parse "^1.5.3"
+
 tough-cookie@~2.5.0:
   version "2.5.0"
   resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2"
@@ -5458,6 +5573,11 @@ universalify@^0.1.2:
   resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66"
   integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==
 
+universalify@^0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.2.0.tgz#6451760566fa857534745ab1dde952d1b1761be0"
+  integrity sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==
+
 update-browserslist-db@^1.0.9:
   version "1.0.10"
   resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz#0f54b876545726f17d00cd9a2561e6dade943ff3"
@@ -5500,6 +5620,14 @@ url-parse-lax@^3.0.0:
   dependencies:
     prepend-http "^2.0.0"
 
+url-parse@^1.5.3:
+  version "1.5.10"
+  resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1"
+  integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==
+  dependencies:
+    querystringify "^2.1.1"
+    requires-port "^1.0.0"
+
 url@0.10.3:
   version "0.10.3"
   resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64"

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.1.22-alpha.6",
+  "version": "2.1.32-alpha.1",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "^1.2.1",
-    "@budibase/string-templates": "2.1.22-alpha.6",
+    "@budibase/string-templates": "2.1.32-alpha.1",
     "@spectrum-css/actionbutton": "^1.0.1",
     "@spectrum-css/actiongroup": "^1.0.1",
     "@spectrum-css/avatar": "^3.0.2",
@@ -85,5 +85,8 @@
     "svelte-flatpickr": "^3.2.3",
     "svelte-portal": "^1.0.0"
   },
+  "resolutions": {
+    "loader-utils": "1.4.1"
+  },
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.1.22-alpha.6",
+  "version": "2.1.32-alpha.1",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -71,10 +71,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.1.22-alpha.6",
+    "@budibase/bbui": "2.1.32-alpha.1",
-    "@budibase/client": "2.1.22-alpha.6",
+    "@budibase/client": "2.1.32-alpha.1",
-    "@budibase/frontend-core": "2.1.22-alpha.6",
+    "@budibase/frontend-core": "2.1.32-alpha.1",
-    "@budibase/string-templates": "2.1.22-alpha.6",
+    "@budibase/string-templates": "2.1.32-alpha.1",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -304,6 +304,8 @@
     const newError = {}
     if (!external && fieldInfo.name?.startsWith("_")) {
       newError.name = `Column name cannot start with an underscore.`
+    } else if (fieldInfo.name && !fieldInfo.name.match(/^[_a-zA-Z0-9\s]*$/g)) {
+      newError.name = `Illegal character; must be alpha-numeric.`
     } else if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) {
       newError.name = `${PROHIBITED_COLUMN_NAMES.join(
         ", "

packages/builder/src/components/backend/DataTable/modals/FilterModal.svelte

@@ -97,7 +97,7 @@
   }
 
   function fieldOptions(field) {
-    return schema[field]?.type === "options"
+    return schema[field]?.type === "options" || schema[field]?.type === "array"
       ? schema[field]?.constraints.inclusion
       : [true, false]
   }

packages/builder/src/components/design/settings/controls/ButtonActionEditor/ButtonActionDrawer.svelte

@@ -23,14 +23,18 @@
   export let bindings = []
   export let nested
 
-  $: showAvailableActions = !actions?.length
-
   let actionQuery
-  $: parsedQuery =
-    typeof actionQuery === "string" ? actionQuery.toLowerCase().trim() : ""
-
   let selectedAction = actions?.length ? actions[0] : null
 
+  $: {
+    // Ensure parameters object is never null
+    if (selectedAction && !selectedAction.parameters) {
+      selectedAction.parameters = {}
+    }
+  }
+  $: parsedQuery =
+    typeof actionQuery === "string" ? actionQuery.toLowerCase().trim() : ""
+  $: showAvailableActions = !actions?.length
   $: mappedActionTypes = actionTypes.reduce((acc, action) => {
     let parsedName = action.name.toLowerCase().trim()
     if (parsedQuery.length && parsedName.indexOf(parsedQuery) < 0) {
@@ -40,7 +44,6 @@
     acc[action.type].push(action)
     return acc
   }, {})
-
   // These are ephemeral bindings which only exist while executing actions
   $: eventContexBindings = getEventContextBindings(
     $currentAsset,
@@ -50,9 +53,8 @@
     selectedAction?.id
   )
   $: allBindings = eventContexBindings.concat(bindings)
-
-  // Assign a unique ID to each action
   $: {
+    // Ensure each action has a unique ID
     if (actions) {
       actions.forEach(action => {
         if (!action.id) {
@@ -61,13 +63,11 @@
       })
     }
   }
-
   $: selectedActionComponent =
     selectedAction &&
     actionTypes.find(t => t.name === selectedAction[EVENT_TYPE_KEY])?.component
-
-  // Select the first action if we delete an action
   $: {
+    // Select the first action if we delete an action
     if (selectedAction && !actions?.includes(selectedAction)) {
       selectedAction = actions?.[0]
     }

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/CloseScreenModal.svelte

@@ -1,14 +1,16 @@
 <script>
-  import { Label, Body } from "@budibase/bbui"
+  import { Label } from "@budibase/bbui"
   import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
 
   export let parameters
   export let bindings = []
 </script>
 
-<Body size="S">Navigate To screen, or leave blank.</Body>
-<br />
 <div class="root">
+  <span>
+    You can optionally navigate to another screen after closing the screen
+    modal.
+  </span>
   <Label small>Screen</Label>
   <DrawerBindableInput
     title="Destination URL"
@@ -20,6 +22,9 @@
 </div>
 
 <style>
+  span {
+    grid-column: 1 / 3;
+  }
   .root {
     display: grid;
     align-items: center;

packages/builder/src/pages/builder/portal/manage/auth/index.svelte

@@ -405,6 +405,7 @@
                 bind:value={providers.oidc.config.configs[0][field.name]}
                 readonly={field.readonly}
                 placeholder={field.placeholder}
+                dataCy={field.name}
               />
             </div>
             {#if field.copyButton}

packages/builder/src/stores/portal/auth.js

@@ -155,9 +155,9 @@ export function createAuthStore() {
       await actions.getSelf()
     },
     logout: async () => {
-      setUser(null)
-      setPostLogout()
       await API.logOut()
+      setPostLogout()
+      setUser(null)
       await setInitInfo({})
     },
     updateSelf: async fields => {

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "2.1.22-alpha.6",
+  "version": "2.1.32-alpha.1",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -26,9 +26,9 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "2.1.22-alpha.6",
+    "@budibase/backend-core": "2.1.32-alpha.1",
-    "@budibase/string-templates": "2.1.22-alpha.6",
+    "@budibase/string-templates": "2.1.32-alpha.1",
-    "@budibase/types": "2.1.22-alpha.6",
+    "@budibase/types": "2.1.32-alpha.1",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",
@@ -41,7 +41,7 @@
     "joi": "17.6.0",
     "lookpath": "1.1.0",
     "node-fetch": "2",
-    "pkg": "5.7.0",
+    "pkg": "5.8.0",
     "posthog-node": "1.0.7",
     "pouchdb": "7.3.0",
     "pouchdb-replication-stream": "1.2.9",

packages/cli/yarn.lock

@@ -9,11 +9,30 @@
   dependencies:
     "@babel/highlight" "^7.10.4"
 
+"@babel/generator@7.18.2":
+  version "7.18.2"
+  resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.18.2.tgz#33873d6f89b21efe2da63fe554460f3df1c5880d"
+  integrity sha512-W1lG5vUwFvfMd8HVXqdfbuG7RuaSrTCCD8cl8fP8wOivdbtbIg2Db3IWUcgvfxKbbn6ZBGYRW/Zk1MIwK49mgw==
+  dependencies:
+    "@babel/types" "^7.18.2"
+    "@jridgewell/gen-mapping" "^0.3.0"
+    jsesc "^2.5.1"
+
+"@babel/helper-string-parser@^7.19.4":
+  version "7.19.4"
+  resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.19.4.tgz#38d3acb654b4701a9b77fb0615a96f775c3a9e63"
+  integrity sha512-nHtDoQcuqFmwYNYPz3Rah5ph2p8PFeFCsZk9A/48dPc/rGocJ5J3hAAZ7pb76VWX3fZKu+uEr/FhH5jLx7umrw==
+
 "@babel/helper-validator-identifier@^7.16.7", "@babel/helper-validator-identifier@^7.18.6":
   version "7.18.6"
   resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.18.6.tgz#9c97e30d31b2b8c72a1d08984f2ca9b574d7a076"
   integrity sha512-MmetCkz9ej86nJQV+sFCxoGGrUbU3q02kgLciwkrt9QqEB7cP39oKEY0PakknEO0Gu20SskMRi+AYZ3b1TpN9g==
 
+"@babel/helper-validator-identifier@^7.19.1":
+  version "7.19.1"
+  resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz#7eea834cf32901ffdc1a7ee555e2f9c27e249ca2"
+  integrity sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w==
+
 "@babel/highlight@^7.10.4":
   version "7.18.6"
   resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.18.6.tgz#81158601e93e2563795adcbfbdf5d64be3f2ecdf"
@@ -23,10 +42,10 @@
     chalk "^2.0.0"
     js-tokens "^4.0.0"
 
-"@babel/parser@7.17.10":
+"@babel/parser@7.18.4":
-  version "7.17.10"
+  version "7.18.4"
-  resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.17.10.tgz#873b16db82a8909e0fbd7f115772f4b739f6ce78"
+  resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.18.4.tgz#6774231779dd700e0af29f6ad8d479582d7ce5ef"
-  integrity sha512-n2Q6i+fnJqzOaq2VkdXxy2TCPCWQZHiCo0XqmrCvDWcZQKRyZzYi4Z0yxlBuN0w+r2ZHmre+Q087DSrw3pbJDQ==
+  integrity sha512-FDge0dFazETFcxGw/EXzOkN8uJp0PC7Qbm+Pe9T+av2zlBpOgunFHkQPPn+eRuClU73JF+98D531UgayY89tow==
 
 "@babel/runtime@^7.15.4":
   version "7.18.9"
@@ -35,14 +54,23 @@
   dependencies:
     regenerator-runtime "^0.13.4"
 
-"@babel/types@7.17.10":
+"@babel/types@7.18.4":
-  version "7.17.10"
+  version "7.18.4"
-  resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.17.10.tgz#d35d7b4467e439fcf06d195f8100e0fea7fc82c4"
+  resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.18.4.tgz#27eae9b9fd18e9dccc3f9d6ad051336f307be354"
-  integrity sha512-9O26jG0mBYfGkUYCYZRnBwbVLd1UZOICEr2Em6InB6jVfsAv1GKgwXHmrSg+WFWDmeKTA6vyTZiN8tCSM5Oo3A==
+  integrity sha512-ThN1mBcMq5pG/Vm2IcBmPPfyPXbd8S02rS+OBIDENdufvqC7Z/jHPCv9IcP01277aKtDI8g/2XysBN4hA8niiw==
   dependencies:
     "@babel/helper-validator-identifier" "^7.16.7"
     to-fast-properties "^2.0.0"
 
+"@babel/types@^7.18.2":
+  version "7.20.2"
+  resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.20.2.tgz#67ac09266606190f496322dbaff360fdaa5e7842"
+  integrity sha512-FnnvsNWgZCr232sqtXggapvlkk/tuwR/qhGzcmxI0GXLCjmPYQPzio2FbdlWuY6y1sHFfQKk+rRbUZ9VStQMog==
+  dependencies:
+    "@babel/helper-string-parser" "^7.19.4"
+    "@babel/helper-validator-identifier" "^7.19.1"
+    to-fast-properties "^2.0.0"
+
 "@eslint/eslintrc@^0.4.3":
   version "0.4.3"
   resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.4.3.tgz#9e42981ef035beb3dd49add17acb96e8ff6f394c"
@@ -84,6 +112,38 @@
   resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45"
   integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==
 
+"@jridgewell/gen-mapping@^0.3.0":
+  version "0.3.2"
+  resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz#c1aedc61e853f2bb9f5dfe6d4442d3b565b253b9"
+  integrity sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==
+  dependencies:
+    "@jridgewell/set-array" "^1.0.1"
+    "@jridgewell/sourcemap-codec" "^1.4.10"
+    "@jridgewell/trace-mapping" "^0.3.9"
+
+"@jridgewell/resolve-uri@3.1.0":
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78"
+  integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==
+
+"@jridgewell/set-array@^1.0.1":
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72"
+  integrity sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==
+
+"@jridgewell/sourcemap-codec@1.4.14", "@jridgewell/sourcemap-codec@^1.4.10":
+  version "1.4.14"
+  resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24"
+  integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==
+
+"@jridgewell/trace-mapping@^0.3.9":
+  version "0.3.17"
+  resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.17.tgz#793041277af9073b0951a7fe0f0d8c4c98c36985"
+  integrity sha512-MCNzAp77qzKca9+W/+I0+sEpaUnZoeasnghNeVc41VZCEKaCH73Vq3BZZ/SzWIgrqE4H4ceI+p+b6C0mHf9T4g==
+  dependencies:
+    "@jridgewell/resolve-uri" "3.1.0"
+    "@jridgewell/sourcemap-codec" "1.4.14"
+
 "@nodelib/fs.scandir@2.1.5":
   version "2.1.5"
   resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
@@ -701,7 +761,7 @@ deep-extend@^0.6.0, deep-extend@~0.6.0:
   resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac"
   integrity sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==
 
-deep-is@^0.1.3, deep-is@~0.1.3:
+deep-is@^0.1.3:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831"
   integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==
@@ -833,18 +893,6 @@ escape-string-regexp@^4.0.0:
   resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34"
   integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==
 
-escodegen@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-2.0.0.tgz#5e32b12833e8aa8fa35e1bf0befa89380484c7dd"
-  integrity sha512-mmHKys/C8BFUGI+MAWNcSYoORYLMdPzjrknd2Vc+bUsjN5bXcr8EhrNB+UTqfL1y3I9c4fw2ihgtMPQLBRiQxw==
-  dependencies:
-    esprima "^4.0.1"
-    estraverse "^5.2.0"
-    esutils "^2.0.2"
-    optionator "^0.8.1"
-  optionalDependencies:
-    source-map "~0.6.1"
-
 eslint-scope@^5.1.1:
   version "5.1.1"
   resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-5.1.1.tgz#e786e59a66cb92b3f6c1fb0d508aab174848f48c"
@@ -925,7 +973,7 @@ espree@^7.3.0, espree@^7.3.1:
     acorn-jsx "^5.3.1"
     eslint-visitor-keys "^1.3.0"
 
-esprima@^4.0.0, esprima@^4.0.1:
+esprima@^4.0.0:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71"
   integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==
@@ -1019,7 +1067,7 @@ fast-json-stable-stringify@^2.0.0:
   resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633"
   integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==
 
-fast-levenshtein@^2.0.6, fast-levenshtein@~2.0.6:
+fast-levenshtein@^2.0.6:
   version "2.0.6"
   resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917"
   integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==
@@ -1585,6 +1633,11 @@ js-yaml@^3.13.1:
     argparse "^1.0.7"
     esprima "^4.0.0"
 
+jsesc@^2.5.1:
+  version "2.5.2"
+  resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4"
+  integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==
+
 json-buffer@3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.0.tgz#5b1f397afc75d677bde8bcfc0e47e1f9a3d9a898"
@@ -1723,14 +1776,6 @@ levn@^0.4.1:
     prelude-ls "^1.2.1"
     type-check "~0.4.0"
 
-levn@~0.3.0:
-  version "0.3.0"
-  resolved "https://registry.yarnpkg.com/levn/-/levn-0.3.0.tgz#3b09924edf9f083c0490fdd4c0bc4421e04764ee"
-  integrity sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==
-  dependencies:
-    prelude-ls "~1.1.2"
-    type-check "~0.3.2"
-
 lie@3.1.1:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e"
@@ -2009,18 +2054,6 @@ onetime@^5.1.0:
   dependencies:
     mimic-fn "^2.1.0"
 
-optionator@^0.8.1:
-  version "0.8.3"
-  resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.3.tgz#84fa1d036fe9d3c7e21d99884b601167ec8fb495"
-  integrity sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==
-  dependencies:
-    deep-is "~0.1.3"
-    fast-levenshtein "~2.0.6"
-    levn "~0.3.0"
-    prelude-ls "~1.1.2"
-    type-check "~0.3.2"
-    word-wrap "~1.2.3"
-
 optionator@^0.9.1:
   version "0.9.1"
   resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.1.tgz#4f236a6373dae0566a6d43e1326674f50c291499"
@@ -2136,10 +2169,10 @@ pinkie@^2.0.0:
   resolved "https://registry.yarnpkg.com/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870"
   integrity sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==
 
-pkg-fetch@3.4.1:
+pkg-fetch@3.4.2:
-  version "3.4.1"
+  version "3.4.2"
-  resolved "https://registry.yarnpkg.com/pkg-fetch/-/pkg-fetch-3.4.1.tgz#be68bb9f7fdb0f6ed995abc518ab2e35aa64d2fd"
+  resolved "https://registry.yarnpkg.com/pkg-fetch/-/pkg-fetch-3.4.2.tgz#6f68ebc54842b73f8c0808959a9df3739dcb28b7"
-  integrity sha512-fS4cdayCa1r4jHkOKGPJKnS9PEs6OWZst+s+m0+CmhmPZObMnxoRnf9T9yUWl+lzM2b5aJF7cnQIySCT7Hq8Dg==
+  integrity sha512-0+uijmzYcnhC0hStDjm/cl2VYdrmVVBpe7Q8k9YBojxmR5tG8mvR9/nooQq3QSXiQqORDVOTY3XqMEqJVIzkHA==
   dependencies:
     chalk "^4.1.2"
     fs-extra "^9.1.0"
@@ -2150,22 +2183,22 @@ pkg-fetch@3.4.1:
     tar-fs "^2.1.1"
     yargs "^16.2.0"
 
-pkg@5.7.0:
+pkg@5.8.0:
-  version "5.7.0"
+  version "5.8.0"
-  resolved "https://registry.yarnpkg.com/pkg/-/pkg-5.7.0.tgz#6422df05e8aa147764be6ef912921d0fa719ea95"
+  resolved "https://registry.yarnpkg.com/pkg/-/pkg-5.8.0.tgz#a77644aeff0b94a1656d7f76558837f7c754a4c0"
-  integrity sha512-PTiAjNq/CGAtK5qUBR6pjheqnipTFjeecgSgIKEcAOJA4GpmZeOZC8pMOoT0rfes5vHsmcFo7wbSRTAmXQurrg==
+  integrity sha512-8h9PUDYFi+LOMLbIyGRdP21g08mAtHidSpofSrf8LWhxUWGHymaRzcopEGiynB5EhQmZUKM6PQ9kCImV2TpdjQ==
   dependencies:
-    "@babel/parser" "7.17.10"
+    "@babel/generator" "7.18.2"
-    "@babel/types" "7.17.10"
+    "@babel/parser" "7.18.4"
+    "@babel/types" "7.18.4"
     chalk "^4.1.2"
-    escodegen "^2.0.0"
     fs-extra "^9.1.0"
     globby "^11.1.0"
     into-stream "^6.0.0"
     is-core-module "2.9.0"
     minimist "^1.2.6"
     multistream "^4.1.0"
-    pkg-fetch "3.4.1"
+    pkg-fetch "3.4.2"
     prebuild-install "6.1.4"
     resolve "^1.22.0"
     stream-meter "^1.0.4"
@@ -2262,11 +2295,6 @@ prelude-ls@^1.2.1:
   resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396"
   integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==
 
-prelude-ls@~1.1.2:
-  version "1.1.2"
-  resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54"
-  integrity sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==
-
 prepend-http@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-2.0.0.tgz#e92434bfa5ea8c19f41cdfd401d741a3c819d897"
@@ -2611,11 +2639,6 @@ sort-keys@^2.0.0:
   dependencies:
     is-plain-obj "^1.0.0"
 
-source-map@~0.6.1:
-  version "0.6.1"
-  resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
-  integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
-
 spark-md5@3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/spark-md5/-/spark-md5-3.0.2.tgz#7952c4a30784347abcee73268e473b9c0167e3fc"
@@ -2907,13 +2930,6 @@ type-check@^0.4.0, type-check@~0.4.0:
   dependencies:
     prelude-ls "^1.2.1"
 
-type-check@~0.3.2:
-  version "0.3.2"
-  resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.3.2.tgz#5884cab512cf1d355e3fb784f30804b2b520db72"
-  integrity sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==
-  dependencies:
-    prelude-ls "~1.1.2"
-
 type-fest@^0.20.2:
   version "0.20.2"
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4"
@@ -3036,7 +3052,7 @@ wide-align@^1.1.0:
   dependencies:
     string-width "^1.0.2 || 2 || 3 || 4"
 
-word-wrap@^1.2.3, word-wrap@~1.2.3:
+word-wrap@^1.2.3:
   version "1.2.3"
   resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c"
   integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==

packages/client/manifest.json

@@ -84,7 +84,6 @@
     "description": "This component contains things within itself",
     "icon": "Selection",
     "hasChildren": true,
-    "showSettingsBar": true,
     "size": {
       "width": 400,
       "height": 200
@@ -283,7 +282,6 @@
     "description": "A basic html button that is ready for styling",
     "icon": "Button",
     "editable": true,
-    "showSettingsBar": true,
     "size": {
       "width": 105,
       "height": 35
@@ -420,7 +418,6 @@
       "section"
     ],
     "hasChildren": true,
-    "showSettingsBar": true,
     "size": {
       "width": 400,
       "height": 100
@@ -683,7 +680,6 @@
     "illegalChildren": [
       "section"
     ],
-    "showSettingsBar": true,
     "editable": true,
     "size": {
       "width": 400,
@@ -809,7 +805,6 @@
     "illegalChildren": [
       "section"
     ],
-    "showSettingsBar": true,
     "editable": true,
     "size": {
       "width": 400,
@@ -931,7 +926,6 @@
   "tag": {
     "name": "Tag",
     "icon": "Label",
-    "showSettingsBar": true,
     "size": {
       "width": 100,
       "height": 25
@@ -1189,7 +1183,6 @@
     "name": "Link",
     "description": "A basic link component for internal and external links",
     "icon": "Link",
-    "showSettingsBar": true,
     "editable": true,
     "size": {
       "width": 200,
@@ -3927,7 +3920,6 @@
   "dynamicfilter": {
     "name": "Dynamic Filter",
     "icon": "Filter",
-    "showSettingsBar": true,
     "size": {
       "width": 100,
       "height": 35
@@ -4797,7 +4789,6 @@
       "section"
     ],
     "hasChildren": true,
-    "showSettingsBar": true,
     "size": {
       "width": 400,
       "height": 100

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "2.1.22-alpha.6",
+  "version": "2.1.32-alpha.1",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "2.1.22-alpha.6",
+    "@budibase/bbui": "2.1.32-alpha.1",
-    "@budibase/frontend-core": "2.1.22-alpha.6",
+    "@budibase/frontend-core": "2.1.32-alpha.1",
-    "@budibase/string-templates": "2.1.22-alpha.6",
+    "@budibase/string-templates": "2.1.32-alpha.1",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",
@@ -59,5 +59,8 @@
     "rollup-plugin-terser": "^7.0.2",
     "rollup-plugin-visualizer": "^5.5.4"
   },
+  "resolutions": {
+    "loader-utils": "1.4.1"
+  },
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }

packages/client/src/components/Block.svelte

@@ -9,7 +9,7 @@
   let structureLookupMap = {}
 
   const registerBlockComponent = (id, order, parentId, instance) => {
-    // Ensure child array exists
+    // Ensure child map exists
     if (!structureLookupMap[parentId]) {
       structureLookupMap[parentId] = {}
     }
@@ -18,6 +18,14 @@
     structureLookupMap[parentId][order] = instance
   }
 
+  const unregisterBlockComponent = (order, parentId) => {
+    // Ensure child map exists
+    if (!structureLookupMap[parentId]) {
+      return
+    }
+    delete structureLookupMap[parentId][order]
+  }
+
   const eject = () => {
     // Start the new structure with the root component
     let definition = structureLookupMap[$component.id][0]
@@ -73,6 +81,7 @@
     // We register block components with their raw props so that we can eject
     // blocks later on
     registerComponent: registerBlockComponent,
+    unregisterComponent: unregisterBlockComponent,
   })
 
   onMount(() => {

packages/client/src/components/BlockComponent.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { getContext } from "svelte"
+  import { getContext, onDestroy } from "svelte"
   import { generate } from "shortid"
   import { builderStore } from "../stores/builder.js"
   import Component from "components/Component.svelte"
@@ -41,6 +41,12 @@
       block.registerComponent(id, order ?? 0, $component?.id, instance)
     }
   }
+
+  onDestroy(() => {
+    if ($builderStore.inBuilder) {
+      block.unregisterComponent(order ?? 0, $component?.id)
+    }
+  })
 </script>
 
 <Component {instance} isBlock>

packages/client/src/components/app/blocks/FormBlock.svelte

@@ -1,242 +0,0 @@
-<script>
-  import { getContext } from "svelte"
-  import BlockComponent from "../../BlockComponent.svelte"
-  import Block from "../../Block.svelte"
-  import Placeholder from "../Placeholder.svelte"
-  import { makePropSafe as safe } from "@budibase/string-templates"
-
-  export let actionType
-  export let dataSource
-  export let size
-  export let disabled
-  export let fields
-  export let labelPosition
-  export let title
-  export let showSaveButton
-  export let showDeleteButton
-  export let rowId
-  export let actionUrl
-
-  const { fetchDatasourceSchema } = getContext("sdk")
-  const FieldTypeToComponentMap = {
-    string: "stringfield",
-    number: "numberfield",
-    options: "optionsfield",
-    array: "multifieldselect",
-    boolean: "booleanfield",
-    longform: "longformfield",
-    datetime: "datetimefield",
-    attachment: "attachmentfield",
-    link: "relationshipfield",
-    json: "jsonfield",
-    barcodeqr: "codescanner",
-  }
-
-  let schema
-  let formId
-  let providerId
-  let repeaterId
-
-  $: fetchSchema(dataSource)
-  $: onSave = [
-    {
-      "##eventHandlerType": "Save Row",
-      parameters: {
-        providerId: formId,
-        tableId: dataSource?.tableId,
-      },
-    },
-    {
-      "##eventHandlerType": "Close Screen Modal",
-    },
-    {
-      "##eventHandlerType": "Navigate To",
-      parameters: {
-        url: actionUrl,
-      },
-    },
-  ]
-  $: onDelete = [
-    {
-      "##eventHandlerType": "Delete Row",
-      parameters: {
-        confirm: true,
-        tableId: dataSource?.tableId,
-        rowId: `{{ ${safe(repeaterId)}.${safe("_id")} }}`,
-        revId: `{{ ${safe(repeaterId)}.${safe("_rev")} }}`,
-      },
-    },
-    {
-      "##eventHandlerType": "Close Screen Modal",
-    },
-    {
-      "##eventHandlerType": "Navigate To",
-      parameters: {
-        url: actionUrl,
-      },
-    },
-  ]
-  $: filter = [
-    {
-      field: "_id",
-      operator: "equal",
-      type: "string",
-      value: !rowId ? `{{ ${safe("url")}.${safe("id")} }}` : rowId,
-      valueType: "Binding",
-    },
-  ]
-
-  $: dataProvider = `{{ literal ${safe(providerId)} }}`
-  $: renderDeleteButton = showDeleteButton && actionType === "Update"
-  $: renderSaveButton = showSaveButton && actionType !== "View"
-  $: renderButtons = renderDeleteButton || renderSaveButton
-  $: renderHeader = renderButtons || title
-
-  const fetchSchema = async () => {
-    schema = (await fetchDatasourceSchema(dataSource)) || {}
-  }
-
-  const getComponentForField = field => {
-    if (!field || !schema?.[field]) {
-      return null
-    }
-    const type = schema[field].type
-    return FieldTypeToComponentMap[type]
-  }
-</script>
-
-<Block>
-  {#if fields?.length}
-    <BlockComponent
-      type="dataprovider"
-      context="provider"
-      bind:id={providerId}
-      props={{
-        dataSource,
-        filter,
-        limit: 1,
-        paginate: false,
-      }}
-    >
-      <BlockComponent
-        type="repeater"
-        context="repeater"
-        bind:id={repeaterId}
-        props={{
-          dataProvider,
-          noRowsMessage: "We couldn't find a row to display",
-          direction: "column",
-          hAlign: "center",
-        }}
-      >
-        <BlockComponent
-          type="form"
-          props={{
-            actionType: actionType === "Create" ? "Create" : "Update",
-            dataSource,
-            size,
-            disabled: disabled || actionType === "View",
-          }}
-          styles={{
-            normal: {
-              width: "600px",
-            },
-          }}
-          context="form"
-          bind:id={formId}
-        >
-          <BlockComponent
-            type="container"
-            props={{
-              direction: "column",
-              hAlign: "stretch",
-              vAlign: "top",
-              gap: "M",
-            }}
-          >
-            {#if renderHeader}
-              <BlockComponent
-                type="container"
-                props={{
-                  direction: "row",
-                  hAlign: "stretch",
-                  vAlign: "center",
-                  gap: "M",
-                  wrap: true,
-                }}
-                order={0}
-              >
-                <BlockComponent
-                  type="heading"
-                  props={{ text: title || "" }}
-                  order={0}
-                />
-                {#if renderButtons}
-                  <BlockComponent
-                    type="container"
-                    props={{
-                      direction: "row",
-                      hAlign: "stretch",
-                      vAlign: "center",
-                      gap: "M",
-                      wrap: true,
-                    }}
-                    order={1}
-                  >
-                    {#if renderDeleteButton}
-                      <BlockComponent
-                        type="button"
-                        props={{
-                          text: "Delete",
-                          onClick: onDelete,
-                          quiet: true,
-                          type: "secondary",
-                        }}
-                        order={0}
-                      />
-                    {/if}
-                    {#if renderSaveButton}
-                      <BlockComponent
-                        type="button"
-                        props={{
-                          text: "Save",
-                          onClick: onSave,
-                          type: "cta",
-                        }}
-                        order={1}
-                      />
-                    {/if}
-                  </BlockComponent>
-                {/if}
-              </BlockComponent>
-            {/if}
-            <BlockComponent
-              type="fieldgroup"
-              props={{ labelPosition }}
-              order={1}
-            >
-              {#each fields as field, idx}
-                {#if getComponentForField(field)}
-                  <BlockComponent
-                    type={getComponentForField(field)}
-                    props={{
-                      field,
-                      label: field,
-                      placeholder: field,
-                      disabled,
-                    }}
-                    order={idx}
-                  />
-                {/if}
-              {/each}
-            </BlockComponent>
-          </BlockComponent>
-        </BlockComponent>
-      </BlockComponent>
-    </BlockComponent>
-  {:else}
-    <Placeholder
-      text="Choose your table and add some fields to your form to get started"
-    />
-  {/if}
-</Block>

packages/client/src/components/app/blocks/form/FormBlock.svelte

@@ -0,0 +1,101 @@
+<script>
+  import { getContext } from "svelte"
+  import BlockComponent from "components/BlockComponent.svelte"
+  import Block from "components/Block.svelte"
+  import { makePropSafe as safe } from "@budibase/string-templates"
+  import InnerFormBlock from "./InnerFormBlock.svelte"
+
+  export let actionType
+  export let dataSource
+  export let size
+  export let disabled
+  export let fields
+  export let labelPosition
+  export let title
+  export let showSaveButton
+  export let showDeleteButton
+  export let rowId
+  export let actionUrl
+
+  const { fetchDatasourceSchema } = getContext("sdk")
+
+  let schema
+  let providerId
+  let repeaterId
+
+  $: fetchSchema(dataSource)
+  $: dataProvider = `{{ literal ${safe(providerId)} }}`
+  $: filter = [
+    {
+      field: "_id",
+      operator: "equal",
+      type: "string",
+      value: !rowId ? `{{ ${safe("url")}.${safe("id")} }}` : rowId,
+      valueType: "Binding",
+    },
+  ]
+  // We could simply spread $$props into the inner form and append our
+  // additions, but that would create svelte warnings about unused props and
+  // make maintenance in future more confusing as we typically always have a
+  // proper mapping of schema settings to component exports, without having to
+  // search multiple files
+  $: innerProps = {
+    dataSource,
+    actionUrl,
+    actionType,
+    size,
+    disabled,
+    fields,
+    labelPosition,
+    title,
+    showSaveButton,
+    showDeleteButton,
+    schema,
+    repeaterId,
+  }
+
+  const fetchSchema = async () => {
+    schema = (await fetchDatasourceSchema(dataSource)) || {}
+  }
+</script>
+
+<Block>
+  {#if actionType === "Create"}
+    <BlockComponent
+      type="container"
+      props={{
+        direction: "column",
+        hAlign: "left",
+        vAlign: "stretch",
+      }}
+    >
+      <InnerFormBlock {...innerProps} />
+    </BlockComponent>
+  {:else}
+    <BlockComponent
+      type="dataprovider"
+      context="provider"
+      bind:id={providerId}
+      props={{
+        dataSource,
+        filter,
+        limit: 1,
+        paginate: false,
+      }}
+    >
+      <BlockComponent
+        type="repeater"
+        context="repeater"
+        bind:id={repeaterId}
+        props={{
+          dataProvider,
+          noRowsMessage: "We couldn't find a row to display",
+          direction: "column",
+          hAlign: "center",
+        }}
+      >
+        <InnerFormBlock {...innerProps} />
+      </BlockComponent>
+    </BlockComponent>
+  {/if}
+</Block>

packages/client/src/components/app/blocks/form/InnerFormBlock.svelte

@@ -0,0 +1,194 @@
+<script>
+  import BlockComponent from "components/BlockComponent.svelte"
+  import Placeholder from "components/app/Placeholder.svelte"
+  import { makePropSafe as safe } from "@budibase/string-templates"
+
+  export let dataSource
+  export let actionUrl
+  export let actionType
+  export let size
+  export let disabled
+  export let fields
+  export let labelPosition
+  export let title
+  export let showSaveButton
+  export let showDeleteButton
+  export let schema
+  export let repeaterId
+
+  const FieldTypeToComponentMap = {
+    string: "stringfield",
+    number: "numberfield",
+    options: "optionsfield",
+    array: "multifieldselect",
+    boolean: "booleanfield",
+    longform: "longformfield",
+    datetime: "datetimefield",
+    attachment: "attachmentfield",
+    link: "relationshipfield",
+    json: "jsonfield",
+    barcodeqr: "codescanner",
+  }
+
+  let formId
+
+  $: onSave = [
+    {
+      "##eventHandlerType": "Save Row",
+      parameters: {
+        providerId: formId,
+        tableId: dataSource?.tableId,
+      },
+    },
+    {
+      "##eventHandlerType": "Close Screen Modal",
+    },
+    {
+      "##eventHandlerType": "Navigate To",
+      parameters: {
+        url: actionUrl,
+      },
+    },
+  ]
+  $: onDelete = [
+    {
+      "##eventHandlerType": "Delete Row",
+      parameters: {
+        confirm: true,
+        tableId: dataSource?.tableId,
+        rowId: `{{ ${safe(repeaterId)}.${safe("_id")} }}`,
+        revId: `{{ ${safe(repeaterId)}.${safe("_rev")} }}`,
+      },
+    },
+    {
+      "##eventHandlerType": "Close Screen Modal",
+    },
+    {
+      "##eventHandlerType": "Navigate To",
+      parameters: {
+        url: actionUrl,
+      },
+    },
+  ]
+
+  $: renderDeleteButton = showDeleteButton && actionType === "Update"
+  $: renderSaveButton = showSaveButton && actionType !== "View"
+  $: renderButtons = renderDeleteButton || renderSaveButton
+  $: renderHeader = renderButtons || title
+
+  const getComponentForField = field => {
+    if (!field || !schema?.[field]) {
+      return null
+    }
+    const type = schema[field].type
+    return FieldTypeToComponentMap[type]
+  }
+</script>
+
+{#if fields?.length}
+  <BlockComponent
+    type="form"
+    props={{
+      actionType: actionType === "Create" ? "Create" : "Update",
+      dataSource,
+      size,
+      disabled: disabled || actionType === "View",
+    }}
+    styles={{
+      normal: {
+        width: "600px",
+        "margin-left": "auto",
+        "margin-right": "auto",
+      },
+    }}
+    context="form"
+    bind:id={formId}
+  >
+    <BlockComponent
+      type="container"
+      props={{
+        direction: "column",
+        hAlign: "stretch",
+        vAlign: "top",
+        gap: "M",
+      }}
+    >
+      {#if renderHeader}
+        <BlockComponent
+          type="container"
+          props={{
+            direction: "row",
+            hAlign: "stretch",
+            vAlign: "center",
+            gap: "M",
+            wrap: true,
+          }}
+          order={0}
+        >
+          <BlockComponent
+            type="heading"
+            props={{ text: title || "" }}
+            order={0}
+          />
+          {#if renderButtons}
+            <BlockComponent
+              type="container"
+              props={{
+                direction: "row",
+                hAlign: "stretch",
+                vAlign: "center",
+                gap: "M",
+                wrap: true,
+              }}
+              order={1}
+            >
+              {#if renderDeleteButton}
+                <BlockComponent
+                  type="button"
+                  props={{
+                    text: "Delete",
+                    onClick: onDelete,
+                    quiet: true,
+                    type: "secondary",
+                  }}
+                  order={0}
+                />
+              {/if}
+              {#if renderSaveButton}
+                <BlockComponent
+                  type="button"
+                  props={{
+                    text: "Save",
+                    onClick: onSave,
+                    type: "cta",
+                  }}
+                  order={1}
+                />
+              {/if}
+            </BlockComponent>
+          {/if}
+        </BlockComponent>
+      {/if}
+      <BlockComponent type="fieldgroup" props={{ labelPosition }} order={1}>
+        {#each fields as field, idx}
+          {#if getComponentForField(field)}
+            <BlockComponent
+              type={getComponentForField(field)}
+              props={{
+                field,
+                label: field,
+                placeholder: field,
+                disabled,
+              }}
+              order={idx}
+            />
+          {/if}
+        {/each}
+      </BlockComponent>
+    </BlockComponent>
+  </BlockComponent>
+{:else}
+  <Placeholder
+    text="Choose your table and add some fields to your form to get started"
+  />
+{/if}

packages/client/src/components/app/blocks/index.js

@@ -1,6 +1,6 @@
 export { default as tableblock } from "./TableBlock.svelte"
 export { default as cardsblock } from "./CardsBlock.svelte"
 export { default as repeaterblock } from "./RepeaterBlock.svelte"
-export { default as formblock } from "./FormBlock.svelte"
+export { default as formblock } from "./form/FormBlock.svelte"
 export { default as chartblock } from "./ChartBlock.svelte"
 export { default as rowexplorer } from "./RowExplorer.svelte"

packages/client/src/components/app/forms/DateTimeField.svelte

@@ -6,7 +6,7 @@
   export let label
   export let placeholder
   export let disabled = false
-  export let enableTime = false
+  export let enableTime = true
   export let timeOnly = false
   export let time24hr = false
   export let ignoreTimezones = false

packages/client/src/components/preview/SettingsBar.svelte

@@ -16,13 +16,16 @@
   let measured = false
 
   $: definition = $componentStore.selectedComponentDefinition
-  $: showBar = definition?.showSettingsBar && !$dndIsDragging
+  $: showBar =
+    definition?.showSettingsBar !== false && !$dndIsDragging && definition
   $: {
     if (!showBar) {
       measured = false
     }
   }
   $: settings = getBarSettings(definition)
+  $: isScreen =
+    $builderStore.selectedComponentId === $builderStore.screen?.props?._id
 
   const getBarSettings = definition => {
     let allSettings = []
@@ -152,10 +155,11 @@
       {:else if setting.type === "color"}
         <SettingsColorPicker prop={setting.key} />
       {/if}
-      {#if setting.barSeparator !== false}
+      {#if setting.barSeparator !== false && (settings.length != idx + 1 || !isScreen)}
         <div class="divider" />
       {/if}
     {/each}
+    {#if !isScreen}
       <SettingsButton
         icon="Duplicate"
         on:click={() => {
@@ -168,10 +172,13 @@
       <SettingsButton
         icon="Delete"
         on:click={() => {
-        builderStore.actions.deleteComponent($builderStore.selectedComponentId)
+          builderStore.actions.deleteComponent(
+            $builderStore.selectedComponentId
+          )
         }}
         title="Delete component"
       />
+    {/if}
   </div>
 {/if}
 

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "2.1.22-alpha.6",
+  "version": "2.1.32-alpha.1",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "2.1.22-alpha.6",
+    "@budibase/bbui": "2.1.32-alpha.1",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/sdk",
-  "version": "2.1.22-alpha.6",
+  "version": "2.1.32-alpha.1",
   "description": "Budibase Public API SDK",
   "author": "Budibase",
   "license": "MPL-2.0",

packages/server/__mocks__/node-fetch.ts

@@ -2,7 +2,7 @@ module FetchMock {
   const fetch = jest.requireActual("node-fetch")
   let failCount = 0
 
-  module.exports = async (url: any, opts: any) => {
+  const func = async (url: any, opts: any) => {
     function json(body: any, status = 200) {
       return {
         status,
@@ -106,4 +106,8 @@ module FetchMock {
     }
     return fetch(url, opts)
   }
+
+  func.Headers = fetch.Headers
+
+  module.exports = func
 }

packages/server/jest.config.ts

@@ -19,6 +19,7 @@ if (!process.env.CI) {
     "@budibase/backend-core/(.*)": "<rootDir>/../backend-core/$1",
     "@budibase/backend-core": "<rootDir>/../backend-core/src",
     "@budibase/types": "<rootDir>/../types/src",
+    "^axios.*$": "<rootDir>/node_modules/axios/lib/axios.js",
   }
   // add pro sources if they exist
   if (fs.existsSync("../../../budibase-pro")) {

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.1.22-alpha.6",
+  "version": "2.1.32-alpha.1",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -43,11 +43,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.1.22-alpha.6",
+    "@budibase/backend-core": "2.1.32-alpha.1",
-    "@budibase/client": "2.1.22-alpha.6",
+    "@budibase/client": "2.1.32-alpha.1",
-    "@budibase/pro": "2.1.22-alpha.6",
+    "@budibase/pro": "2.1.31",
-    "@budibase/string-templates": "2.1.22-alpha.6",
+    "@budibase/string-templates": "2.1.32-alpha.1",
-    "@budibase/types": "2.1.22-alpha.6",
+    "@budibase/types": "2.1.32-alpha.1",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",

packages/server/src/api/controllers/application.ts

@@ -14,17 +14,16 @@ import {
   DocumentType,
   AppStatus,
 } from "../../db/utils"
-const {
+import {
-  BUILTIN_ROLE_IDS,
+  db as dbCore,
-  AccessController,
+  roles,
-} = require("@budibase/backend-core/roles")
+  cache,
-const { CacheKeys, bustCache } = require("@budibase/backend-core/cache")
+  tenancy,
-const {
+  context,
-  getAllApps,
+  errors,
-  isDevAppID,
+  events,
-  getProdAppID,
+  migrations,
-  Replication,
+} from "@budibase/backend-core"
-} = require("@budibase/backend-core/db")
 import { USERS_TABLE_SCHEMA } from "../../constants"
 import { removeAppFromUserRoles } from "../../utilities/workerRequests"
 import { clientLibraryPath, stringToReadStream } from "../../utilities"
@@ -34,15 +33,11 @@ import {
   backupClientLibrary,
   revertClientLibrary,
 } from "../../utilities/fileSystem/clientLibrary"
-const { getTenantId, isMultiTenant } = require("@budibase/backend-core/tenancy")
 import { syncGlobalUsers } from "./user"
-const { app: appCache } = require("@budibase/backend-core/cache")
 import { cleanupAutomations } from "../../automations/utils"
-import { context } from "@budibase/backend-core"
 import { checkAppMetadata } from "../../automations/logging"
 import { getUniqueRows } from "../../utilities/usageQuota/rows"
 import { quotas, groups } from "@budibase/pro"
-import { errors, events, migrations } from "@budibase/backend-core"
 import { App, Layout, Screen, MigrationType } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import { enrichPluginURLs } from "../../utilities/plugins"
@@ -75,7 +70,7 @@ async function getScreens() {
 
 function getUserRoleId(ctx: any) {
   return !ctx.user.role || !ctx.user.role._id
-    ? BUILTIN_ROLE_IDS.PUBLIC
+    ? roles.BUILTIN_ROLE_IDS.PUBLIC
     : ctx.user.role._id
 }
 
@@ -123,7 +118,7 @@ const checkAppName = (
 }
 
 async function createInstance(template: any) {
-  const tenantId = isMultiTenant() ? getTenantId() : null
+  const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null
   const baseAppId = generateAppID(tenantId)
   const appId = generateDevAppID(baseAppId)
   await context.updateAppId(appId)
@@ -162,7 +157,7 @@ async function createInstance(template: any) {
 export const fetch = async (ctx: any) => {
   const dev = ctx.query && ctx.query.status === AppStatus.DEV
   const all = ctx.query && ctx.query.status === AppStatus.ALL
-  const apps = await getAllApps({ dev, all })
+  const apps = (await dbCore.getAllApps({ dev, all })) as App[]
 
   const appIds = apps
     .filter((app: any) => app.status === "development")
@@ -187,7 +182,7 @@ export const fetch = async (ctx: any) => {
 export const fetchAppDefinition = async (ctx: any) => {
   const layouts = await getLayouts()
   const userRoleId = getUserRoleId(ctx)
-  const accessController = new AccessController()
+  const accessController = new roles.AccessController()
   const screens = await accessController.checkScreensAccess(
     await getScreens(),
     userRoleId
@@ -211,7 +206,7 @@ export const fetchAppPackage = async (ctx: any) => {
   // Only filter screens if the user is not a builder
   if (!(ctx.user.builder && ctx.user.builder.global)) {
     const userRoleId = getUserRoleId(ctx)
-    const accessController = new AccessController()
+    const accessController = new roles.AccessController()
     screens = await accessController.checkScreensAccess(screens, userRoleId)
   }
 
@@ -224,7 +219,7 @@ export const fetchAppPackage = async (ctx: any) => {
 }
 
 const performAppCreate = async (ctx: any) => {
-  const apps = await getAllApps({ dev: true })
+  const apps = await dbCore.getAllApps({ dev: true })
   const name = ctx.request.body.name
   checkAppName(ctx, apps, name)
   const url = getAppUrl(ctx)
@@ -254,7 +249,7 @@ const performAppCreate = async (ctx: any) => {
     url: url,
     template: templateKey,
     instance,
-    tenantId: getTenantId(),
+    tenantId: tenancy.getTenantId(),
     updatedAt: new Date().toISOString(),
     createdAt: new Date().toISOString(),
     status: AppStatus.DEV,
@@ -313,7 +308,7 @@ const performAppCreate = async (ctx: any) => {
     await createApp(appId)
   }
 
-  await appCache.invalidateAppMetadata(appId, newApplication)
+  await cache.app.invalidateAppMetadata(appId, newApplication)
   return newApplication
 }
 
@@ -343,7 +338,7 @@ const creationEvents = async (request: any, app: App) => {
 }
 
 const appPostCreate = async (ctx: any, app: App) => {
-  const tenantId = getTenantId()
+  const tenantId = tenancy.getTenantId()
   await migrations.backPopulateMigrations({
     type: MigrationType.APP,
     tenantId,
@@ -356,7 +351,9 @@ const appPostCreate = async (ctx: any, app: App) => {
     const rowCount = rows ? rows.length : 0
     if (rowCount) {
       try {
-        await quotas.addRows(rowCount)
+        await context.doInAppContext(app.appId, () => {
+          return quotas.addRows(rowCount)
+        })
       } catch (err: any) {
         if (err.code && err.code === errors.codes.USAGE_LIMIT_EXCEEDED) {
           // this import resulted in row usage exceeding the quota
@@ -374,7 +371,7 @@ const appPostCreate = async (ctx: any, app: App) => {
 export const create = async (ctx: any) => {
   const newApplication = await quotas.addApp(() => performAppCreate(ctx))
   await appPostCreate(ctx, newApplication)
-  await bustCache(CacheKeys.CHECKLIST)
+  await cache.bustCache(cache.CacheKeys.CHECKLIST)
   ctx.body = newApplication
   ctx.status = 200
 }
@@ -382,7 +379,7 @@ export const create = async (ctx: any) => {
 // This endpoint currently operates as a PATCH rather than a PUT
 // Thus name and url fields are handled only if present
 export const update = async (ctx: any) => {
-  const apps = await getAllApps({ dev: true })
+  const apps = await dbCore.getAllApps({ dev: true })
   // validation
   const name = ctx.request.body.name
   if (name) {
@@ -455,7 +452,7 @@ const destroyApp = async (ctx: any) => {
   let isUnpublish = ctx.query && ctx.query.unpublish
 
   if (isUnpublish) {
-    appId = getProdAppID(appId)
+    appId = dbCore.getProdAppID(appId)
   }
 
   const db = isUnpublish ? context.getProdAppDB() : context.getAppDB()
@@ -481,7 +478,7 @@ const destroyApp = async (ctx: any) => {
   else {
     await removeAppFromUserRoles(ctx, appId)
   }
-  await appCache.invalidateAppMetadata(appId)
+  await cache.app.invalidateAppMetadata(appId)
   return result
 }
 
@@ -517,19 +514,17 @@ export const sync = async (ctx: any, next: any) => {
   }
 
   const appId = ctx.params.appId
-  if (!isDevAppID(appId)) {
+  if (!dbCore.isDevAppID(appId)) {
     ctx.throw(400, "This action cannot be performed for production apps")
   }
 
   // replicate prod to dev
-  const prodAppId = getProdAppID(appId)
+  const prodAppId = dbCore.getProdAppID(appId)
 
-  try {
   // specific case, want to make sure setup is skipped
   const prodDb = context.getProdAppDB({ skip_setup: true })
-    const info = await prodDb.info()
+  const exists = await prodDb.exists()
-    if (info.error) throw info.error
+  if (!exists) {
-  } catch (err) {
     // the database doesn't exist. Don't replicate
     ctx.status = 200
     ctx.body = {
@@ -538,7 +533,7 @@ export const sync = async (ctx: any, next: any) => {
     return next()
   }
 
-  const replication = new Replication({
+  const replication = new dbCore.Replication({
     source: prodAppId,
     target: appId,
   })
@@ -579,7 +574,7 @@ export const updateAppPackage = async (appPackage: any, appId: any) => {
 
     await db.put(newAppPackage)
     // remove any cached metadata, so that it will be updated
-    await appCache.invalidateAppMetadata(appId)
+    await cache.app.invalidateAppMetadata(appId)
     return newAppPackage
   })
 }

packages/server/src/api/controllers/auth.ts

@@ -3,6 +3,7 @@ import { InternalTables } from "../../db/utils"
 import { getFullUser } from "../../utilities/users"
 import { roles, context } from "@budibase/backend-core"
 import { groups } from "@budibase/pro"
+import { ContextUser, User } from "@budibase/types"
 
 const PUBLIC_ROLE = roles.BUILTIN_ROLE_IDS.PUBLIC
 
@@ -24,7 +25,7 @@ export async function fetchSelf(ctx: any) {
   }
 
   const appId = context.getAppId()
-  const user = await getFullUser(ctx, userId)
+  const user: ContextUser = await getFullUser(ctx, userId)
   // this shouldn't be returned by the app self
   delete user.roles
   // forward the csrf token from the session
@@ -34,7 +35,7 @@ export async function fetchSelf(ctx: any) {
     const db = context.getAppDB()
     // check for group permissions
     if (!user.roleId || user.roleId === PUBLIC_ROLE) {
-      const groupRoleId = await groups.getGroupRoleId(user, appId)
+      const groupRoleId = await groups.getGroupRoleId(user as User, appId)
       user.roleId = groupRoleId || user.roleId
     }
     // remove the full roles structure

packages/server/src/api/controllers/deploy/index.ts

@@ -1,23 +1,11 @@
 import Deployment from "./Deployment"
-import {
+import { context, db as dbCore, events, cache } from "@budibase/backend-core"
-  getDevelopmentAppID,
-  getProdAppID,
-  Replication,
-} from "@budibase/backend-core/db"
 import { DocumentType, getAutomationParams } from "../../../db/utils"
 import {
   clearMetadata,
   disableAllCrons,
   enableCronTrigger,
 } from "../../../automations/utils"
-import { app as appCache } from "@budibase/backend-core/cache"
-import {
-  getAppDB,
-  getAppId,
-  getDevAppDB,
-  getProdAppDB,
-} from "@budibase/backend-core/context"
-import { events } from "@budibase/backend-core"
 import { backups } from "@budibase/pro"
 import { AppBackupTrigger } from "@budibase/types"
 
@@ -49,7 +37,7 @@ async function checkAllDeployments(deployments: any) {
 
 async function storeDeploymentHistory(deployment: any) {
   const deploymentJSON = deployment.getJSON()
-  const db = getAppDB()
+  const db = context.getAppDB()
 
   let deploymentDoc
   try {
@@ -77,7 +65,7 @@ async function storeDeploymentHistory(deployment: any) {
 }
 
 async function initDeployedApp(prodAppId: any) {
-  const db = getProdAppDB()
+  const db = context.getProdAppDB()
   console.log("Reading automation docs")
   const automations = (
     await db.allDocs(
@@ -103,9 +91,9 @@ async function initDeployedApp(prodAppId: any) {
 async function deployApp(deployment: any, userId: string) {
   let replication
   try {
-    const appId = getAppId()
+    const appId = context.getAppId()!
-    const devAppId = getDevelopmentAppID(appId)
+    const devAppId = dbCore.getDevelopmentAppID(appId)
-    const productionAppId = getProdAppID(appId)
+    const productionAppId = dbCore.getProdAppID(appId)
 
     // don't try this if feature isn't allowed, will error
     if (await backups.isEnabled()) {
@@ -122,8 +110,8 @@ async function deployApp(deployment: any, userId: string) {
       source: devAppId,
       target: productionAppId,
     }
-    replication = new Replication(config)
+    replication = new dbCore.Replication(config)
-    const devDb = getDevAppDB()
+    const devDb = context.getDevAppDB()
     console.log("Compacting development DB")
     await devDb.compact()
     console.log("Replication object created")
@@ -131,7 +119,7 @@ async function deployApp(deployment: any, userId: string) {
     console.log("replication complete.. replacing app meta doc")
     // app metadata is excluded as it is likely to be in conflict
     // replicate the app metadata document manually
-    const db = getProdAppDB()
+    const db = context.getProdAppDB()
     const appDoc = await devDb.get(DocumentType.APP_METADATA)
     try {
       const prodAppDoc = await db.get(DocumentType.APP_METADATA)
@@ -147,7 +135,7 @@ async function deployApp(deployment: any, userId: string) {
     // remove automation errors if they exist
     delete appDoc.automationErrors
     await db.put(appDoc)
-    await appCache.invalidateAppMetadata(productionAppId)
+    await cache.app.invalidateAppMetadata(productionAppId)
     console.log("New app doc written successfully.")
     await initDeployedApp(productionAppId)
     console.log("Deployed app initialised, setting deployment to successful")
@@ -170,7 +158,7 @@ async function deployApp(deployment: any, userId: string) {
 
 export async function fetchDeployments(ctx: any) {
   try {
-    const db = getAppDB()
+    const db = context.getAppDB()
     const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS)
     const { updated, deployments } = await checkAllDeployments(deploymentDoc)
     if (updated) {
@@ -184,7 +172,7 @@ export async function fetchDeployments(ctx: any) {
 
 export async function deploymentProgress(ctx: any) {
   try {
-    const db = getAppDB()
+    const db = context.getAppDB()
     const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS)
     ctx.body = deploymentDoc[ctx.params.deploymentId]
   } catch (err) {
@@ -197,7 +185,7 @@ export async function deploymentProgress(ctx: any) {
 
 const isFirstDeploy = async () => {
   try {
-    const db = getProdAppDB()
+    const db = context.getProdAppDB()
     await db.get(DocumentType.APP_METADATA)
   } catch (e: any) {
     if (e.status === 404) {

packages/server/src/api/controllers/dev.ts

@@ -1,29 +1,23 @@
-const fetch = require("node-fetch")
+import fetch from "node-fetch"
-const env = require("../../environment")
+import env from "../../environment"
-const { checkSlashesInUrl } = require("../../utilities")
+import { checkSlashesInUrl } from "../../utilities"
-const { request } = require("../../utilities/workerRequests")
+import { request } from "../../utilities/workerRequests"
-const { clearLock } = require("../../utilities/redis")
+import { clearLock as redisClearLock } from "../../utilities/redis"
-const { Replication, getProdAppID } = require("@budibase/backend-core/db")
+import { DocumentType } from "../../db/utils"
-const { DocumentType } = require("../../db/utils")
+import { context } from "@budibase/backend-core"
-const { app: appCache } = require("@budibase/backend-core/cache")
+import { events, db as dbCore, cache } from "@budibase/backend-core"
-const { getProdAppDB, getAppDB } = require("@budibase/backend-core/context")
-const { events } = require("@budibase/backend-core")
 
-async function redirect(ctx, method, path = "global") {
+async function redirect(ctx: any, method: string, path: string = "global") {
   const { devPath } = ctx.params
   const queryString = ctx.originalUrl.split("?")[1] || ""
   const response = await fetch(
     checkSlashesInUrl(
       `${env.WORKER_URL}/api/${path}/${devPath}?${queryString}`
     ),
-    request(
+    request(ctx, {
-      ctx,
-      {
       method,
       body: ctx.request.body,
-      },
+    })
-      true
-    )
   )
   if (response.status !== 200) {
     const err = await response.text()
@@ -46,28 +40,28 @@ async function redirect(ctx, method, path = "global") {
   ctx.cookies
 }
 
-exports.buildRedirectGet = path => {
+export function buildRedirectGet(path: string) {
-  return async ctx => {
+  return async (ctx: any) => {
     await redirect(ctx, "GET", path)
   }
 }
 
-exports.buildRedirectPost = path => {
+export function buildRedirectPost(path: string) {
-  return async ctx => {
+  return async (ctx: any) => {
     await redirect(ctx, "POST", path)
   }
 }
 
-exports.buildRedirectDelete = path => {
+export function buildRedirectDelete(path: string) {
-  return async ctx => {
+  return async (ctx: any) => {
     await redirect(ctx, "DELETE", path)
   }
 }
 
-exports.clearLock = async ctx => {
+export async function clearLock(ctx: any) {
   const { appId } = ctx.params
   try {
-    await clearLock(appId, ctx.user)
+    await redisClearLock(appId, ctx.user)
   } catch (err) {
     ctx.throw(400, `Unable to remove lock. ${err}`)
   }
@@ -76,16 +70,16 @@ exports.clearLock = async ctx => {
   }
 }
 
-exports.revert = async ctx => {
+export async function revert(ctx: any) {
   const { appId } = ctx.params
-  const productionAppId = getProdAppID(appId)
+  const productionAppId = dbCore.getProdAppID(appId)
 
   // App must have been deployed first
   try {
-    const db = getProdAppDB({ skip_setup: true })
+    const db = context.getProdAppDB({ skip_setup: true })
-    const info = await db.info()
+    const exists = await db.exists()
-    if (info.error) {
+    if (!exists) {
-      throw info.error
+      throw new Error("App must be deployed to be reverted.")
     }
     const deploymentDoc = await db.get(DocumentType.DEPLOYMENTS)
     if (
@@ -98,7 +92,7 @@ exports.revert = async ctx => {
     return ctx.throw(400, "App has not yet been deployed")
   }
 
-  const replication = new Replication({
+  const replication = new dbCore.Replication({
     source: productionAppId,
     target: appId,
   })
@@ -109,12 +103,12 @@ exports.revert = async ctx => {
     }
 
     // update appID in reverted app to be dev version again
-    const db = getAppDB()
+    const db = context.getAppDB()
     const appDoc = await db.get(DocumentType.APP_METADATA)
     appDoc.appId = appId
     appDoc.instance._id = appId
     await db.put(appDoc)
-    await appCache.invalidateAppMetadata(appId)
+    await cache.app.invalidateAppMetadata(appId)
     ctx.body = {
       message: "Reverted changes successfully.",
     }
@@ -126,7 +120,7 @@ exports.revert = async ctx => {
   }
 }
 
-exports.getBudibaseVersion = async ctx => {
+export async function getBudibaseVersion(ctx: any) {
   const version = require("../../../package.json").version
   ctx.body = {
     version,

packages/server/src/api/controllers/public/applications.ts

@@ -1,5 +1,4 @@
-const { getAllApps } = require("@budibase/backend-core/db")
+import { db as dbCore, context } from "@budibase/backend-core"
-const { doInAppContext } = require("@budibase/backend-core/context")
 import { search as stringSearch, addRev } from "./utils"
 import * as controller from "../application"
 import { Application } from "../../../definitions/common"
@@ -15,15 +14,22 @@ function fixAppID(app: Application, params: any) {
 }
 
 async function setResponseApp(ctx: any) {
-  if (ctx.body && ctx.body.appId && (!ctx.params || !ctx.params.appId)) {
+  const appId = ctx.body?.appId
-    ctx.params = { appId: ctx.body.appId }
+  if (appId && (!ctx.params || !ctx.params.appId)) {
+    ctx.params = { appId }
+  }
+  if (appId) {
+    await context.doInContext(appId, () => {
+      return controller.fetchAppPackage(ctx)
+    })
+  } else {
+    return controller.fetchAppPackage(ctx)
   }
-  await controller.fetchAppPackage(ctx)
 }
 
 export async function search(ctx: any, next: any) {
   const { name } = ctx.request.body
-  const apps = await getAllApps({ all: true })
+  const apps = await dbCore.getAllApps({ all: true })
   ctx.body = stringSearch(apps, name)
   await next()
 }
@@ -41,7 +47,7 @@ export async function create(ctx: any, next: any) {
 }
 
 export async function read(ctx: any, next: any) {
-  await doInAppContext(ctx.params.appId, async () => {
+  await context.doInAppContext(ctx.params.appId, async () => {
     await setResponseApp(ctx)
     await next()
   })
@@ -49,7 +55,7 @@ export async function read(ctx: any, next: any) {
 
 export async function update(ctx: any, next: any) {
   ctx.request.body = await addRev(fixAppID(ctx.request.body, ctx.params))
-  await doInAppContext(ctx.params.appId, async () => {
+  await context.doInAppContext(ctx.params.appId, async () => {
     await controller.update(ctx)
     await setResponseApp(ctx)
     await next()
@@ -57,7 +63,7 @@ export async function update(ctx: any, next: any) {
 }
 
 export async function destroy(ctx: any, next: any) {
-  await doInAppContext(ctx.params.appId, async () => {
+  await context.doInAppContext(ctx.params.appId, async () => {
     // get the app before deleting it
     await setResponseApp(ctx)
     const body = ctx.body

packages/server/src/api/controllers/query/index.ts

@@ -5,11 +5,8 @@ import { save as saveDatasource } from "../datasource"
 import { RestImporter } from "./import"
 import { invalidateDynamicVariables } from "../../../threads/utils"
 import { QUERY_THREAD_TIMEOUT } from "../../../environment"
-import { getAppDB } from "@budibase/backend-core/context"
 import { quotas } from "@budibase/pro"
-import { events } from "@budibase/backend-core"
+import { events, context, utils, constants } from "@budibase/backend-core"
-import { getCookie } from "@budibase/backend-core/utils"
-import { Cookies, Configs } from "@budibase/backend-core/constants"
 
 const Runner = new Thread(ThreadType.QUERY, {
   timeoutMs: QUERY_THREAD_TIMEOUT || 10000,
@@ -28,7 +25,7 @@ function enrichQueries(input: any) {
 }
 
 export async function fetch(ctx: any) {
-  const db = getAppDB()
+  const db = context.getAppDB()
 
   const body = await db.allDocs(
     getQueryParams(null, {
@@ -81,7 +78,7 @@ const _import = async (ctx: any) => {
 export { _import as import }
 
 export async function save(ctx: any) {
-  const db = getAppDB()
+  const db = context.getAppDB()
   const query = ctx.request.body
 
   const datasource = await db.get(query.datasourceId)
@@ -103,7 +100,7 @@ export async function save(ctx: any) {
 }
 
 export async function find(ctx: any) {
-  const db = getAppDB()
+  const db = context.getAppDB()
   const query = enrichQueries(await db.get(ctx.params.queryId))
   // remove properties that could be dangerous in real app
   if (isProdAppID(ctx.appId)) {
@@ -115,13 +112,13 @@ export async function find(ctx: any) {
 
 //Required to discern between OIDC OAuth config entries
 function getOAuthConfigCookieId(ctx: any) {
-  if (ctx.user.providerType === Configs.OIDC) {
+  if (ctx.user.providerType === constants.Config.OIDC) {
-    return getCookie(ctx, Cookies.OIDC_CONFIG)
+    return utils.getCookie(ctx, constants.Cookie.OIDC_CONFIG)
   }
 }
 
 function getAuthConfig(ctx: any) {
-  const authCookie = getCookie(ctx, Cookies.Auth)
+  const authCookie = utils.getCookie(ctx, constants.Cookie.Auth)
   let authConfigCtx: any = {}
   authConfigCtx["configId"] = getOAuthConfigCookieId(ctx)
   authConfigCtx["sessionId"] = authCookie ? authCookie.sessionId : null
@@ -129,7 +126,7 @@ function getAuthConfig(ctx: any) {
 }
 
 export async function preview(ctx: any) {
-  const db = getAppDB()
+  const db = context.getAppDB()
 
   const datasource = await db.get(ctx.request.body.datasourceId)
   const query = ctx.request.body
@@ -201,7 +198,7 @@ async function execute(
   ctx: any,
   opts: any = { rowsOnly: false, isAutomation: false }
 ) {
-  const db = getAppDB()
+  const db = context.getAppDB()
 
   const query = await db.get(ctx.params.queryId)
   const datasource = await db.get(query.datasourceId)
@@ -267,7 +264,7 @@ export async function executeV2(
 }
 
 const removeDynamicVariables = async (queryId: any) => {
-  const db = getAppDB()
+  const db = context.getAppDB()
   const query = await db.get(queryId)
   const datasource = await db.get(query.datasourceId)
   const dynamicVariables = datasource.config.dynamicVariables
@@ -288,7 +285,7 @@ const removeDynamicVariables = async (queryId: any) => {
 }
 
 export async function destroy(ctx: any) {
-  const db = getAppDB()
+  const db = context.getAppDB()
   const queryId = ctx.params.queryId
   await removeDynamicVariables(queryId)
   const query = await db.get(queryId)

packages/server/src/api/controllers/row/internal.js

@@ -6,7 +6,7 @@ const {
   DocumentType,
   InternalTables,
 } = require("../../../db/utils")
-const { dangerousGetDB } = require("@budibase/backend-core/db")
+const { getDB } = require("@budibase/backend-core/db")
 const userController = require("../user")
 const {
   inputProcessing,
@@ -251,7 +251,7 @@ exports.fetch = async ctx => {
 }
 
 exports.find = async ctx => {
-  const db = dangerousGetDB(ctx.appId)
+  const db = getDB(ctx.appId)
   const table = await db.get(ctx.params.tableId)
   let row = await findRow(ctx, ctx.params.tableId, ctx.params.rowId)
   row = await outputProcessing(table, row)

packages/server/src/api/controllers/row/staticFormula.js

@@ -62,6 +62,7 @@ exports.updateRelatedFormula = async (table, enrichedRows) => {
               })
             )
           )
+          break
         }
       }
     }
@@ -124,7 +125,6 @@ exports.finaliseRow = async (
     dynamic: false,
     contextRows: enrichedRow,
   })
-
   // don't worry about rev, tables handle rev/lastID updates
   // if another row has been written since processing this will
   // handle the auto ID clash

packages/server/src/api/controllers/table/internal.ts

@@ -2,7 +2,7 @@ import { updateLinks, EventType } from "../../../db/linkedRows"
 import { getRowParams, generateTableID } from "../../../db/utils"
 import { FieldTypes } from "../../../constants"
 import { TableSaveFunctions, hasTypeChanged, handleDataImport } from "./utils"
-const { getAppDB } = require("@budibase/backend-core/context")
+import { context } from "@budibase/backend-core"
 import { isTest } from "../../../environment"
 import {
   cleanupAttachments,
@@ -35,7 +35,7 @@ function checkAutoColumns(table: Table, oldTable: Table) {
 }
 
 export async function save(ctx: any) {
-  const db = getAppDB()
+  const db = context.getAppDB()
   const { dataImport, ...rest } = ctx.request.body
   let tableToSave = {
     type: "table",
@@ -138,19 +138,19 @@ export async function save(ctx: any) {
 }
 
 export async function destroy(ctx: any) {
-  const db = getAppDB()
+  const db = context.getAppDB()
   const tableToDelete = await db.get(ctx.params.tableId)
 
   // Delete all rows for that table
-  const rows = await db.allDocs(
+  const rowsData = await db.allDocs(
     getRowParams(ctx.params.tableId, null, {
       include_docs: true,
     })
   )
   await db.bulkDocs(
-    rows.rows.map((row: any) => ({ ...row.doc, _deleted: true }))
+    rowsData.rows.map((row: any) => ({ ...row.doc, _deleted: true }))
   )
-  await quotas.removeRows(rows.rows.length, {
+  await quotas.removeRows(rowsData.rows.length, {
     tableId: ctx.params.tableId,
   })
 
@@ -161,7 +161,7 @@ export async function destroy(ctx: any) {
   })
 
   // don't remove the table itself until very end
-  await db.remove(tableToDelete)
+  await db.remove(tableToDelete._id, tableToDelete._rev)
 
   // remove table search index
   if (!isTest() || env.COUCH_DB_URL) {
@@ -179,7 +179,9 @@ export async function destroy(ctx: any) {
     oldTable: null,
     deletion: true,
   })
-  await cleanupAttachments(tableToDelete, { rows })
+  await cleanupAttachments(tableToDelete, {
+    rows: rowsData.rows.map((row: any) => row.doc),
+  })
   return tableToDelete
 }
 

packages/server/src/api/routes/automation.js

@@ -3,8 +3,8 @@ const controller = require("../controllers/automation")
 const authorized = require("../../middleware/authorized")
 const {
   BUILDER,
-  PermissionLevels,
+  PermissionLevel,
-  PermissionTypes,
+  PermissionType,
 } = require("@budibase/backend-core/permissions")
 const { bodyResource, paramResource } = require("../../middleware/resourceId")
 const {
@@ -71,14 +71,14 @@ router
     "/api/automations/:id/trigger",
     appInfoMiddleware({ appType: AppType.PROD }),
     paramResource("id"),
-    authorized(PermissionTypes.AUTOMATION, PermissionLevels.EXECUTE),
+    authorized(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
     controller.trigger
   )
   .post(
     "/api/automations/:id/test",
     appInfoMiddleware({ appType: AppType.DEV }),
     paramResource("id"),
-    authorized(PermissionTypes.AUTOMATION, PermissionLevels.EXECUTE),
+    authorized(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
     controller.test
   )