Merge branch 'develop' of github.com:Budibase/budibase into side-panel

.eslintignore

@@ -3,6 +3,8 @@ public
 dist
 packages/server/builder
 packages/server/coverage
+packages/worker/coverage
+packages/backend-core/coverage
 packages/server/client
 packages/builder/.routify
 packages/builder/cypress/support/queryLevelTransformerFunction.js

.github/workflows/budibase_ci.yml

@@ -58,7 +58,7 @@ jobs:
     - uses: codecov/codecov-action@v1
       with:
         token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
-        files: ./packages/server/coverage/clover.xml
+        files: ./packages/server/coverage/clover.xml,./packages/worker/coverage/clover.xml,./packages/backend-core/coverage/clover.xml
         name: codecov-umbrella
         verbose: true 
 

.prettierignore

@@ -4,6 +4,8 @@ dist
 packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte
 packages/server/builder
 packages/server/coverage
+packages/worker/coverage
+packages/backend-core/coverage
 packages/server/client
 packages/server/src/definitions/openapi.ts
 packages/builder/.routify

hosting/nginx.dev.conf.hbs

@@ -58,12 +58,15 @@ http {
     }
 
     location ~ ^/api/(system|admin|global)/ {
-      proxy_pass      http://worker-service;
       proxy_read_timeout 120s;
       proxy_connect_timeout 120s;
       proxy_send_timeout 120s;
       proxy_http_version 1.1;
+
+      proxy_set_header Host $host;
       proxy_set_header Connection "";
+
+      proxy_pass      http://worker-service;
     }
 
     location /api/backups/ {
@@ -78,60 +81,78 @@ http {
     location /api/ {
       proxy_read_timeout 120s;
       proxy_connect_timeout 120s;
-      proxy_send_timeout 120s; 
+      proxy_send_timeout 120s;
-      proxy_pass      http://app-service;
       proxy_http_version 1.1;
+
+      proxy_set_header Host $host;
       proxy_set_header Connection "";
+
+      proxy_pass      http://app-service;
     }
 
     location = / {
-      proxy_pass      http://app-service;
       proxy_read_timeout 120s;
       proxy_connect_timeout 120s;
       proxy_send_timeout 120s;
       proxy_http_version 1.1;
+
+      proxy_set_header Host $host;
       proxy_set_header Connection "";
+
+      proxy_pass      http://app-service;
     }
 
     location /app_ {
-      proxy_pass      http://app-service;
       proxy_read_timeout 120s;
       proxy_connect_timeout 120s;
       proxy_send_timeout 120s;
       proxy_http_version 1.1;
+
+      proxy_set_header Host $host;
       proxy_set_header Connection "";
+
+      proxy_pass      http://app-service;
     }
 
     location /app {
-      proxy_pass      http://app-service;
       proxy_read_timeout 120s;
       proxy_connect_timeout 120s;
       proxy_send_timeout 120s;
       proxy_http_version 1.1;
+
+      proxy_set_header Host $host;
       proxy_set_header Connection "";
+
+      proxy_pass      http://app-service;
     }
 
     location /builder {
-      proxy_pass      http://builder;
       proxy_read_timeout 120s;
       proxy_connect_timeout 120s;
       proxy_send_timeout 120s;
       proxy_http_version 1.1;
+
+      proxy_set_header Host $host;
       proxy_set_header Connection "";
+
+      proxy_pass      http://builder;
       rewrite ^/builder(.*)$ /builder/$1 break;
     }
 
     location /builder/ {
-      proxy_pass      http://builder;
-
       proxy_http_version  1.1;
+
+      proxy_set_header    Host                $host;
       proxy_set_header    Connection          $connection_upgrade;
       proxy_set_header    Upgrade             $http_upgrade;
       proxy_set_header    X-Real-IP           $remote_addr;
       proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+
       proxy_read_timeout 120s;
       proxy_connect_timeout 120s;
       proxy_send_timeout 120s;
+
+      proxy_pass      http://builder;
     }
 
     location /vite/ {

hosting/nginx.prod.conf.hbs

@@ -100,18 +100,25 @@ http {
 
     location ~ ^/(builder|app_) {
       proxy_http_version  1.1;
+
       proxy_set_header    Connection          $connection_upgrade;
       proxy_set_header    Upgrade             $http_upgrade;
       proxy_set_header    X-Real-IP           $remote_addr;
       proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+      proxy_set_header    Host                $host;
+
       proxy_pass      http://$apps:4002;
     }
 
     location ~ ^/api/(system|admin|global)/ {
+      proxy_set_header    Host                $host;
+
       proxy_pass      http://$worker:4003;
     }
 
     location /worker/ {
+      proxy_set_header    Host                $host;
+
       proxy_pass      http://$worker:4003;
       rewrite ^/worker/(.*)$ /$1 break;
     }
@@ -139,6 +146,7 @@ http {
       proxy_set_header    Upgrade             $http_upgrade;
       proxy_set_header    X-Real-IP           $remote_addr;
       proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+      proxy_set_header    Host                $host;
 
       proxy_pass      http://$apps:4002;
     }
@@ -158,6 +166,7 @@ http {
       proxy_set_header    Upgrade             $http_upgrade;
       proxy_set_header    X-Real-IP           $remote_addr;
       proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+      proxy_set_header    Host                $host;
 
       proxy_pass      http://$apps:4002;
     }

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.1.22-alpha.2",
+  "version": "2.1.22-alpha.8",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -18,7 +18,7 @@
     "rimraf": "^3.0.2",
     "rollup-plugin-replace": "^2.2.0",
     "svelte": "^3.38.2",
-    "typescript": "4.5.5"
+    "typescript": "4.7.3"
   },
   "scripts": {
     "setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
@@ -75,8 +75,8 @@
     "env:multi:disable": "lerna run env:multi:disable",
     "env:selfhost:enable": "lerna run env:selfhost:enable",
     "env:selfhost:disable": "lerna run env:selfhost:disable",
-    "env:localdomain:enable": "lerna run env:localdomain:enable",
+    "env:localdomain:enable": "./scripts/localdomain.sh enable",
-    "env:localdomain:disable": "lerna run env:localdomain:disable",
+    "env:localdomain:disable": "./scripts/localdomain.sh disable",
     "env:account:enable": "lerna run env:account:enable",
     "env:account:disable": "lerna run env:account:disable",
     "mode:self": "yarn env:selfhost:enable && yarn env:multi:disable && yarn env:account:disable",

packages/backend-core/__mocks__/aws-sdk.ts

@@ -0,0 +1,15 @@
+const mockS3 = {
+  headBucket: jest.fn().mockReturnThis(),
+  deleteObject: jest.fn().mockReturnThis(),
+  deleteObjects: jest.fn().mockReturnThis(),
+  createBucket: jest.fn().mockReturnThis(),
+  listObjects: jest.fn().mockReturnThis(),
+  promise: jest.fn().mockReturnThis(),
+  catch: jest.fn(),
+}
+
+const AWS = {
+  S3: jest.fn(() => mockS3),
+}
+
+export default AWS

packages/backend-core/db.js

@@ -1,7 +1 @@
-module.exports = {
+module.exports = require("./src/db")
-  ...require("./src/db/utils"),
-  ...require("./src/db/constants"),
-  ...require("./src/db"),
-  ...require("./src/db/views"),
-  ...require("./src/db/pouch"),
-}

packages/backend-core/jest.config.ts

@@ -0,0 +1,21 @@
+import { Config } from "@jest/types"
+
+const config: Config.InitialOptions = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  setupFiles: ["./tests/jestSetup.ts"],
+  collectCoverageFrom: ["src/**/*.{js,ts}"],
+  coverageReporters: ["lcov", "json", "clover"],
+}
+
+if (!process.env.CI) {
+  // use sources when not in CI
+  config.moduleNameMapper = {
+    "@budibase/types": "<rootDir>/../types/src",
+    "^axios.*$": "<rootDir>/node_modules/axios/lib/axios.js",
+  }
+} else {
+  console.log("Running tests with compiled dependency sources")
+}
+
+export default config

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.1.22-alpha.2",
+  "version": "2.1.22-alpha.8",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -16,11 +16,11 @@
     "prepack": "cp package.json dist",
     "build": "tsc -p tsconfig.build.json",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
-    "test": "jest",
+    "test": "jest --coverage",
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
-    "@budibase/types": "2.1.22-alpha.2",
+    "@budibase/types": "2.1.22-alpha.8",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-sdk": "2.1030.0",
@@ -35,6 +35,7 @@
     "koa-passport": "4.1.4",
     "lodash": "4.17.21",
     "lodash.isarguments": "3.1.0",
+    "nano": "^10.1.0",
     "node-fetch": "2.6.7",
     "passport-google-auth": "1.0.2",
     "passport-google-oauth": "2.0.0",
@@ -52,21 +53,11 @@
     "uuid": "8.3.2",
     "zlib": "1.0.5"
   },
-  "jest": {
-    "preset": "ts-jest",
-    "testEnvironment": "node",
-    "moduleNameMapper": {
-      "@budibase/types": "<rootDir>/../types/src"
-    },
-    "setupFiles": [
-      "./scripts/jestSetup.ts"
-    ]
-  },
   "devDependencies": {
     "@types/chance": "1.1.3",
     "@types/ioredis": "4.28.0",
     "@types/jest": "27.5.1",
-    "@types/koa": "2.0.52",
+    "@types/koa": "2.13.4",
     "@types/lodash": "4.14.180",
     "@types/node": "14.18.20",
     "@types/node-fetch": "2.6.1",
@@ -77,12 +68,14 @@
     "@types/uuid": "8.3.4",
     "chance": "1.1.3",
     "ioredis-mock": "5.8.0",
-    "jest": "27.5.1",
+    "jest": "28.1.1",
-    "koa": "2.7.0",
+    "koa": "2.13.4",
     "nodemon": "2.0.16",
     "pouchdb-adapter-memory": "7.2.2",
     "timekeeper": "2.2.0",
-    "ts-jest": "27.1.5",
+    "ts-jest": "28.0.4",
+    "ts-node": "10.8.1",
+    "tsconfig-paths": "4.0.0",
     "typescript": "4.7.3"
   },
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"

packages/backend-core/scripts/jestSetup.ts

@@ -1,12 +0,0 @@
-import env from "../src/environment"
-import { mocks } from "../tests/utilities"
-
-// mock all dates to 2020-01-01T00:00:00.000Z
-// use tk.reset() to use real dates in individual tests
-import tk from "timekeeper"
-tk.freeze(mocks.date.MOCK_DATE)
-
-env._set("SELF_HOSTED", "1")
-env._set("NODE_ENV", "jest")
-env._set("JWT_SECRET", "test-jwtsecret")
-env._set("LOG_LEVEL", "silent")

packages/backend-core/src/auth.ts

@@ -3,7 +3,7 @@ const LocalStrategy = require("passport-local").Strategy
 const JwtStrategy = require("passport-jwt").Strategy
 import { getGlobalDB } from "./tenancy"
 const refresh = require("passport-oauth2-refresh")
-import { Configs } from "./constants"
+import { Config } from "./constants"
 import { getScopedConfig } from "./db/utils"
 import {
   jwt,
@@ -76,7 +76,7 @@ async function refreshOIDCAccessToken(
 
   return new Promise(resolve => {
     refresh.requestNewAccessToken(
-      Configs.OIDC,
+      Config.OIDC,
       refreshToken,
       (err: any, accessToken: string, refreshToken: any, params: any) => {
         resolve({ err, accessToken, refreshToken, params })
@@ -106,7 +106,7 @@ async function refreshGoogleAccessToken(
 
   return new Promise(resolve => {
     refresh.requestNewAccessToken(
-      Configs.GOOGLE,
+      Config.GOOGLE,
       refreshToken,
       (err: any, accessToken: string, refreshToken: string, params: any) => {
         resolve({ err, accessToken, refreshToken, params })
@@ -129,7 +129,7 @@ async function refreshOAuthToken(
 
   let chosenConfig = {}
   let refreshResponse
-  if (configType === Configs.OIDC) {
+  if (configType === Config.OIDC) {
     // configId - retrieved from cookie.
     chosenConfig = config.configs.filter((c: any) => c.uuid === configId)[0]
     if (!chosenConfig) {

packages/backend-core/src/cache/tests/writethrough.spec.js

@@ -1,6 +1,6 @@
-require("../../../tests/utilities/TestConfiguration")
+require("../../../tests")
 const { Writethrough } = require("../writethrough")
-const { dangerousGetDB } = require("../../db")
+const { getDB } = require("../../db")
 const tk = require("timekeeper")
 
 const START_DATE = Date.now()
@@ -8,8 +8,8 @@ tk.freeze(START_DATE)
 
 const DELAY = 5000
 
-const db = dangerousGetDB("test")
+const db = getDB("test")
-const db2 = dangerousGetDB("test2")
+const db2 = getDB("test2")
 const writethrough = new Writethrough(db, DELAY), writethrough2 = new Writethrough(db2, DELAY)
 
 describe("writethrough", () => {

packages/backend-core/src/cache/writethrough.ts

@@ -1,7 +1,7 @@
 import BaseCache from "./base"
 import { getWritethroughClient } from "../redis/init"
 import { logWarn } from "../logging"
-import PouchDB from "pouchdb"
+import { Database } from "@budibase/types"
 
 const DEFAULT_WRITE_RATE_MS = 10000
 let CACHE: BaseCache | null = null
@@ -19,7 +19,7 @@ async function getCache() {
   return CACHE
 }
 
-function makeCacheKey(db: PouchDB.Database, key: string) {
+function makeCacheKey(db: Database, key: string) {
   return db.name + key
 }
 
@@ -28,7 +28,7 @@ function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem {
 }
 
 export async function put(
-  db: PouchDB.Database,
+  db: Database,
   doc: any,
   writeRateMs: number = DEFAULT_WRITE_RATE_MS
 ) {
@@ -64,7 +64,7 @@ export async function put(
   return { ok: true, id: output._id, rev: output._rev }
 }
 
-export async function get(db: PouchDB.Database, id: string): Promise<any> {
+export async function get(db: Database, id: string): Promise<any> {
   const cache = await getCache()
   const cacheKey = makeCacheKey(db, id)
   let cacheItem: CacheItem = await cache.get(cacheKey)
@@ -77,7 +77,7 @@ export async function get(db: PouchDB.Database, id: string): Promise<any> {
 }
 
 export async function remove(
-  db: PouchDB.Database,
+  db: Database,
   docOrId: any,
   rev?: any
 ): Promise<void> {
@@ -95,13 +95,10 @@ export async function remove(
 }
 
 export class Writethrough {
-  db: PouchDB.Database
+  db: Database
   writeRateMs: number
 
-  constructor(
+  constructor(db: Database, writeRateMs: number = DEFAULT_WRITE_RATE_MS) {
-    db: PouchDB.Database,
-    writeRateMs: number = DEFAULT_WRITE_RATE_MS
-  ) {
     this.db = db
     this.writeRateMs = writeRateMs
   }

packages/backend-core/src/cloud/accounts.ts

@@ -1,6 +1,6 @@
 import API from "./api"
 import env from "../environment"
-import { Headers } from "../constants"
+import { Header } from "../constants"
 import { CloudAccount } from "@budibase/types"
 
 const api = new API(env.ACCOUNT_PORTAL_URL)
@@ -14,7 +14,7 @@ export const getAccount = async (
   const response = await api.post(`/api/accounts/search`, {
     body: payload,
     headers: {
-      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+      [Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
     },
   })
 
@@ -35,7 +35,7 @@ export const getAccountByTenantId = async (
   const response = await api.post(`/api/accounts/search`, {
     body: payload,
     headers: {
-      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+      [Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
     },
   })
 
@@ -50,7 +50,7 @@ export const getAccountByTenantId = async (
 export const getStatus = async () => {
   const response = await api.get(`/api/status`, {
     headers: {
-      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+      [Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
     },
   })
   const json = await response.json()

packages/backend-core/src/clshooked/index.js

@@ -1,650 +0,0 @@
-const util = require("util")
-const assert = require("assert")
-const wrapEmitter = require("emitter-listener")
-const async_hooks = require("async_hooks")
-
-const CONTEXTS_SYMBOL = "cls@contexts"
-const ERROR_SYMBOL = "error@context"
-
-const DEBUG_CLS_HOOKED = process.env.DEBUG_CLS_HOOKED
-
-let currentUid = -1
-
-module.exports = {
-  getNamespace: getNamespace,
-  createNamespace: createNamespace,
-  destroyNamespace: destroyNamespace,
-  reset: reset,
-  ERROR_SYMBOL: ERROR_SYMBOL,
-}
-
-function Namespace(name) {
-  this.name = name
-  // changed in 2.7: no default context
-  this.active = null
-  this._set = []
-  this.id = null
-  this._contexts = new Map()
-  this._indent = 0
-  this._hook = null
-}
-
-Namespace.prototype.set = function set(key, value) {
-  if (!this.active) {
-    throw new Error(
-      "No context available. ns.run() or ns.bind() must be called first."
-    )
-  }
-
-  this.active[key] = value
-
-  if (DEBUG_CLS_HOOKED) {
-    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
-    debug2(
-      indentStr +
-        "CONTEXT-SET KEY:" +
-        key +
-        "=" +
-        value +
-        " in ns:" +
-        this.name +
-        " currentUid:" +
-        currentUid +
-        " active:" +
-        util.inspect(this.active, { showHidden: true, depth: 2, colors: true })
-    )
-  }
-
-  return value
-}
-
-Namespace.prototype.get = function get(key) {
-  if (!this.active) {
-    if (DEBUG_CLS_HOOKED) {
-      const asyncHooksCurrentId = async_hooks.currentId()
-      const triggerId = async_hooks.triggerAsyncId()
-      const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
-      debug2(
-        `${indentStr}CONTEXT-GETTING KEY NO ACTIVE NS: (${this.name}) ${key}=undefined currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${this._set.length}`
-      )
-    }
-    return undefined
-  }
-  if (DEBUG_CLS_HOOKED) {
-    const asyncHooksCurrentId = async_hooks.executionAsyncId()
-    const triggerId = async_hooks.triggerAsyncId()
-    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
-    debug2(
-      indentStr +
-        "CONTEXT-GETTING KEY:" +
-        key +
-        "=" +
-        this.active[key] +
-        " (" +
-        this.name +
-        ") currentUid:" +
-        currentUid +
-        " active:" +
-        util.inspect(this.active, { showHidden: true, depth: 2, colors: true })
-    )
-    debug2(
-      `${indentStr}CONTEXT-GETTING KEY: (${this.name}) ${key}=${
-        this.active[key]
-      } currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${
-        this._set.length
-      } active:${util.inspect(this.active)}`
-    )
-  }
-  return this.active[key]
-}
-
-Namespace.prototype.createContext = function createContext() {
-  // Prototype inherit existing context if created a new child context within existing context.
-  let context = Object.create(this.active ? this.active : Object.prototype)
-  context._ns_name = this.name
-  context.id = currentUid
-
-  if (DEBUG_CLS_HOOKED) {
-    const asyncHooksCurrentId = async_hooks.executionAsyncId()
-    const triggerId = async_hooks.triggerAsyncId()
-    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
-    debug2(
-      `${indentStr}CONTEXT-CREATED Context: (${
-        this.name
-      }) currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${
-        this._set.length
-      } context:${util.inspect(context, {
-        showHidden: true,
-        depth: 2,
-        colors: true,
-      })}`
-    )
-  }
-
-  return context
-}
-
-Namespace.prototype.run = function run(fn) {
-  let context = this.createContext()
-  this.enter(context)
-
-  try {
-    if (DEBUG_CLS_HOOKED) {
-      const triggerId = async_hooks.triggerAsyncId()
-      const asyncHooksCurrentId = async_hooks.executionAsyncId()
-      const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
-      debug2(
-        `${indentStr}CONTEXT-RUN BEGIN: (${
-          this.name
-        }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
-          this._set.length
-        } context:${util.inspect(context)}`
-      )
-    }
-    fn(context)
-    return context
-  } catch (exception) {
-    if (exception) {
-      exception[ERROR_SYMBOL] = context
-    }
-    throw exception
-  } finally {
-    if (DEBUG_CLS_HOOKED) {
-      const triggerId = async_hooks.triggerAsyncId()
-      const asyncHooksCurrentId = async_hooks.executionAsyncId()
-      const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
-      debug2(
-        `${indentStr}CONTEXT-RUN END: (${
-          this.name
-        }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
-          this._set.length
-        } ${util.inspect(context)}`
-      )
-    }
-    this.exit(context)
-  }
-}
-
-Namespace.prototype.runAndReturn = function runAndReturn(fn) {
-  let value
-  this.run(function (context) {
-    value = fn(context)
-  })
-  return value
-}
-
-/**
- * Uses global Promise and assumes Promise is cls friendly or wrapped already.
- * @param {function} fn
- * @returns {*}
- */
-Namespace.prototype.runPromise = function runPromise(fn) {
-  let context = this.createContext()
-  this.enter(context)
-
-  let promise = fn(context)
-  if (!promise || !promise.then || !promise.catch) {
-    throw new Error("fn must return a promise.")
-  }
-
-  if (DEBUG_CLS_HOOKED) {
-    debug2(
-      "CONTEXT-runPromise BEFORE: (" +
-        this.name +
-        ") currentUid:" +
-        currentUid +
-        " len:" +
-        this._set.length +
-        " " +
-        util.inspect(context)
-    )
-  }
-
-  return promise
-    .then(result => {
-      if (DEBUG_CLS_HOOKED) {
-        debug2(
-          "CONTEXT-runPromise AFTER then: (" +
-            this.name +
-            ") currentUid:" +
-            currentUid +
-            " len:" +
-            this._set.length +
-            " " +
-            util.inspect(context)
-        )
-      }
-      this.exit(context)
-      return result
-    })
-    .catch(err => {
-      err[ERROR_SYMBOL] = context
-      if (DEBUG_CLS_HOOKED) {
-        debug2(
-          "CONTEXT-runPromise AFTER catch: (" +
-            this.name +
-            ") currentUid:" +
-            currentUid +
-            " len:" +
-            this._set.length +
-            " " +
-            util.inspect(context)
-        )
-      }
-      this.exit(context)
-      throw err
-    })
-}
-
-Namespace.prototype.bind = function bindFactory(fn, context) {
-  if (!context) {
-    if (!this.active) {
-      context = this.createContext()
-    } else {
-      context = this.active
-    }
-  }
-
-  let self = this
-  return function clsBind() {
-    self.enter(context)
-    try {
-      return fn.apply(this, arguments)
-    } catch (exception) {
-      if (exception) {
-        exception[ERROR_SYMBOL] = context
-      }
-      throw exception
-    } finally {
-      self.exit(context)
-    }
-  }
-}
-
-Namespace.prototype.enter = function enter(context) {
-  assert.ok(context, "context must be provided for entering")
-  if (DEBUG_CLS_HOOKED) {
-    const asyncHooksCurrentId = async_hooks.executionAsyncId()
-    const triggerId = async_hooks.triggerAsyncId()
-    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
-    debug2(
-      `${indentStr}CONTEXT-ENTER: (${
-        this.name
-      }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
-        this._set.length
-      } ${util.inspect(context)}`
-    )
-  }
-
-  this._set.push(this.active)
-  this.active = context
-}
-
-Namespace.prototype.exit = function exit(context) {
-  assert.ok(context, "context must be provided for exiting")
-  if (DEBUG_CLS_HOOKED) {
-    const asyncHooksCurrentId = async_hooks.executionAsyncId()
-    const triggerId = async_hooks.triggerAsyncId()
-    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
-    debug2(
-      `${indentStr}CONTEXT-EXIT: (${
-        this.name
-      }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
-        this._set.length
-      } ${util.inspect(context)}`
-    )
-  }
-
-  // Fast path for most exits that are at the top of the stack
-  if (this.active === context) {
-    assert.ok(this._set.length, "can't remove top context")
-    this.active = this._set.pop()
-    return
-  }
-
-  // Fast search in the stack using lastIndexOf
-  let index = this._set.lastIndexOf(context)
-
-  if (index < 0) {
-    if (DEBUG_CLS_HOOKED) {
-      debug2(
-        "??ERROR?? context exiting but not entered - ignoring: " +
-          util.inspect(context)
-      )
-    }
-    assert.ok(
-      index >= 0,
-      "context not currently entered; can't exit. \n" +
-        util.inspect(this) +
-        "\n" +
-        util.inspect(context)
-    )
-  } else {
-    assert.ok(index, "can't remove top context")
-    this._set.splice(index, 1)
-  }
-}
-
-Namespace.prototype.bindEmitter = function bindEmitter(emitter) {
-  assert.ok(
-    emitter.on && emitter.addListener && emitter.emit,
-    "can only bind real EEs"
-  )
-
-  let namespace = this
-  let thisSymbol = "context@" + this.name
-
-  // Capture the context active at the time the emitter is bound.
-  function attach(listener) {
-    if (!listener) {
-      return
-    }
-    if (!listener[CONTEXTS_SYMBOL]) {
-      listener[CONTEXTS_SYMBOL] = Object.create(null)
-    }
-
-    listener[CONTEXTS_SYMBOL][thisSymbol] = {
-      namespace: namespace,
-      context: namespace.active,
-    }
-  }
-
-  // At emit time, bind the listener within the correct context.
-  function bind(unwrapped) {
-    if (!(unwrapped && unwrapped[CONTEXTS_SYMBOL])) {
-      return unwrapped
-    }
-
-    let wrapped = unwrapped
-    let unwrappedContexts = unwrapped[CONTEXTS_SYMBOL]
-    Object.keys(unwrappedContexts).forEach(function (name) {
-      let thunk = unwrappedContexts[name]
-      wrapped = thunk.namespace.bind(wrapped, thunk.context)
-    })
-    return wrapped
-  }
-
-  wrapEmitter(emitter, attach, bind)
-}
-
-/**
- * If an error comes out of a namespace, it will have a context attached to it.
- * This function knows how to find it.
- *
- * @param {Error} exception Possibly annotated error.
- */
-Namespace.prototype.fromException = function fromException(exception) {
-  return exception[ERROR_SYMBOL]
-}
-
-function getNamespace(name) {
-  return process.namespaces[name]
-}
-
-function createNamespace(name) {
-  assert.ok(name, "namespace must be given a name.")
-
-  if (DEBUG_CLS_HOOKED) {
-    debug2(`NS-CREATING NAMESPACE (${name})`)
-  }
-  let namespace = new Namespace(name)
-  namespace.id = currentUid
-
-  const hook = async_hooks.createHook({
-    init(asyncId, type, triggerId, resource) {
-      currentUid = async_hooks.executionAsyncId()
-
-      //CHAIN Parent's Context onto child if none exists. This is needed to pass net-events.spec
-      // let initContext = namespace.active;
-      // if(!initContext && triggerId) {
-      //   let parentContext = namespace._contexts.get(triggerId);
-      //   if (parentContext) {
-      //     namespace.active = parentContext;
-      //     namespace._contexts.set(currentUid, parentContext);
-      //     if (DEBUG_CLS_HOOKED) {
-      //       const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent);
-      //       debug2(`${indentStr}INIT [${type}] (${name}) WITH PARENT CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`);
-      //     }
-      //   } else if (DEBUG_CLS_HOOKED) {
-      //       const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent);
-      //       debug2(`${indentStr}INIT [${type}] (${name}) MISSING CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`);
-      //     }
-      // }else {
-      //   namespace._contexts.set(currentUid, namespace.active);
-      //   if (DEBUG_CLS_HOOKED) {
-      //     const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent);
-      //     debug2(`${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`);
-      //   }
-      // }
-      if (namespace.active) {
-        namespace._contexts.set(asyncId, namespace.active)
-
-        if (DEBUG_CLS_HOOKED) {
-          const indentStr = " ".repeat(
-            namespace._indent < 0 ? 0 : namespace._indent
-          )
-          debug2(
-            `${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
-              namespace.active,
-              { showHidden: true, depth: 2, colors: true }
-            )} resource:${resource}`
-          )
-        }
-      } else if (currentUid === 0) {
-        // CurrentId will be 0 when triggered from C++. Promise events
-        // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid
-        const triggerId = async_hooks.triggerAsyncId()
-        const triggerIdContext = namespace._contexts.get(triggerId)
-        if (triggerIdContext) {
-          namespace._contexts.set(asyncId, triggerIdContext)
-          if (DEBUG_CLS_HOOKED) {
-            const indentStr = " ".repeat(
-              namespace._indent < 0 ? 0 : namespace._indent
-            )
-            debug2(
-              `${indentStr}INIT USING CONTEXT FROM TRIGGERID [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
-                namespace.active,
-                { showHidden: true, depth: 2, colors: true }
-              )} resource:${resource}`
-            )
-          }
-        } else if (DEBUG_CLS_HOOKED) {
-          const indentStr = " ".repeat(
-            namespace._indent < 0 ? 0 : namespace._indent
-          )
-          debug2(
-            `${indentStr}INIT MISSING CONTEXT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
-              namespace.active,
-              { showHidden: true, depth: 2, colors: true }
-            )} resource:${resource}`
-          )
-        }
-      }
-
-      if (DEBUG_CLS_HOOKED && type === "PROMISE") {
-        debug2(util.inspect(resource, { showHidden: true }))
-        const parentId = resource.parentId
-        const indentStr = " ".repeat(
-          namespace._indent < 0 ? 0 : namespace._indent
-        )
-        debug2(
-          `${indentStr}INIT RESOURCE-PROMISE [${type}] (${name}) parentId:${parentId} asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
-            namespace.active,
-            { showHidden: true, depth: 2, colors: true }
-          )} resource:${resource}`
-        )
-      }
-    },
-    before(asyncId) {
-      currentUid = async_hooks.executionAsyncId()
-      let context
-
-      /*
-      if(currentUid === 0){
-        // CurrentId will be 0 when triggered from C++. Promise events
-        // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid
-        //const triggerId = async_hooks.triggerAsyncId();
-        context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId);
-      }else{
-        context = namespace._contexts.get(currentUid);
-      }
-      */
-
-      //HACK to work with promises until they are fixed in node > 8.1.1
-      context =
-        namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid)
-
-      if (context) {
-        if (DEBUG_CLS_HOOKED) {
-          const triggerId = async_hooks.triggerAsyncId()
-          const indentStr = " ".repeat(
-            namespace._indent < 0 ? 0 : namespace._indent
-          )
-          debug2(
-            `${indentStr}BEFORE (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
-              namespace.active,
-              { showHidden: true, depth: 2, colors: true }
-            )} context:${util.inspect(context)}`
-          )
-          namespace._indent += 2
-        }
-
-        namespace.enter(context)
-      } else if (DEBUG_CLS_HOOKED) {
-        const triggerId = async_hooks.triggerAsyncId()
-        const indentStr = " ".repeat(
-          namespace._indent < 0 ? 0 : namespace._indent
-        )
-        debug2(
-          `${indentStr}BEFORE MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
-            namespace.active,
-            { showHidden: true, depth: 2, colors: true }
-          )} namespace._contexts:${util.inspect(namespace._contexts, {
-            showHidden: true,
-            depth: 2,
-            colors: true,
-          })}`
-        )
-        namespace._indent += 2
-      }
-    },
-    after(asyncId) {
-      currentUid = async_hooks.executionAsyncId()
-      let context // = namespace._contexts.get(currentUid);
-      /*
-      if(currentUid === 0){
-        // CurrentId will be 0 when triggered from C++. Promise events
-        // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid
-        //const triggerId = async_hooks.triggerAsyncId();
-        context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId);
-      }else{
-        context = namespace._contexts.get(currentUid);
-      }
-      */
-      //HACK to work with promises until they are fixed in node > 8.1.1
-      context =
-        namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid)
-
-      if (context) {
-        if (DEBUG_CLS_HOOKED) {
-          const triggerId = async_hooks.triggerAsyncId()
-          namespace._indent -= 2
-          const indentStr = " ".repeat(
-            namespace._indent < 0 ? 0 : namespace._indent
-          )
-          debug2(
-            `${indentStr}AFTER (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
-              namespace.active,
-              { showHidden: true, depth: 2, colors: true }
-            )} context:${util.inspect(context)}`
-          )
-        }
-
-        namespace.exit(context)
-      } else if (DEBUG_CLS_HOOKED) {
-        const triggerId = async_hooks.triggerAsyncId()
-        namespace._indent -= 2
-        const indentStr = " ".repeat(
-          namespace._indent < 0 ? 0 : namespace._indent
-        )
-        debug2(
-          `${indentStr}AFTER MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
-            namespace.active,
-            { showHidden: true, depth: 2, colors: true }
-          )} context:${util.inspect(context)}`
-        )
-      }
-    },
-    destroy(asyncId) {
-      currentUid = async_hooks.executionAsyncId()
-      if (DEBUG_CLS_HOOKED) {
-        const triggerId = async_hooks.triggerAsyncId()
-        const indentStr = " ".repeat(
-          namespace._indent < 0 ? 0 : namespace._indent
-        )
-        debug2(
-          `${indentStr}DESTROY (${name}) currentUid:${currentUid} asyncId:${asyncId} triggerId:${triggerId} active:${util.inspect(
-            namespace.active,
-            { showHidden: true, depth: 2, colors: true }
-          )} context:${util.inspect(namespace._contexts.get(currentUid))}`
-        )
-      }
-
-      namespace._contexts.delete(asyncId)
-    },
-  })
-
-  hook.enable()
-  namespace._hook = hook
-
-  process.namespaces[name] = namespace
-  return namespace
-}
-
-function destroyNamespace(name) {
-  let namespace = getNamespace(name)
-
-  assert.ok(namespace, "can't delete nonexistent namespace! \"" + name + '"')
-  assert.ok(
-    namespace.id,
-    "don't assign to process.namespaces directly! " + util.inspect(namespace)
-  )
-
-  namespace._hook.disable()
-  namespace._contexts = null
-  process.namespaces[name] = null
-}
-
-function reset() {
-  // must unregister async listeners
-  if (process.namespaces) {
-    Object.keys(process.namespaces).forEach(function (name) {
-      destroyNamespace(name)
-    })
-  }
-  process.namespaces = Object.create(null)
-}
-
-process.namespaces = process.namespaces || {}
-
-//const fs = require('fs');
-function debug2(...args) {
-  if (DEBUG_CLS_HOOKED) {
-    //fs.writeSync(1, `${util.format(...args)}\n`);
-    process._rawDebug(`${util.format(...args)}`)
-  }
-}
-
-/*function getFunctionName(fn) {
-  if (!fn) {
-    return fn;
-  }
-  if (typeof fn === 'function') {
-    if (fn.name) {
-      return fn.name;
-    }
-    return (fn.toString().trim().match(/^function\s*([^\s(]+)/) || [])[1];
-  } else if (fn.constructor && fn.constructor.name) {
-    return fn.constructor.name;
-  }
-}*/

packages/backend-core/src/constants.js

@@ -1,44 +0,0 @@
-exports.UserStatus = {
-  ACTIVE: "active",
-  INACTIVE: "inactive",
-}
-
-exports.Cookies = {
-  CurrentApp: "budibase:currentapp",
-  Auth: "budibase:auth",
-  Init: "budibase:init",
-  ACCOUNT_RETURN_URL: "budibase:account:returnurl",
-  DatasourceAuth: "budibase:datasourceauth",
-  OIDC_CONFIG: "budibase:oidc:config",
-}
-
-exports.Headers = {
-  API_KEY: "x-budibase-api-key",
-  LICENSE_KEY: "x-budibase-license-key",
-  API_VER: "x-budibase-api-version",
-  APP_ID: "x-budibase-app-id",
-  TYPE: "x-budibase-type",
-  PREVIEW_ROLE: "x-budibase-role",
-  TENANT_ID: "x-budibase-tenant-id",
-  TOKEN: "x-budibase-token",
-  CSRF_TOKEN: "x-csrf-token",
-}
-
-exports.GlobalRoles = {
-  OWNER: "owner",
-  ADMIN: "admin",
-  BUILDER: "builder",
-  WORKSPACE_MANAGER: "workspace_manager",
-}
-
-exports.Configs = {
-  SETTINGS: "settings",
-  ACCOUNT: "account",
-  SMTP: "smtp",
-  GOOGLE: "google",
-  OIDC: "oidc",
-  OIDC_LOGOS: "logos_oidc",
-}
-
-exports.MAX_VALID_DATE = new Date(2147483647000)
-exports.DEFAULT_TENANT_ID = "default"

packages/backend-core/src/constants.ts

@@ -0,0 +1,44 @@
+export enum UserStatus {
+  ACTIVE = "active",
+  INACTIVE = "inactive",
+}
+
+export enum Cookie {
+  CurrentApp = "budibase:currentapp",
+  Auth = "budibase:auth",
+  Init = "budibase:init",
+  ACCOUNT_RETURN_URL = "budibase:account:returnurl",
+  DatasourceAuth = "budibase:datasourceauth",
+  OIDC_CONFIG = "budibase:oidc:config",
+}
+
+export enum Header {
+  API_KEY = "x-budibase-api-key",
+  LICENSE_KEY = "x-budibase-license-key",
+  API_VER = "x-budibase-api-version",
+  APP_ID = "x-budibase-app-id",
+  TYPE = "x-budibase-type",
+  PREVIEW_ROLE = "x-budibase-role",
+  TENANT_ID = "x-budibase-tenant-id",
+  TOKEN = "x-budibase-token",
+  CSRF_TOKEN = "x-csrf-token",
+}
+
+export enum GlobalRole {
+  OWNER = "owner",
+  ADMIN = "admin",
+  BUILDER = "builder",
+  WORKSPACE_MANAGER = "workspace_manager",
+}
+
+export enum Config {
+  SETTINGS = "settings",
+  ACCOUNT = "account",
+  SMTP = "smtp",
+  GOOGLE = "google",
+  OIDC = "oidc",
+  OIDC_LOGOS = "logos_oidc",
+}
+
+export const MAX_VALID_DATE = new Date(2147483647000)
+export const DEFAULT_TENANT_ID = "default"

packages/backend-core/src/context/Context.ts

@@ -0,0 +1,18 @@
+import { AsyncLocalStorage } from "async_hooks"
+import { ContextMap } from "./constants"
+
+export default class Context {
+  static storage = new AsyncLocalStorage<ContextMap>()
+
+  static run(context: ContextMap, func: any) {
+    return Context.storage.run(context, () => func())
+  }
+
+  static get(): ContextMap {
+    return Context.storage.getStore() as ContextMap
+  }
+
+  static set(context: ContextMap) {
+    Context.storage.enterWith(context)
+  }
+}

packages/backend-core/src/context/FunctionContext.js

@@ -1,47 +0,0 @@
-const cls = require("../clshooked")
-const { newid } = require("../hashing")
-
-const REQUEST_ID_KEY = "requestId"
-const MAIN_CTX = cls.createNamespace("main")
-
-function getContextStorage(namespace) {
-  if (namespace && namespace.active) {
-    let contextData = namespace.active
-    delete contextData.id
-    delete contextData._ns_name
-    return contextData
-  }
-  return {}
-}
-
-class FunctionContext {
-  static run(callback) {
-    return MAIN_CTX.runAndReturn(async () => {
-      const namespaceId = newid()
-      MAIN_CTX.set(REQUEST_ID_KEY, namespaceId)
-      const namespace = cls.createNamespace(namespaceId)
-      let response = await namespace.runAndReturn(callback)
-      cls.destroyNamespace(namespaceId)
-      return response
-    })
-  }
-
-  static setOnContext(key, value) {
-    const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY)
-    const namespace = cls.getNamespace(namespaceId)
-    namespace.set(key, value)
-  }
-
-  static getFromContext(key) {
-    const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY)
-    const namespace = cls.getNamespace(namespaceId)
-    const context = getContextStorage(namespace)
-    if (context) {
-      return context[key]
-    } else {
-      return null
-    }
-  }
-}
-
-module.exports = FunctionContext

packages/backend-core/src/context/constants.ts

@@ -1,17 +1,7 @@
-export enum ContextKey {
+import { IdentityContext } from "@budibase/types"
-  TENANT_ID = "tenantId",
+
-  GLOBAL_DB = "globalDb",
+export type ContextMap = {
-  APP_ID = "appId",
+  tenantId?: string
-  IDENTITY = "identity",
+  appId?: string
-  // whatever the request app DB was
+  identity?: IdentityContext
-  CURRENT_DB = "currentDb",
-  // get the prod app DB from the request
-  PROD_DB = "prodDb",
-  // get the dev app DB from the request
-  DEV_DB = "devDb",
-  DB_OPTS = "dbOpts",
-  // check if something else is using the context, don't close DB
-  TENANCY_IN_USE = "tenancyInUse",
-  APP_IN_USE = "appInUse",
-  IDENTITY_IN_USE = "identityInUse",
 }

packages/backend-core/src/context/deprovision.ts

@@ -1,15 +1,19 @@
-const { getGlobalUserParams, getAllApps } = require("../db/utils")
+import {
-const { doWithDB } = require("../db")
+  getGlobalUserParams,
-const { doWithGlobalDB } = require("../tenancy")
+  getAllApps,
-const { StaticDatabases } = require("../db/constants")
+  doWithDB,
+  StaticDatabases,
+} from "../db"
+import { doWithGlobalDB } from "../tenancy"
+import { App, Tenants, User, Database } from "@budibase/types"
 
 const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
 const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
 
-const removeTenantFromInfoDB = async tenantId => {
+async function removeTenantFromInfoDB(tenantId: string) {
   try {
-    await doWithDB(PLATFORM_INFO_DB, async infoDb => {
+    await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => {
-      let tenants = await infoDb.get(TENANT_DOC)
+      const tenants = (await infoDb.get(TENANT_DOC)) as Tenants
       tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)
 
       await infoDb.put(tenants)
@@ -20,14 +24,14 @@ const removeTenantFromInfoDB = async tenantId => {
   }
 }
 
-exports.removeUserFromInfoDB = async dbUser => {
+export async function removeUserFromInfoDB(dbUser: User) {
-  await doWithDB(PLATFORM_INFO_DB, async infoDb => {
+  await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => {
-    const keys = [dbUser._id, dbUser.email]
+    const keys = [dbUser._id!, dbUser.email]
     const userDocs = await infoDb.allDocs({
       keys,
       include_docs: true,
     })
-    const toDelete = userDocs.rows.map(row => {
+    const toDelete = userDocs.rows.map((row: any) => {
       return {
         ...row.doc,
         _deleted: true,
@@ -37,18 +41,18 @@ exports.removeUserFromInfoDB = async dbUser => {
   })
 }
 
-const removeUsersFromInfoDB = async tenantId => {
+async function removeUsersFromInfoDB(tenantId: string) {
-  return doWithGlobalDB(tenantId, async db => {
+  return doWithGlobalDB(tenantId, async (db: any) => {
     try {
       const allUsers = await db.allDocs(
         getGlobalUserParams(null, {
           include_docs: true,
         })
       )
-      await doWithDB(PLATFORM_INFO_DB, async infoDb => {
+      await doWithDB(PLATFORM_INFO_DB, async (infoDb: any) => {
-        const allEmails = allUsers.rows.map(row => row.doc.email)
+        const allEmails = allUsers.rows.map((row: any) => row.doc.email)
         // get the id docs
-        let keys = allUsers.rows.map(row => row.id)
+        let keys = allUsers.rows.map((row: any) => row.id)
         // and the email docs
         keys = keys.concat(allEmails)
         // retrieve the docs and delete them
@@ -56,7 +60,7 @@ const removeUsersFromInfoDB = async tenantId => {
           keys,
           include_docs: true,
         })
-        const toDelete = userDocs.rows.map(row => {
+        const toDelete = userDocs.rows.map((row: any) => {
           return {
             ...row.doc,
             _deleted: true,
@@ -71,8 +75,8 @@ const removeUsersFromInfoDB = async tenantId => {
   })
 }
 
-const removeGlobalDB = async tenantId => {
+async function removeGlobalDB(tenantId: string) {
-  return doWithGlobalDB(tenantId, async db => {
+  return doWithGlobalDB(tenantId, async (db: Database) => {
     try {
       await db.destroy()
     } catch (err) {
@@ -82,11 +86,11 @@ const removeGlobalDB = async tenantId => {
   })
 }
 
-const removeTenantApps = async tenantId => {
+async function removeTenantApps(tenantId: string) {
   try {
-    const apps = await getAllApps({ all: true })
+    const apps = (await getAllApps({ all: true })) as App[]
     const destroyPromises = apps.map(app =>
-      doWithDB(app.appId, db => db.destroy())
+      doWithDB(app.appId, (db: Database) => db.destroy())
     )
     await Promise.allSettled(destroyPromises)
   } catch (err) {
@@ -96,7 +100,7 @@ const removeTenantApps = async tenantId => {
 }
 
 // can't live in tenancy package due to circular dependency on db/utils
-exports.deleteTenant = async tenantId => {
+export async function deleteTenant(tenantId: string) {
   await removeTenantFromInfoDB(tenantId)
   await removeUsersFromInfoDB(tenantId)
   await removeGlobalDB(tenantId)

packages/backend-core/src/context/index.ts

@@ -1,47 +1,32 @@
 import env from "../environment"
-import { SEPARATOR, DocumentType } from "../db/constants"
-import cls from "./FunctionContext"
-import { dangerousGetDB, closeDB } from "../db"
-import { baseGlobalDBName } from "../db/tenancy"
-import { IdentityContext } from "@budibase/types"
-import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants"
-import { ContextKey } from "./constants"
-import PouchDB from "pouchdb"
 import {
-  updateUsing,
+  SEPARATOR,
-  closeWithUsing,
+  DocumentType,
-  setAppTenantId,
+  getDevelopmentAppID,
-  setIdentity,
+  getProdAppID,
-  closeAppDBs,
+  baseGlobalDBName,
-  getContextDB,
+  getDB,
-} from "./utils"
+} from "../db"
-
+import Context from "./Context"
+import { IdentityContext, Database } from "@budibase/types"
+import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants"
+import { ContextMap } from "./constants"
 export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID
 
 // some test cases call functions directly, need to
 // store an app ID to pretend there is a context
 let TEST_APP_ID: string | null = null
 
-export const closeTenancy = async () => {
+export function isMultiTenant() {
-  try {
+  return env.MULTI_TENANCY
-    if (env.USE_COUCH) {
-      const db = getGlobalDB()
-      await closeDB(db)
-    }
-  } catch (err) {
-    // no DB found - skip closing
-    return
-  }
-  // clear from context now that database is closed/task is finished
-  cls.setOnContext(ContextKey.TENANT_ID, null)
-  cls.setOnContext(ContextKey.GLOBAL_DB, null)
 }
 
-// export const isDefaultTenant = () => {
+export function isTenantIdSet() {
-//   return getTenantId() === DEFAULT_TENANT_ID
+  const context = Context.get()
-// }
+  return !!context?.tenantId
+}
 
-export const isMultiTenant = () => {
+export function isTenancyEnabled() {
   return env.MULTI_TENANCY
 }
 
@@ -49,9 +34,9 @@ export const isMultiTenant = () => {
  * Given an app ID this will attempt to retrieve the tenant ID from it.
  * @return {null|string} The tenant ID found within the app ID.
  */
-export const getTenantIDFromAppID = (appId: string) => {
+export function getTenantIDFromAppID(appId: string) {
   if (!appId) {
-    return null
+    return undefined
   }
   if (!isMultiTenant()) {
     return DEFAULT_TENANT_ID
@@ -59,7 +44,7 @@ export const getTenantIDFromAppID = (appId: string) => {
   const split = appId.split(SEPARATOR)
   const hasDev = split[1] === DocumentType.DEV
   if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) {
-    return null
+    return undefined
   }
   if (hasDev) {
     return split[2]
@@ -68,127 +53,125 @@ export const getTenantIDFromAppID = (appId: string) => {
   }
 }
 
-export const doInContext = async (appId: string, task: any) => {
+function updateContext(updates: ContextMap) {
-  // gets the tenant ID from the app ID
+  let context: ContextMap
-  const tenantId = getTenantIDFromAppID(appId)
+  try {
-  return doInTenant(tenantId, async () => {
+    context = Context.get()
-    return doInAppContext(appId, async () => {
+  } catch (err) {
-      return task()
+    // no context, start empty
-    })
+    context = {}
-  })
+  }
+  context = {
+    ...context,
+    ...updates,
+  }
+  return context
 }
 
-export const doInTenant = (tenantId: string | null, task: any) => {
+async function newContext(updates: ContextMap, task: any) {
+  // see if there already is a context setup
+  let context: ContextMap = updateContext(updates)
+  return Context.run(context, task)
+}
+
+export async function doInContext(appId: string, task: any): Promise<any> {
+  const tenantId = getTenantIDFromAppID(appId)
+  return newContext(
+    {
+      tenantId,
+      appId,
+    },
+    task
+  )
+}
+
+export async function doInTenant(
+  tenantId: string | null,
+  task: any
+): Promise<any> {
   // make sure default always selected in single tenancy
   if (!env.MULTI_TENANCY) {
     tenantId = tenantId || DEFAULT_TENANT_ID
   }
-  // the internal function is so that we can re-use an existing
-  // context - don't want to close DB on a parent context
-  async function internal(opts = { existing: false }) {
-    // set the tenant id + global db if this is a new context
-    if (!opts.existing) {
-      updateTenantId(tenantId)
-    }
 
-    try {
+  const updates = tenantId ? { tenantId } : {}
-      // invoke the task
+  return newContext(updates, task)
-      return await task()
-    } finally {
-      await closeWithUsing(ContextKey.TENANCY_IN_USE, () => {
-        return closeTenancy()
-      })
-    }
-  }
-
-  const existing = cls.getFromContext(ContextKey.TENANT_ID) === tenantId
-  return updateUsing(ContextKey.TENANCY_IN_USE, existing, internal)
 }
 
-export const doInAppContext = (appId: string, task: any) => {
+export async function doInAppContext(appId: string, task: any): Promise<any> {
   if (!appId) {
     throw new Error("appId is required")
   }
 
-  const identity = getIdentity()
+  const tenantId = getTenantIDFromAppID(appId)
-
+  const updates: ContextMap = { appId }
-  // the internal function is so that we can re-use an existing
+  if (tenantId) {
-  // context - don't want to close DB on a parent context
+    updates.tenantId = tenantId
-  async function internal(opts = { existing: false }) {
-    // set the app tenant id
-    if (!opts.existing) {
-      setAppTenantId(appId)
-    }
-    // set the app ID
-    cls.setOnContext(ContextKey.APP_ID, appId)
-
-    // preserve the identity
-    if (identity) {
-      setIdentity(identity)
-    }
-    try {
-      // invoke the task
-      return await task()
-    } finally {
-      await closeWithUsing(ContextKey.APP_IN_USE, async () => {
-        await closeAppDBs()
-        await closeTenancy()
-      })
-    }
   }
-  const existing = cls.getFromContext(ContextKey.APP_ID) === appId
+  return newContext(updates, task)
-  return updateUsing(ContextKey.APP_IN_USE, existing, internal)
 }
 
-export const doInIdentityContext = (identity: IdentityContext, task: any) => {
+export async function doInIdentityContext(
+  identity: IdentityContext,
+  task: any
+): Promise<any> {
   if (!identity) {
     throw new Error("identity is required")
   }
 
-  async function internal(opts = { existing: false }) {
+  const context: ContextMap = {
-    if (!opts.existing) {
+    identity,
-      cls.setOnContext(ContextKey.IDENTITY, identity)
-      // set the tenant so that doInTenant will preserve identity
-      if (identity.tenantId) {
-        updateTenantId(identity.tenantId)
-      }
-    }
-
-    try {
-      // invoke the task
-      return await task()
-    } finally {
-      await closeWithUsing(ContextKey.IDENTITY_IN_USE, async () => {
-        setIdentity(null)
-        await closeTenancy()
-      })
-    }
   }
-
+  if (identity.tenantId) {
-  const existing = cls.getFromContext(ContextKey.IDENTITY)
+    context.tenantId = identity.tenantId
-  return updateUsing(ContextKey.IDENTITY_IN_USE, existing, internal)
+  }
+  return newContext(context, task)
 }
 
-export const getIdentity = (): IdentityContext | undefined => {
+export function getIdentity(): IdentityContext | undefined {
   try {
-    return cls.getFromContext(ContextKey.IDENTITY)
+    const context = Context.get()
+    return context?.identity
   } catch (e) {
     // do nothing - identity is not in context
   }
 }
 
-export const updateTenantId = (tenantId: string | null) => {
+export function getTenantId(): string {
-  cls.setOnContext(ContextKey.TENANT_ID, tenantId)
+  if (!isMultiTenant()) {
-  if (env.USE_COUCH) {
+    return DEFAULT_TENANT_ID
-    setGlobalDB(tenantId)
+  }
+  const context = Context.get()
+  const tenantId = context?.tenantId
+  if (!tenantId) {
+    throw new Error("Tenant id not found")
+  }
+  return tenantId
+}
+
+export function getAppId(): string | undefined {
+  const context = Context.get()
+  const foundId = context?.appId
+  if (!foundId && env.isTest() && TEST_APP_ID) {
+    return TEST_APP_ID
+  } else {
+    return foundId
   }
 }
 
-export const updateAppId = async (appId: string) => {
+export function updateTenantId(tenantId?: string) {
+  let context: ContextMap = updateContext({
+    tenantId,
+  })
+  Context.set(context)
+}
+
+export function updateAppId(appId: string) {
+  let context: ContextMap = updateContext({
+    appId,
+  })
   try {
-    // have to close first, before removing the databases from context
+    Context.set(context)
-    await closeAppDBs()
-    cls.setOnContext(ContextKey.APP_ID, appId)
   } catch (err) {
     if (env.isTest()) {
       TEST_APP_ID = appId
@@ -198,70 +181,43 @@ export const updateAppId = async (appId: string) => {
   }
 }
 
-export const setGlobalDB = (tenantId: string | null) => {
+export function getGlobalDB(): Database {
-  const dbName = baseGlobalDBName(tenantId)
+  const context = Context.get()
-  const db = dangerousGetDB(dbName)
+  if (!context || (env.MULTI_TENANCY && !context.tenantId)) {
-  cls.setOnContext(ContextKey.GLOBAL_DB, db)
-  return db
-}
-
-export const getGlobalDB = () => {
-  const db = cls.getFromContext(ContextKey.GLOBAL_DB)
-  if (!db) {
     throw new Error("Global DB not found")
   }
-  return db
+  return getDB(baseGlobalDBName(context?.tenantId))
-}
-
-export const isTenantIdSet = () => {
-  const tenantId = cls.getFromContext(ContextKey.TENANT_ID)
-  return !!tenantId
-}
-
-export const getTenantId = () => {
-  if (!isMultiTenant()) {
-    return DEFAULT_TENANT_ID
-  }
-  const tenantId = cls.getFromContext(ContextKey.TENANT_ID)
-  if (!tenantId) {
-    throw new Error("Tenant id not found")
-  }
-  return tenantId
-}
-
-export const getAppId = () => {
-  const foundId = cls.getFromContext(ContextKey.APP_ID)
-  if (!foundId && env.isTest() && TEST_APP_ID) {
-    return TEST_APP_ID
-  } else {
-    return foundId
-  }
-}
-
-export const isTenancyEnabled = () => {
-  return env.MULTI_TENANCY
 }
 
 /**
- * Opens the app database based on whatever the request
+ * Gets the app database based on whatever the request
  * contained, dev or prod.
  */
-export const getAppDB = (opts?: any) => {
+export function getAppDB(opts?: any): Database {
-  return getContextDB(ContextKey.CURRENT_DB, opts)
+  const appId = getAppId()
+  return getDB(appId, opts)
 }
 
 /**
  * This specifically gets the prod app ID, if the request
- * contained a development app ID, this will open the prod one.
+ * contained a development app ID, this will get the prod one.
  */
-export const getProdAppDB = (opts?: any) => {
+export function getProdAppDB(opts?: any): Database {
-  return getContextDB(ContextKey.PROD_DB, opts)
+  const appId = getAppId()
+  if (!appId) {
+    throw new Error("Unable to retrieve prod DB - no app ID.")
+  }
+  return getDB(getProdAppID(appId), opts)
 }
 
 /**
  * This specifically gets the dev app ID, if the request
- * contained a prod app ID, this will open the dev one.
+ * contained a prod app ID, this will get the dev one.
  */
-export const getDevAppDB = (opts?: any) => {
+export function getDevAppDB(opts?: any): Database {
-  return getContextDB(ContextKey.DEV_DB, opts)
+  const appId = getAppId()
+  if (!appId) {
+    throw new Error("Unable to retrieve dev DB - no app ID.")
+  }
+  return getDB(getDevelopmentAppID(appId), opts)
 }

packages/backend-core/src/context/tests/index.spec.js

@@ -1,18 +1,9 @@
-import "../../../tests/utilities/TestConfiguration"
+require("../../../tests")
-import * as context from ".."
+const context = require("../")
-import { DEFAULT_TENANT_ID } from "../../constants"
+const { DEFAULT_TENANT_ID } = require("../../constants")
-import env from "../../environment"
+const env = require("../../environment")
-
-// must use require to spy index file exports due to known issue in jest
-const dbUtils = require("../../db")
-jest.spyOn(dbUtils, "closeDB")
-jest.spyOn(dbUtils, "dangerousGetDB")
 
 describe("context", () => {
-  beforeEach(() => {
-    jest.clearAllMocks()
-  })
-
   describe("doInTenant", () => {
     describe("single-tenancy", () => {
       it("defaults to the default tenant", () => {
@@ -25,8 +16,6 @@ describe("context", () => {
           const db = context.getGlobalDB()
           expect(db.name).toBe("global-db")
         })
-        expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
-        expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
       })
     })
 
@@ -40,7 +29,7 @@ describe("context", () => {
           let error
           try {
             context.getTenantId()
-          } catch (e: any) {
+          } catch (e) {
             error = e
           }
           expect(error.message).toBe("Tenant id not found")
@@ -59,7 +48,7 @@ describe("context", () => {
           let error
           try {
             context.getGlobalDB()
-          } catch (e: any) {
+          } catch (e) {
             error = e
           }
           expect(error.message).toBe("Global DB not found")
@@ -85,8 +74,6 @@ describe("context", () => {
           const db = context.getGlobalDB()
           expect(db.name).toBe("test_global-db")
         })
-        expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
-        expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
       })
 
       it("sets the tenant id when nested with same tenant id", async () => {
@@ -121,10 +108,6 @@ describe("context", () => {
             })
           })
         })
-
-        // only 1 db is opened and closed
-        expect(dbUtils.dangerousGetDB).toHaveBeenCalledTimes(1)
-        expect(dbUtils.closeDB).toHaveBeenCalledTimes(1)
       })
 
       it("sets different tenant id inside another context", () => {

packages/backend-core/src/context/utils.ts

@@ -1,109 +0,0 @@
-import {
-  DEFAULT_TENANT_ID,
-  getAppId,
-  getTenantIDFromAppID,
-  updateTenantId,
-} from "./index"
-import cls from "./FunctionContext"
-import { IdentityContext } from "@budibase/types"
-import { ContextKey } from "./constants"
-import { dangerousGetDB, closeDB } from "../db"
-import { isEqual } from "lodash"
-import { getDevelopmentAppID, getProdAppID } from "../db/conversions"
-import env from "../environment"
-
-export async function updateUsing(
-  usingKey: string,
-  existing: boolean,
-  internal: (opts: { existing: boolean }) => Promise<any>
-) {
-  const using = cls.getFromContext(usingKey)
-  if (using && existing) {
-    cls.setOnContext(usingKey, using + 1)
-    return internal({ existing: true })
-  } else {
-    return cls.run(async () => {
-      cls.setOnContext(usingKey, 1)
-      return internal({ existing: false })
-    })
-  }
-}
-
-export async function closeWithUsing(
-  usingKey: string,
-  closeFn: () => Promise<any>
-) {
-  const using = cls.getFromContext(usingKey)
-  if (!using || using <= 1) {
-    await closeFn()
-  } else {
-    cls.setOnContext(usingKey, using - 1)
-  }
-}
-
-export const setAppTenantId = (appId: string) => {
-  const appTenantId = getTenantIDFromAppID(appId) || DEFAULT_TENANT_ID
-  updateTenantId(appTenantId)
-}
-
-export const setIdentity = (identity: IdentityContext | null) => {
-  cls.setOnContext(ContextKey.IDENTITY, identity)
-}
-
-// this function makes sure the PouchDB objects are closed and
-// fully deleted when finished - this protects against memory leaks
-export async function closeAppDBs() {
-  const dbKeys = [ContextKey.CURRENT_DB, ContextKey.PROD_DB, ContextKey.DEV_DB]
-  for (let dbKey of dbKeys) {
-    const db = cls.getFromContext(dbKey)
-    if (!db) {
-      continue
-    }
-    await closeDB(db)
-    // clear the DB from context, incase someone tries to use it again
-    cls.setOnContext(dbKey, null)
-  }
-  // clear the app ID now that the databases are closed
-  if (cls.getFromContext(ContextKey.APP_ID)) {
-    cls.setOnContext(ContextKey.APP_ID, null)
-  }
-  if (cls.getFromContext(ContextKey.DB_OPTS)) {
-    cls.setOnContext(ContextKey.DB_OPTS, null)
-  }
-}
-
-export function getContextDB(key: string, opts: any) {
-  const dbOptsKey = `${key}${ContextKey.DB_OPTS}`
-  let storedOpts = cls.getFromContext(dbOptsKey)
-  let db = cls.getFromContext(key)
-  if (db && isEqual(opts, storedOpts)) {
-    return db
-  }
-
-  const appId = getAppId()
-  let toUseAppId
-
-  switch (key) {
-    case ContextKey.CURRENT_DB:
-      toUseAppId = appId
-      break
-    case ContextKey.PROD_DB:
-      toUseAppId = getProdAppID(appId)
-      break
-    case ContextKey.DEV_DB:
-      toUseAppId = getDevelopmentAppID(appId)
-      break
-  }
-  db = dangerousGetDB(toUseAppId, opts)
-  try {
-    cls.setOnContext(key, db)
-    if (opts) {
-      cls.setOnContext(dbOptsKey, opts)
-    }
-  } catch (err) {
-    if (!env.isTest()) {
-      throw err
-    }
-  }
-  return db
-}

packages/backend-core/src/db/Replication.ts

@@ -1,4 +1,4 @@
-import { dangerousGetDB, closeDB } from "."
+import { getPouchDB, closePouchDB } from "./couch/pouchDB"
 import { DocumentType } from "./constants"
 
 class Replication {
@@ -12,12 +12,12 @@ class Replication {
    * @param {String} target - the DB you want to replicate to, or rollback from
    */
   constructor({ source, target }: any) {
-    this.source = dangerousGetDB(source)
+    this.source = getPouchDB(source)
-    this.target = dangerousGetDB(target)
+    this.target = getPouchDB(target)
   }
 
   close() {
-    return Promise.all([closeDB(this.source), closeDB(this.target)])
+    return Promise.all([closePouchDB(this.source), closePouchDB(this.target)])
   }
 
   promisify(operation: any, opts = {}) {
@@ -68,7 +68,7 @@ class Replication {
   async rollback() {
     await this.target.destroy()
     // Recreate the DB again
-    this.target = dangerousGetDB(this.target.name)
+    this.target = getPouchDB(this.target.name)
     // take the opportunity to remove deleted tombstones
     await this.replicate()
   }

packages/backend-core/src/db/conversions.ts

@@ -1,32 +1,33 @@
+import { APP_DEV_PREFIX, APP_PREFIX } from "./constants"
+import { App } from "@budibase/types"
 const NO_APP_ERROR = "No app provided"
-const { APP_DEV_PREFIX, APP_PREFIX } = require("./constants")
 
-exports.isDevAppID = appId => {
+export function isDevAppID(appId?: string) {
   if (!appId) {
     throw NO_APP_ERROR
   }
   return appId.startsWith(APP_DEV_PREFIX)
 }
 
-exports.isProdAppID = appId => {
+export function isProdAppID(appId?: string) {
   if (!appId) {
     throw NO_APP_ERROR
   }
-  return appId.startsWith(APP_PREFIX) && !exports.isDevAppID(appId)
+  return appId.startsWith(APP_PREFIX) && !isDevAppID(appId)
 }
 
-exports.isDevApp = app => {
+export function isDevApp(app: App) {
   if (!app) {
     throw NO_APP_ERROR
   }
-  return exports.isDevAppID(app.appId)
+  return isDevAppID(app.appId)
 }
 
 /**
  * Generates a development app ID from a real app ID.
  * @returns {string} the dev app ID which can be used for dev database.
  */
-exports.getDevelopmentAppID = appId => {
+export function getDevelopmentAppID(appId: string) {
   if (!appId || appId.startsWith(APP_DEV_PREFIX)) {
     return appId
   }
@@ -36,12 +37,12 @@ exports.getDevelopmentAppID = appId => {
   const rest = split.join(APP_PREFIX)
   return `${APP_DEV_PREFIX}${rest}`
 }
-exports.getDevAppID = exports.getDevelopmentAppID
+export const getDevAppID = getDevelopmentAppID
 
 /**
  * Convert a development app ID to a deployed app ID.
  */
-exports.getProdAppID = appId => {
+export function getProdAppID(appId: string) {
   if (!appId || !appId.startsWith(APP_DEV_PREFIX)) {
     return appId
   }
@@ -52,7 +53,7 @@ exports.getProdAppID = appId => {
   return `${APP_PREFIX}${rest}`
 }
 
-exports.extractAppUUID = id => {
+export function extractAppUUID(id: string) {
   const split = id?.split("_") || []
   return split.length ? split[split.length - 1] : null
 }

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -0,0 +1,179 @@
+import Nano from "nano"
+import {
+  AllDocsResponse,
+  AnyDocument,
+  Database,
+  DatabaseOpts,
+  DatabaseQueryOpts,
+  DatabasePutOpts,
+} from "@budibase/types"
+import { getCouchInfo } from "./connections"
+import { directCouchCall } from "./utils"
+import { getPouchDB } from "./pouchDB"
+
+export class DatabaseImpl implements Database {
+  public readonly name: string
+  private static nano: Nano.ServerScope
+  private readonly pouchOpts: DatabaseOpts
+
+  constructor(dbName?: string, opts?: DatabaseOpts) {
+    if (dbName == null) {
+      throw new Error("Database name cannot be undefined.")
+    }
+    this.name = dbName
+    this.pouchOpts = opts || {}
+    if (!DatabaseImpl.nano) {
+      DatabaseImpl.init()
+    }
+  }
+
+  static init() {
+    const couchInfo = getCouchInfo()
+    DatabaseImpl.nano = Nano({
+      url: couchInfo.url,
+      requestDefaults: {
+        headers: {
+          Authorization: couchInfo.cookie,
+        },
+      },
+      parseUrl: false,
+    })
+  }
+
+  async exists() {
+    let response = await directCouchCall(`/${this.name}`, "HEAD")
+    return response.status === 200
+  }
+
+  async checkSetup() {
+    let shouldCreate = !this.pouchOpts?.skip_setup
+    // check exists in a lightweight fashion
+    let exists = await this.exists()
+    if (!shouldCreate && !exists) {
+      throw new Error("DB does not exist")
+    }
+    if (!exists) {
+      await DatabaseImpl.nano.db.create(this.name)
+    }
+    return DatabaseImpl.nano.db.use(this.name)
+  }
+
+  private async updateOutput(fnc: any) {
+    try {
+      return await fnc()
+    } catch (err: any) {
+      if (err.statusCode) {
+        err.status = err.statusCode
+      }
+      throw err
+    }
+  }
+
+  async get<T>(id?: string): Promise<T | any> {
+    const db = await this.checkSetup()
+    if (!id) {
+      throw new Error("Unable to get doc without a valid _id.")
+    }
+    return this.updateOutput(() => db.get(id))
+  }
+
+  async remove(id?: string, rev?: string) {
+    const db = await this.checkSetup()
+    if (!id || !rev) {
+      throw new Error("Unable to remove doc without a valid _id and _rev.")
+    }
+    return this.updateOutput(() => db.destroy(id, rev))
+  }
+
+  async put(document: AnyDocument, opts?: DatabasePutOpts) {
+    if (!document._id) {
+      throw new Error("Cannot store document without _id field.")
+    }
+    const db = await this.checkSetup()
+    if (!document.createdAt) {
+      document.createdAt = new Date().toISOString()
+    }
+    document.updatedAt = new Date().toISOString()
+    if (opts?.force && document._id) {
+      try {
+        const existing = await this.get(document._id)
+        if (existing) {
+          document._rev = existing._rev
+        }
+      } catch (err: any) {
+        if (err.status !== 404) {
+          throw err
+        }
+      }
+    }
+    return this.updateOutput(() => db.insert(document))
+  }
+
+  async bulkDocs(documents: AnyDocument[]) {
+    const db = await this.checkSetup()
+    return this.updateOutput(() => db.bulk({ docs: documents }))
+  }
+
+  async allDocs<T>(params: DatabaseQueryOpts): Promise<AllDocsResponse<T>> {
+    const db = await this.checkSetup()
+    return this.updateOutput(() => db.list(params))
+  }
+
+  async query<T>(
+    viewName: string,
+    params: DatabaseQueryOpts
+  ): Promise<AllDocsResponse<T>> {
+    const db = await this.checkSetup()
+    const [database, view] = viewName.split("/")
+    return this.updateOutput(() => db.view(database, view, params))
+  }
+
+  async destroy() {
+    try {
+      await DatabaseImpl.nano.db.destroy(this.name)
+    } catch (err: any) {
+      // didn't exist, don't worry
+      if (err.statusCode === 404) {
+        return
+      } else {
+        throw { ...err, status: err.statusCode }
+      }
+    }
+  }
+
+  async compact() {
+    const db = await this.checkSetup()
+    return this.updateOutput(() => db.compact())
+  }
+
+  private doWithPouchDB(func: string) {
+    const dbName = this.name
+    return async (args: any[]) => {
+      const pouch = getPouchDB(dbName)
+      // @ts-ignore
+      return pouch[func](...args)
+    }
+  }
+
+  // All below functions are in-frequently called, just utilise PouchDB
+  // for them as it implements them better than we can
+  async dump(...args: any[]) {
+    return this.doWithPouchDB("dump")(args)
+  }
+
+  async load(...args: any[]) {
+    return this.doWithPouchDB("load")(args)
+  }
+
+  async createIndex(...args: any[]) {
+    return this.doWithPouchDB("createIndex")(args)
+  }
+
+  async deleteIndex(...args: any[]) {
+    return this.doWithPouchDB("createIndex")(args)
+  }
+
+  async getIndexes(...args: any[]) {
+    return this.doWithPouchDB("createIndex")(args)
+  }
+}

packages/backend-core/src/db/couch/connections.ts

@@ -1,5 +1,37 @@
-import PouchDB from "pouchdb"
+import env from "../../environment"
-import env from "../environment"
+
+export const getCouchInfo = () => {
+  const urlInfo = getUrlInfo()
+  let username
+  let password
+  if (env.COUCH_DB_USERNAME) {
+    // set from env
+    username = env.COUCH_DB_USERNAME
+  } else if (urlInfo.auth.username) {
+    // set from url
+    username = urlInfo.auth.username
+  } else if (!env.isTest()) {
+    throw new Error("CouchDB username not set")
+  }
+  if (env.COUCH_DB_PASSWORD) {
+    // set from env
+    password = env.COUCH_DB_PASSWORD
+  } else if (urlInfo.auth.password) {
+    // set from url
+    password = urlInfo.auth.password
+  } else if (!env.isTest()) {
+    throw new Error("CouchDB password not set")
+  }
+  const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
+  return {
+    url: urlInfo.url!,
+    auth: {
+      username: username,
+      password: password,
+    },
+    cookie: `Basic ${authCookie}`,
+  }
+}
 
 export const getUrlInfo = (url = env.COUCH_DB_URL) => {
   let cleanUrl, username, password, host
@@ -43,85 +75,3 @@ export const getUrlInfo = (url = env.COUCH_DB_URL) => {
     },
   }
 }
-
-export const getCouchInfo = () => {
-  const urlInfo = getUrlInfo()
-  let username
-  let password
-  if (env.COUCH_DB_USERNAME) {
-    // set from env
-    username = env.COUCH_DB_USERNAME
-  } else if (urlInfo.auth.username) {
-    // set from url
-    username = urlInfo.auth.username
-  } else if (!env.isTest()) {
-    throw new Error("CouchDB username not set")
-  }
-  if (env.COUCH_DB_PASSWORD) {
-    // set from env
-    password = env.COUCH_DB_PASSWORD
-  } else if (urlInfo.auth.password) {
-    // set from url
-    password = urlInfo.auth.password
-  } else if (!env.isTest()) {
-    throw new Error("CouchDB password not set")
-  }
-  const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
-  return {
-    url: urlInfo.url,
-    auth: {
-      username: username,
-      password: password,
-    },
-    cookie: `Basic ${authCookie}`,
-  }
-}
-
-/**
- * Return a constructor for PouchDB.
- * This should be rarely used outside of the main application config.
- * Exposed for exceptional cases such as in-memory views.
- */
-export const getPouch = (opts: any = {}) => {
-  let { url, cookie } = getCouchInfo()
-  let POUCH_DB_DEFAULTS = {
-    prefix: url,
-    fetch: (url: string, opts: any) => {
-      // use a specific authorization cookie - be very explicit about how we authenticate
-      opts.headers.set("Authorization", cookie)
-      return PouchDB.fetch(url, opts)
-    },
-  }
-
-  if (opts.inMemory) {
-    const inMemory = require("pouchdb-adapter-memory")
-    PouchDB.plugin(inMemory)
-    POUCH_DB_DEFAULTS = {
-      prefix: undefined,
-      // @ts-ignore
-      adapter: "memory",
-    }
-  }
-
-  if (opts.onDisk) {
-    POUCH_DB_DEFAULTS = {
-      prefix: undefined,
-      // @ts-ignore
-      adapter: "leveldb",
-    }
-  }
-
-  if (opts.replication) {
-    const replicationStream = require("pouchdb-replication-stream")
-    PouchDB.plugin(replicationStream.plugin)
-    // @ts-ignore
-    PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
-  }
-
-  if (opts.find) {
-    const find = require("pouchdb-find")
-    PouchDB.plugin(find)
-  }
-
-  return PouchDB.defaults(POUCH_DB_DEFAULTS)
-}

packages/backend-core/src/db/couch/index.ts

@@ -0,0 +1,4 @@
+export * from "./connections"
+export * from "./DatabaseImpl"
+export * from "./utils"
+export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB"

packages/backend-core/src/db/couch/pouchDB.ts

@@ -0,0 +1,97 @@
+import PouchDB from "pouchdb"
+import env from "../../environment"
+import { PouchOptions } from "@budibase/types"
+import { getCouchInfo } from "./connections"
+
+let Pouch: any
+let initialised = false
+
+/**
+ * Return a constructor for PouchDB.
+ * This should be rarely used outside of the main application config.
+ * Exposed for exceptional cases such as in-memory views.
+ */
+export const getPouch = (opts: PouchOptions = {}) => {
+  let { url, cookie } = getCouchInfo()
+  let POUCH_DB_DEFAULTS = {
+    prefix: url,
+    fetch: (url: string, opts: any) => {
+      // use a specific authorization cookie - be very explicit about how we authenticate
+      opts.headers.set("Authorization", cookie)
+      return PouchDB.fetch(url, opts)
+    },
+  }
+
+  if (opts.inMemory) {
+    const inMemory = require("pouchdb-adapter-memory")
+    PouchDB.plugin(inMemory)
+    POUCH_DB_DEFAULTS = {
+      // @ts-ignore
+      adapter: "memory",
+    }
+  }
+
+  if (opts.onDisk) {
+    POUCH_DB_DEFAULTS = {
+      // @ts-ignore
+      adapter: "leveldb",
+    }
+  }
+
+  if (opts.replication) {
+    const replicationStream = require("pouchdb-replication-stream")
+    PouchDB.plugin(replicationStream.plugin)
+    // @ts-ignore
+    PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
+  }
+
+  if (opts.find) {
+    const find = require("pouchdb-find")
+    PouchDB.plugin(find)
+  }
+
+  return PouchDB.defaults(POUCH_DB_DEFAULTS)
+}
+
+export function init(opts?: PouchOptions) {
+  Pouch = getPouch(opts)
+  initialised = true
+}
+
+const checkInitialised = () => {
+  if (!initialised) {
+    throw new Error("init has not been called")
+  }
+}
+
+export function getPouchDB(dbName: string, opts?: any): PouchDB.Database {
+  checkInitialised()
+  const db = new Pouch(dbName, opts)
+  const dbPut = db.put
+  db.put = async (doc: any, options = {}) => {
+    if (!doc.createdAt) {
+      doc.createdAt = new Date().toISOString()
+    }
+    doc.updatedAt = new Date().toISOString()
+    return dbPut(doc, options)
+  }
+  db.exists = async () => {
+    const info = await db.info()
+    return !info.error
+  }
+  return db
+}
+
+// use this function if you have called getPouchDB - close
+// the databases you've opened once finished
+export async function closePouchDB(db: PouchDB.Database) {
+  if (!db || env.isTest()) {
+    return
+  }
+  try {
+    // specifically await so that if there is an error, it can be ignored
+    return await db.close()
+  } catch (err) {
+    // ignore error, already closed
+  }
+}

packages/backend-core/src/db/couch/utils.ts

@@ -0,0 +1,36 @@
+import { getCouchInfo } from "./connections"
+import fetch from "node-fetch"
+import { checkSlashesInUrl } from "../../helpers"
+
+export async function directCouchCall(
+  path: string,
+  method: string = "GET",
+  body?: any
+) {
+  let { url, cookie } = getCouchInfo()
+  const couchUrl = `${url}/${path}`
+  const params: any = {
+    method: method,
+    headers: {
+      Authorization: cookie,
+    },
+  }
+  if (body && method !== "GET") {
+    params.body = JSON.stringify(body)
+    params.headers["Content-Type"] = "application/json"
+  }
+  return await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params)
+}
+
+export async function directCouchQuery(
+  path: string,
+  method: string = "GET",
+  body?: any
+) {
+  const response = await directCouchCall(path, method, body)
+  if (response.status < 300) {
+    return await response.json()
+  } else {
+    throw "Cannot connect to CouchDB instance"
+  }
+}

packages/backend-core/src/db/db.ts

@@ -0,0 +1,46 @@
+import env from "../environment"
+import { directCouchQuery, getPouchDB } from "./couch"
+import { CouchFindOptions, Database } from "@budibase/types"
+import { DatabaseImpl } from "../db"
+
+const dbList = new Set()
+
+export function getDB(dbName?: string, opts?: any): Database {
+  // TODO: once using the test image, need to remove this
+  if (env.isTest()) {
+    dbList.add(dbName)
+    // @ts-ignore
+    return getPouchDB(dbName, opts)
+  }
+  return new DatabaseImpl(dbName, opts)
+}
+
+// we have to use a callback for this so that we can close
+// the DB when we're done, without this manual requests would
+// need to close the database when done with it to avoid memory leaks
+export async function doWithDB(dbName: string, cb: any, opts = {}) {
+  const db = getDB(dbName, opts)
+  // need this to be async so that we can correctly close DB after all
+  // async operations have been completed
+  return await cb(db)
+}
+
+export function allDbs() {
+  if (!env.isTest()) {
+    throw new Error("Cannot be used outside test environment.")
+  }
+  return [...dbList]
+}
+
+export async function directCouchAllDbs(queryString?: string) {
+  let couchPath = "/_all_dbs"
+  if (queryString) {
+    couchPath += `?${queryString}`
+  }
+  return await directCouchQuery(couchPath)
+}
+
+export async function directCouchFind(dbName: string, opts: CouchFindOptions) {
+  const json = await directCouchQuery(`${dbName}/_find`, "POST", opts)
+  return { rows: json.docs, bookmark: json.bookmark }
+}

packages/backend-core/src/db/index.ts

@@ -1,133 +1,7 @@
-import * as pouch from "./pouch"
+export * from "./couch"
-import env from "../environment"
+export * from "./db"
-import { checkSlashesInUrl } from "../helpers"
+export * from "./utils"
-import fetch from "node-fetch"
+export * from "./views"
-import { PouchOptions, CouchFindOptions } from "@budibase/types"
+export * from "./constants"
-import PouchDB from "pouchdb"
+export * from "./conversions"
-
+export * from "./tenancy"
-const openDbs: string[] = []
-let Pouch: any
-let initialised = false
-const dbList = new Set()
-
-if (env.MEMORY_LEAK_CHECK) {
-  setInterval(() => {
-    console.log("--- OPEN DBS ---")
-    console.log(openDbs)
-  }, 5000)
-}
-
-const put =
-  (dbPut: any) =>
-  async (doc: any, options = {}) => {
-    if (!doc.createdAt) {
-      doc.createdAt = new Date().toISOString()
-    }
-    doc.updatedAt = new Date().toISOString()
-    return dbPut(doc, options)
-  }
-
-const checkInitialised = () => {
-  if (!initialised) {
-    throw new Error("init has not been called")
-  }
-}
-
-export async function init(opts?: PouchOptions) {
-  Pouch = pouch.getPouch(opts)
-  initialised = true
-}
-
-// NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION
-// this function is prone to leaks, should only be used
-// in situations that using the function doWithDB does not work
-export function dangerousGetDB(dbName: string, opts?: any): PouchDB.Database {
-  checkInitialised()
-  if (env.isTest()) {
-    dbList.add(dbName)
-  }
-  const db = new Pouch(dbName, opts)
-  if (env.MEMORY_LEAK_CHECK) {
-    openDbs.push(db.name)
-  }
-  const dbPut = db.put
-  db.put = put(dbPut)
-  return db
-}
-
-// use this function if you have called dangerousGetDB - close
-// the databases you've opened once finished
-export async function closeDB(db: PouchDB.Database) {
-  if (!db || env.isTest()) {
-    return
-  }
-  if (env.MEMORY_LEAK_CHECK) {
-    openDbs.splice(openDbs.indexOf(db.name), 1)
-  }
-  try {
-    // specifically await so that if there is an error, it can be ignored
-    return await db.close()
-  } catch (err) {
-    // ignore error, already closed
-  }
-}
-
-// we have to use a callback for this so that we can close
-// the DB when we're done, without this manual requests would
-// need to close the database when done with it to avoid memory leaks
-export async function doWithDB(dbName: string, cb: any, opts = {}) {
-  const db = dangerousGetDB(dbName, opts)
-  // need this to be async so that we can correctly close DB after all
-  // async operations have been completed
-  try {
-    return await cb(db)
-  } finally {
-    await closeDB(db)
-  }
-}
-
-export function allDbs() {
-  if (!env.isTest()) {
-    throw new Error("Cannot be used outside test environment.")
-  }
-  checkInitialised()
-  return [...dbList]
-}
-
-export async function directCouchQuery(
-  path: string,
-  method: string = "GET",
-  body?: any
-) {
-  let { url, cookie } = pouch.getCouchInfo()
-  const couchUrl = `${url}/${path}`
-  const params: any = {
-    method: method,
-    headers: {
-      Authorization: cookie,
-    },
-  }
-  if (body && method !== "GET") {
-    params.body = JSON.stringify(body)
-    params.headers["Content-Type"] = "application/json"
-  }
-  const response = await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params)
-  if (response.status < 300) {
-    return await response.json()
-  } else {
-    throw "Cannot connect to CouchDB instance"
-  }
-}
-
-export async function directCouchAllDbs(queryString?: string) {
-  let couchPath = "/_all_dbs"
-  if (queryString) {
-    couchPath += `?${queryString}`
-  }
-  return await directCouchQuery(couchPath)
-}
-
-export async function directCouchFind(dbName: string, opts: CouchFindOptions) {
-  const json = await directCouchQuery(`${dbName}/_find`, "POST", opts)
-  return { rows: json.docs, bookmark: json.bookmark }
-}

packages/backend-core/src/db/tests/index.spec.js

@@ -1,11 +1,11 @@
-require("../../../tests/utilities/TestConfiguration")
+require("../../../tests")
-const { dangerousGetDB } = require("../")
+const { getDB } = require("../")
 
 describe("db", () => { 
   
   describe("getDB", () => {
     it("returns a db", async () => {
-      const db = dangerousGetDB("test")
+      const db = getDB("test")
       expect(db).toBeDefined()
       expect(db._adapter).toBe("memory")
       expect(db.prefix).toBe("_pouch_")
@@ -13,7 +13,7 @@ describe("db", () => {
     })
 
     it("uses the custom put function", async () => {
-      const db = dangerousGetDB("test")
+      const db = getDB("test")
       let doc = { _id: "test" }
       await db.put(doc)
       doc = await db.get(doc._id)

packages/backend-core/src/db/tests/pouch.spec.js

@@ -1,5 +1,5 @@
-require("../../../tests/utilities/TestConfiguration")
+require("../../../tests")
-const getUrlInfo = require("../pouch").getUrlInfo
+const getUrlInfo = require("../couch").getUrlInfo
 
 describe("pouch", () => {
   describe("Couch DB URL parsing", () => {

packages/backend-core/src/db/tests/utils.spec.js

@@ -1,4 +1,4 @@
-require("../../../tests/utilities/TestConfiguration");
+require("../../../tests")
 const {
   generateAppID,
   getDevelopmentAppID,
@@ -8,8 +8,8 @@ const {
   getPlatformUrl,
   getScopedConfig
 } = require("../utils")
-const tenancy = require("../../tenancy");
+const tenancy = require("../../tenancy")
-const { Configs, DEFAULT_TENANT_ID } = require("../../constants");
+const { Config, DEFAULT_TENANT_ID } = require("../../constants")
 const env = require("../../environment")
 
 describe("utils", () => {
@@ -77,7 +77,7 @@ const setDbPlatformUrl = async () => {
   const db = tenancy.getGlobalDB()
   db.put({
     _id: "config_settings",
-    type: Configs.SETTINGS,
+    type: Config.SETTINGS,
     config: {
       platformUrl: DB_URL
     }
@@ -178,7 +178,7 @@ describe("getScopedConfig", () => {
       await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
         await setDbPlatformUrl()
         const db = tenancy.getGlobalDB()
-        const config = await getScopedConfig(db, { type: Configs.SETTINGS })
+        const config = await getScopedConfig(db, { type: Config.SETTINGS })
         expect(config.platformUrl).toBe(DB_URL)
       })
     })
@@ -186,7 +186,7 @@ describe("getScopedConfig", () => {
     it("returns the platform url without an existing config", async () => {
       await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
         const db = tenancy.getGlobalDB()
-        const config = await getScopedConfig(db, { type: Configs.SETTINGS })
+        const config = await getScopedConfig(db, { type: Config.SETTINGS })
         expect(config.platformUrl).toBe(DEFAULT_URL)
       })
     })

packages/backend-core/src/db/utils.ts

@@ -1,5 +1,5 @@
 import { newid } from "../hashing"
-import { DEFAULT_TENANT_ID, Configs } from "../constants"
+import { DEFAULT_TENANT_ID, Config } from "../constants"
 import env from "../environment"
 import {
   SEPARATOR,
@@ -10,11 +10,12 @@ import {
 } from "./constants"
 import { getTenantId, getGlobalDB } from "../context"
 import { getGlobalDBName } from "./tenancy"
-import { doWithDB, allDbs, directCouchAllDbs } from "./index"
+import { doWithDB, allDbs, directCouchAllDbs } from "./db"
 import { getAppMetadata } from "../cache/appMetadata"
 import { isDevApp, isDevAppID, getProdAppID } from "./conversions"
 import { APP_PREFIX } from "./constants"
 import * as events from "../events"
+import { App, Database } from "@budibase/types"
 
 export * from "./constants"
 export * from "./conversions"
@@ -25,7 +26,7 @@ export * from "./tenancy"
  * Generates a new app ID.
  * @returns {string} The new app ID which the app doc can be stored under.
  */
-export const generateAppID = (tenantId = null) => {
+export const generateAppID = (tenantId?: string | null) => {
   let id = APP_PREFIX
   if (tenantId) {
     id += `${tenantId}${SEPARATOR}`
@@ -250,11 +251,11 @@ export function generateRoleID(id: any) {
 /**
  * Gets parameters for retrieving a role, this is a utility function for the getDocParams function.
  */
-export function getRoleParams(roleId = null, otherProps = {}) {
+export function getRoleParams(roleId?: string | null, otherProps = {}) {
   return getDocParams(DocumentType.ROLE, roleId, otherProps)
 }
 
-export function getStartEndKeyURL(baseKey: any, tenantId = null) {
+export function getStartEndKeyURL(baseKey: any, tenantId?: string) {
   const tenancy = tenantId ? `${SEPARATOR}${tenantId}` : ""
   return `startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"`
 }
@@ -301,7 +302,12 @@ export async function getAllDbs(opts = { efficient: false }) {
  *
  * @return {Promise<object[]>} returns the app information document stored in each app database.
  */
-export async function getAllApps({ dev, all, idsOnly, efficient }: any = {}) {
+export async function getAllApps({
+  dev,
+  all,
+  idsOnly,
+  efficient,
+}: any = {}): Promise<App[] | string[]> {
   let tenantId = getTenantId()
   if (!env.MULTI_TENANCY && !tenantId) {
     tenantId = DEFAULT_TENANT_ID
@@ -373,35 +379,23 @@ export async function getAllApps({ dev, all, idsOnly, efficient }: any = {}) {
  * Utility function for getAllApps but filters to production apps only.
  */
 export async function getProdAppIDs() {
-  return (await getAllApps({ idsOnly: true })).filter(
+  const apps = (await getAllApps({ idsOnly: true })) as string[]
-    (id: any) => !isDevAppID(id)
+  return apps.filter((id: any) => !isDevAppID(id))
-  )
 }
 
 /**
  * Utility function for the inverse of above.
  */
 export async function getDevAppIDs() {
-  return (await getAllApps({ idsOnly: true })).filter((id: any) =>
+  const apps = (await getAllApps({ idsOnly: true })) as string[]
-    isDevAppID(id)
+  return apps.filter((id: any) => isDevAppID(id))
-  )
 }
 
 export async function dbExists(dbName: any) {
-  let exists = false
   return doWithDB(
     dbName,
-    async (db: any) => {
+    async (db: Database) => {
-      try {
+      return await db.exists()
-        // check if database exists
-        const info = await db.info()
-        if (info && !info.error) {
-          exists = true
-        }
-      } catch (err) {
-        exists = false
-      }
-      return exists
     },
     { skip_setup: true }
   )
@@ -500,7 +494,7 @@ export const getScopedFullConfig = async function (
   )[0]
 
   // custom logic for settings doc
-  if (type === Configs.SETTINGS) {
+  if (type === Config.SETTINGS) {
     if (scopedConfig && scopedConfig.doc) {
       // overrides affected by environment variables
       scopedConfig.doc.config.platformUrl = await getPlatformUrl({
@@ -539,7 +533,7 @@ export const getPlatformUrl = async (opts = { tenantAware: true }) => {
     // get the doc directly instead of with getScopedConfig to prevent loop
     let settings
     try {
-      settings = await db.get(generateConfigID({ type: Configs.SETTINGS }))
+      settings = await db.get(generateConfigID({ type: Config.SETTINGS }))
     } catch (e: any) {
       if (e.status !== 404) {
         throw e

packages/backend-core/src/db/views.ts

@@ -1,8 +1,8 @@
 import { DocumentType, ViewName, DeprecatedViews, SEPARATOR } from "./utils"
 import { getGlobalDB } from "../context"
-import PouchDB from "pouchdb"
 import { StaticDatabases } from "./constants"
 import { doWithDB } from "./"
+import { Database, DatabaseQueryOpts } from "@budibase/types"
 
 const DESIGN_DB = "_design/database"
 
@@ -19,7 +19,7 @@ interface DesignDocument {
   views: any
 }
 
-async function removeDeprecated(db: PouchDB.Database, viewName: ViewName) {
+async function removeDeprecated(db: Database, viewName: ViewName) {
   // @ts-ignore
   if (!DeprecatedViews[viewName]) {
     return
@@ -70,16 +70,13 @@ export const createAccountEmailView = async () => {
       emit(doc.email.toLowerCase(), doc._id)
     }
   }`
-  await doWithDB(
+  await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
-    StaticDatabases.PLATFORM_INFO.name,
+    await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL)
-    async (db: PouchDB.Database) => {
+  })
-      await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL)
-    }
-  )
 }
 
 export const createUserAppView = async () => {
-  const db = getGlobalDB() as PouchDB.Database
+  const db = getGlobalDB()
   const viewJs = `function(doc) {
     if (doc._id.startsWith("${DocumentType.USER}${SEPARATOR}") && doc.roles) {
       for (let prodAppId of Object.keys(doc.roles)) {
@@ -117,12 +114,9 @@ export const createPlatformUserView = async () => {
       emit(doc._id.toLowerCase(), doc._id)
     }
   }`
-  await doWithDB(
+  await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
-    StaticDatabases.PLATFORM_INFO.name,
+    await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
-    async (db: PouchDB.Database) => {
+  })
-      await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
-    }
-  )
 }
 
 export interface QueryViewOptions {
@@ -131,22 +125,24 @@ export interface QueryViewOptions {
 
 export const queryView = async <T>(
   viewName: ViewName,
-  params: PouchDB.Query.Options<T, T>,
+  params: DatabaseQueryOpts,
-  db: PouchDB.Database,
+  db: Database,
   createFunc: any,
   opts?: QueryViewOptions
 ): Promise<T[] | T | undefined> => {
   try {
-    let response = await db.query<T, T>(`database/${viewName}`, params)
+    let response = await db.query<T>(`database/${viewName}`, params)
     const rows = response.rows
-    const docs = rows.map(row => (params.include_docs ? row.doc : row.value))
+    const docs = rows.map((row: any) =>
+      params.include_docs ? row.doc : row.value
+    )
 
     // if arrayResponse has been requested, always return array regardless of length
     if (opts?.arrayResponse) {
-      return docs
+      return docs as T[]
     } else {
       // return the single document if there is only one
-      return docs.length <= 1 ? docs[0] : docs
+      return docs.length <= 1 ? (docs[0] as T) : (docs as T[])
     }
   } catch (err: any) {
     if (err != null && err.name === "not_found") {
@@ -161,7 +157,7 @@ export const queryView = async <T>(
 
 export const queryPlatformView = async <T>(
   viewName: ViewName,
-  params: PouchDB.Query.Options<T, T>,
+  params: DatabaseQueryOpts,
   opts?: QueryViewOptions
 ): Promise<T[] | T | undefined> => {
   const CreateFuncByName: any = {
@@ -169,19 +165,16 @@ export const queryPlatformView = async <T>(
     [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView,
   }
 
-  return doWithDB(
+  return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
-    StaticDatabases.PLATFORM_INFO.name,
+    const createFn = CreateFuncByName[viewName]
-    async (db: PouchDB.Database) => {
+    return queryView(viewName, params, db, createFn, opts)
-      const createFn = CreateFuncByName[viewName]
+  })
-      return queryView(viewName, params, db, createFn, opts)
-    }
-  )
 }
 
 export const queryGlobalView = async <T>(
   viewName: ViewName,
-  params: PouchDB.Query.Options<T, T>,
+  params: DatabaseQueryOpts,
-  db?: PouchDB.Database,
+  db?: Database,
   opts?: QueryViewOptions
 ): Promise<T[] | T | undefined> => {
   const CreateFuncByName: any = {
@@ -192,8 +185,8 @@ export const queryGlobalView = async <T>(
   }
   // can pass DB in if working with something specific
   if (!db) {
-    db = getGlobalDB() as PouchDB.Database
+    db = getGlobalDB()
   }
   const createFn = CreateFuncByName[viewName]
-  return queryView(viewName, params, db, createFn, opts)
+  return queryView(viewName, params, db!, createFn, opts)
 }

packages/backend-core/src/environment.ts

@@ -69,7 +69,6 @@ const env = {
   DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
   DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,
   SERVICE: process.env.SERVICE || "budibase",
-  MEMORY_LEAK_CHECK: process.env.MEMORY_LEAK_CHECK || false,
   LOG_LEVEL: process.env.LOG_LEVEL,
   SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD,
   DEPLOYMENT_ENVIRONMENT:

packages/backend-core/src/events/analytics.ts

@@ -1,7 +1,7 @@
 import env from "../environment"
 import tenancy from "../tenancy"
 import * as dbUtils from "../db/utils"
-import { Configs } from "../constants"
+import { Config } from "../constants"
 import { withCache, TTL, CacheKeys } from "../cache/generic"
 
 export const enabled = async () => {
@@ -45,9 +45,7 @@ const getSettingsDoc = async () => {
   const db = tenancy.getGlobalDB()
   let settings
   try {
-    settings = await db.get(
+    settings = await db.get(dbUtils.generateConfigID({ type: Config.SETTINGS }))
-      dbUtils.generateConfigID({ type: Configs.SETTINGS })
-    )
   } catch (e: any) {
     if (e.status !== 404) {
       throw e

packages/backend-core/src/events/identification.ts

@@ -19,7 +19,7 @@ import {
 } from "@budibase/types"
 import { processors } from "./processors"
 import * as dbUtils from "../db/utils"
-import { Configs } from "../constants"
+import { Config } from "../constants"
 import * as hashing from "../hashing"
 import * as installation from "../installation"
 import { withCache, TTL, CacheKeys } from "../cache/generic"
@@ -273,7 +273,7 @@ const getUniqueTenantId = async (tenantId: string): Promise<string> => {
     return withCache(CacheKeys.UNIQUE_TENANT_ID, TTL.ONE_DAY, async () => {
       const db = context.getGlobalDB()
       const config: SettingsConfig = await dbUtils.getScopedFullConfig(db, {
-        type: Configs.SETTINGS,
+        type: Config.SETTINGS,
       })
 
       let uniqueTenantId: string

packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts

@@ -1,4 +1,4 @@
-import "../../../../../tests/utilities/TestConfiguration"
+import "../../../../../tests"
 import PosthogProcessor from "../PosthogProcessor"
 import { Event, IdentityType, Hosting } from "@budibase/types"
 const tk = require("timekeeper")

packages/backend-core/src/index.ts

@@ -11,9 +11,9 @@ import env from "./environment"
 import tenancy from "./tenancy"
 import featureFlags from "./featureFlags"
 import * as sessions from "./security/sessions"
-import deprovisioning from "./context/deprovision"
+import * as deprovisioning from "./context/deprovision"
 import auth from "./auth"
-import constants from "./constants"
+import * as constants from "./constants"
 import * as dbConstants from "./db/constants"
 import * as logging from "./logging"
 import pino from "./pino"
@@ -21,9 +21,9 @@ import * as middleware from "./middleware"
 import plugins from "./plugin"
 import encryption from "./security/encryption"
 import * as queue from "./queue"
+import * as db from "./db"
 
 // mimic the outer package exports
-import * as db from "./pkg/db"
 import * as objectStore from "./pkg/objectStore"
 import * as utils from "./pkg/utils"
 import redis from "./pkg/redis"

packages/backend-core/src/middleware/authenticated.ts

@@ -1,11 +1,9 @@
-import { Cookies, Headers } from "../constants"
+import { Cookie, Header } from "../constants"
 import { getCookie, clearCookie, openJwt } from "../utils"
 import { getUser } from "../cache/user"
 import { getSession, updateSessionTTL } from "../security/sessions"
 import { buildMatcherRegex, matches } from "./matchers"
-import { SEPARATOR } from "../db/constants"
+import { SEPARATOR, queryGlobalView, ViewName } from "../db"
-import { ViewName } from "../db/utils"
-import { queryGlobalView } from "../db/views"
 import { getGlobalDB, doInTenant } from "../tenancy"
 import { decrypt } from "../security/encryption"
 const identity = require("../context/identity")
@@ -74,7 +72,7 @@ export = (
   const noAuthOptions = noAuthPatterns ? buildMatcherRegex(noAuthPatterns) : []
   return async (ctx: any, next: any) => {
     let publicEndpoint = false
-    const version = ctx.request.headers[Headers.API_VER]
+    const version = ctx.request.headers[Header.API_VER]
     // the path is not authenticated
     const found = matches(ctx, noAuthOptions)
     if (found) {
@@ -82,10 +80,10 @@ export = (
     }
     try {
       // check the actual user is authenticated first, try header or cookie
-      const headerToken = ctx.request.headers[Headers.TOKEN]
+      const headerToken = ctx.request.headers[Header.TOKEN]
-      const authCookie = getCookie(ctx, Cookies.Auth) || openJwt(headerToken)
+      const authCookie = getCookie(ctx, Cookie.Auth) || openJwt(headerToken)
-      const apiKey = ctx.request.headers[Headers.API_KEY]
+      const apiKey = ctx.request.headers[Header.API_KEY]
-      const tenantId = ctx.request.headers[Headers.TENANT_ID]
+      const tenantId = ctx.request.headers[Header.TENANT_ID]
       let authenticated = false,
         user = null,
         internal = false
@@ -116,7 +114,7 @@ export = (
           authenticated = false
           console.error("Auth Error", err?.message || err)
           // remove the cookie as the user does not exist anymore
-          clearCookie(ctx, Cookies.Auth)
+          clearCookie(ctx, Cookie.Auth)
         }
       }
       // this is an internal request, no user made it
@@ -140,7 +138,7 @@ export = (
         delete user.password
       }
       // be explicit
-      if (authenticated !== true) {
+      if (!authenticated) {
         authenticated = false
       }
       // isAuthenticated is a function, so use a variable to be able to check authed state
@@ -155,7 +153,7 @@ export = (
       console.error("Auth Error", err?.message || err)
       // invalid token, clear the cookie
       if (err && err.name === "JsonWebTokenError") {
-        clearCookie(ctx, Cookies.Auth)
+        clearCookie(ctx, Cookie.Auth)
       }
       // allow configuring for public access
       if ((opts && opts.publicAllowed) || publicEndpoint) {

packages/backend-core/src/middleware/csrf.js

@@ -1,4 +1,4 @@
-const { Headers } = require("../constants")
+const { Header } = require("../constants")
 const { buildMatcherRegex, matches } = require("./matchers")
 
 /**
@@ -68,7 +68,7 @@ module.exports = (opts = { noCsrfPatterns: [] }) => {
     }
 
     // reject if no token in request or mismatch
-    const requestToken = ctx.get(Headers.CSRF_TOKEN)
+    const requestToken = ctx.get(Header.CSRF_TOKEN)
     if (!requestToken || requestToken !== userToken) {
       ctx.throw(403, "Invalid CSRF token")
     }

packages/backend-core/src/middleware/internalApi.js

@@ -1,11 +1,11 @@
 const env = require("../environment")
-const { Headers } = require("../constants")
+const { Header } = require("../constants")
 
 /**
  * API Key only endpoint.
  */
 module.exports = async (ctx, next) => {
-  const apiKey = ctx.request.headers[Headers.API_KEY]
+  const apiKey = ctx.request.headers[Header.API_KEY]
   if (apiKey !== env.INTERNAL_API_KEY) {
     ctx.throw(403, "Unauthorized")
   }

packages/backend-core/src/middleware/matchers.ts

@@ -1,27 +1,34 @@
+import { BBContext, EndpointMatcher, RegexMatcher } from "@budibase/types"
+
 const PARAM_REGEX = /\/:(.*?)(\/.*)?$/g
 
-exports.buildMatcherRegex = patterns => {
+export const buildMatcherRegex = (
+  patterns: EndpointMatcher[]
+): RegexMatcher[] => {
   if (!patterns) {
     return []
   }
   return patterns.map(pattern => {
-    const isObj = typeof pattern === "object" && pattern.route
+    let route = pattern.route
-    const method = isObj ? pattern.method : "GET"
+    const method = pattern.method
     const strict = pattern.strict ? pattern.strict : false
-    let route = isObj ? pattern.route : pattern
 
+    // if there is a param in the route
+    // use a wildcard pattern
     const matches = route.match(PARAM_REGEX)
     if (matches) {
       for (let match of matches) {
-        const pattern = "/.*" + (match.endsWith("/") ? "/" : "")
+        const suffix = match.endsWith("/") ? "/" : ""
+        const pattern = "/.*" + suffix
         route = route.replace(match, pattern)
       }
     }
+
     return { regex: new RegExp(route), method, strict, route }
   })
 }
 
-exports.matches = (ctx, options) => {
+export const matches = (ctx: BBContext, options: RegexMatcher[]) => {
   return options.find(({ regex, method, strict, route }) => {
     let urlMatch
     if (strict) {

packages/backend-core/src/middleware/passport/datasource/google.js

@@ -1,6 +1,6 @@
 const google = require("../google")
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
-const { Cookies, Configs } = require("../../../constants")
+const { Cookie, Config } = require("../../../constants")
 const { clearCookie, getCookie } = require("../../../utils")
 const { getScopedConfig, getPlatformUrl } = require("../../../db/utils")
 const { doWithDB } = require("../../../db")
@@ -11,7 +11,7 @@ async function fetchGoogleCreds() {
   // try and get the config from the tenant
   const db = getGlobalDB()
   const googleConfig = await getScopedConfig(db, {
-    type: Configs.GOOGLE,
+    type: Config.GOOGLE,
   })
   // or fall back to env variables
   return (
@@ -47,7 +47,7 @@ async function postAuth(passport, ctx, next) {
   const platformUrl = await getPlatformUrl({ tenantAware: false })
 
   let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback`
-  const authStateCookie = getCookie(ctx, Cookies.DatasourceAuth)
+  const authStateCookie = getCookie(ctx, Cookie.DatasourceAuth)
 
   return passport.authenticate(
     new GoogleStrategy(
@@ -57,7 +57,7 @@ async function postAuth(passport, ctx, next) {
         callbackURL: callbackUrl,
       },
       (accessToken, refreshToken, profile, done) => {
-        clearCookie(ctx, Cookies.DatasourceAuth)
+        clearCookie(ctx, Cookie.DatasourceAuth)
         done(null, { accessToken, refreshToken })
       }
     ),

packages/backend-core/src/middleware/passport/google.js

@@ -1,7 +1,7 @@
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
 const { ssoCallbackUrl } = require("./utils")
 const { authenticateThirdParty } = require("./third-party-common")
-const { Configs } = require("../../../constants")
+const { Config } = require("../../../constants")
 
 const buildVerifyFn = saveUserFn => {
   return (accessToken, refreshToken, profile, done) => {
@@ -60,7 +60,7 @@ exports.strategyFactory = async function (config, callbackUrl, saveUserFn) {
 }
 
 exports.getCallbackUrl = async function (db, config) {
-  return ssoCallbackUrl(db, config, Configs.GOOGLE)
+  return ssoCallbackUrl(db, config, Config.GOOGLE)
 }
 
 // expose for testing

packages/backend-core/src/middleware/passport/jwt.js

@@ -1,11 +1,11 @@
-const { Cookies } = require("../../constants")
+const { Cookie } = require("../../constants")
 const env = require("../../environment")
 const { authError } = require("./utils")
 
 exports.options = {
   secretOrKey: env.JWT_SECRET,
   jwtFromRequest: function (ctx) {
-    return ctx.cookies.get(Cookies.Auth)
+    return ctx.cookies.get(Cookie.Auth)
   },
 }
 

packages/backend-core/src/middleware/passport/oidc.js

@@ -2,7 +2,7 @@ const fetch = require("node-fetch")
 const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy
 const { authenticateThirdParty } = require("./third-party-common")
 const { ssoCallbackUrl } = require("./utils")
-const { Configs } = require("../../../constants")
+const { Config } = require("../../../constants")
 
 const buildVerifyFn = saveUserFn => {
   /**
@@ -140,7 +140,7 @@ exports.fetchStrategyConfig = async function (enrichedConfig, callbackUrl) {
 }
 
 exports.getCallbackUrl = async function (db, config) {
-  return ssoCallbackUrl(db, config, Configs.OIDC)
+  return ssoCallbackUrl(db, config, Config.OIDC)
 }
 
 // expose for testing

packages/backend-core/src/middleware/passport/tests/oidc.spec.js

@@ -1,7 +1,6 @@
 // Mock data
-
+const mockFetch = require("node-fetch")
 const { data } = require("./utilities/mock-data")
-
 const issuer = "mockIssuer"
 const sub = "mockSub"
 const profile = {
@@ -39,8 +38,6 @@ describe("oidc", () => {
     const mockStrategy = require("@techpass/passport-openidconnect").Strategy
     
     // mock the request to retrieve the oidc configuration
-    jest.mock("node-fetch")
-    const mockFetch = require("node-fetch")
     mockFetch.mockReturnValue({
       ok: true,
       json: () => oidcConfigUrlResponse

packages/backend-core/src/middleware/passport/tests/third-party-common.spec.js

@@ -1,4 +1,4 @@
-require("../../../../tests/utilities/TestConfiguration")
+require("../../../../tests")
 const { authenticateThirdParty } = require("../third-party-common")
 const { data } = require("./utilities/mock-data")
 const { DEFAULT_TENANT_ID } = require("../../../constants")

packages/backend-core/src/middleware/passport/utils.js

@@ -1,6 +1,6 @@
 const { isMultiTenant, getTenantId } = require("../../tenancy")
 const { getScopedConfig } = require("../../db/utils")
-const { Configs } = require("../../constants")
+const { Config } = require("../../constants")
 
 /**
  * Utility to handle authentication errors.
@@ -24,7 +24,7 @@ exports.ssoCallbackUrl = async (db, config, type) => {
     return config.callbackURL
   }
   const publicConfig = await getScopedConfig(db, {
-    type: Configs.SETTINGS,
+    type: Config.SETTINGS,
   })
 
   let callbackUrl = `/api/global/auth`

packages/backend-core/src/middleware/tenancy.js

@@ -1,52 +0,0 @@
-const { doInTenant, isMultiTenant, DEFAULT_TENANT_ID } = require("../tenancy")
-const { buildMatcherRegex, matches } = require("./matchers")
-const { Headers } = require("../constants")
-
-const getTenantID = (ctx, opts = { allowQs: false, allowNoTenant: false }) => {
-  // exit early if not multi-tenant
-  if (!isMultiTenant()) {
-    return DEFAULT_TENANT_ID
-  }
-
-  let tenantId
-  const allowQs = opts && opts.allowQs
-  const allowNoTenant = opts && opts.allowNoTenant
-  const header = ctx.request.headers[Headers.TENANT_ID]
-  const user = ctx.user || {}
-  if (allowQs) {
-    const query = ctx.request.query || {}
-    tenantId = query.tenantId
-  }
-  // override query string (if allowed) by user, or header
-  // URL params cannot be used in a middleware, as they are
-  // processed later in the chain
-  tenantId = user.tenantId || header || tenantId
-
-  // Set the tenantId from the subdomain
-  if (!tenantId) {
-    tenantId = ctx.subdomains && ctx.subdomains[0]
-  }
-
-  if (!tenantId && !allowNoTenant) {
-    ctx.throw(403, "Tenant id not set")
-  }
-
-  return tenantId
-}
-
-module.exports = (
-  allowQueryStringPatterns,
-  noTenancyPatterns,
-  opts = { noTenancyRequired: false }
-) => {
-  const allowQsOptions = buildMatcherRegex(allowQueryStringPatterns)
-  const noTenancyOptions = buildMatcherRegex(noTenancyPatterns)
-
-  return async function (ctx, next) {
-    const allowNoTenant =
-      opts.noTenancyRequired || !!matches(ctx, noTenancyOptions)
-    const allowQs = !!matches(ctx, allowQsOptions)
-    const tenantId = getTenantID(ctx, { allowQs, allowNoTenant })
-    return doInTenant(tenantId, next)
-  }
-}

packages/backend-core/src/middleware/tenancy.ts

@@ -0,0 +1,37 @@
+import { doInTenant, getTenantIDFromCtx } from "../tenancy"
+import { buildMatcherRegex, matches } from "./matchers"
+import { Header } from "../constants"
+import {
+  BBContext,
+  EndpointMatcher,
+  GetTenantIdOptions,
+  TenantResolutionStrategy,
+} from "@budibase/types"
+
+const tenancy = (
+  allowQueryStringPatterns: EndpointMatcher[],
+  noTenancyPatterns: EndpointMatcher[],
+  opts = { noTenancyRequired: false }
+) => {
+  const allowQsOptions = buildMatcherRegex(allowQueryStringPatterns)
+  const noTenancyOptions = buildMatcherRegex(noTenancyPatterns)
+
+  return async function (ctx: BBContext, next: any) {
+    const allowNoTenant =
+      opts.noTenancyRequired || !!matches(ctx, noTenancyOptions)
+    const tenantOpts: GetTenantIdOptions = {
+      allowNoTenant,
+    }
+
+    const allowQs = !!matches(ctx, allowQsOptions)
+    if (!allowQs) {
+      tenantOpts.excludeStrategies = [TenantResolutionStrategy.QUERY]
+    }
+
+    const tenantId = getTenantIDFromCtx(ctx, tenantOpts)
+    ctx.set(Header.TENANT_ID, tenantId as string)
+    return doInTenant(tenantId, next)
+  }
+}
+
+export = tenancy

packages/backend-core/src/middleware/tests/matchers.spec.ts

@@ -0,0 +1,134 @@
+import * as matchers from "../matchers"
+import { structures } from "../../../tests"
+
+describe("matchers", () => {
+  it("matches by path and method", () => {
+    const pattern = [
+      {
+        route: "/api/tests",
+        method: "POST",
+      },
+    ]
+    const ctx = structures.koa.newContext()
+    ctx.request.url = "/api/tests"
+    ctx.request.method = "POST"
+
+    const built = matchers.buildMatcherRegex(pattern)
+
+    expect(!!matchers.matches(ctx, built)).toBe(true)
+  })
+
+  it("wildcards path", () => {
+    const pattern = [
+      {
+        route: "/api/tests",
+        method: "POST",
+      },
+    ]
+    const ctx = structures.koa.newContext()
+    ctx.request.url = "/api/tests/id/something/else"
+    ctx.request.method = "POST"
+
+    const built = matchers.buildMatcherRegex(pattern)
+
+    expect(!!matchers.matches(ctx, built)).toBe(true)
+  })
+
+  it("doesn't wildcard path with strict", () => {
+    const pattern = [
+      {
+        route: "/api/tests",
+        method: "POST",
+        strict: true,
+      },
+    ]
+    const ctx = structures.koa.newContext()
+    ctx.request.url = "/api/tests/id/something/else"
+    ctx.request.method = "POST"
+
+    const built = matchers.buildMatcherRegex(pattern)
+
+    expect(!!matchers.matches(ctx, built)).toBe(false)
+  })
+
+  it("matches with param", () => {
+    const pattern = [
+      {
+        route: "/api/tests/:testId",
+        method: "GET",
+      },
+    ]
+    const ctx = structures.koa.newContext()
+    ctx.request.url = "/api/tests/id"
+    ctx.request.method = "GET"
+
+    const built = matchers.buildMatcherRegex(pattern)
+
+    expect(!!matchers.matches(ctx, built)).toBe(true)
+  })
+
+  // TODO: Support the below behaviour
+  // Strict does not work when a param is present
+  // it("matches with param with strict", () => {
+  //   const pattern = [{
+  //     route: "/api/tests/:testId",
+  //     method: "GET",
+  //     strict: true
+  //   }]
+  //   const ctx = structures.koa.newContext()
+  //   ctx.request.url = "/api/tests/id"
+  //   ctx.request.method = "GET"
+  //
+  //   const built = matchers.buildMatcherRegex(pattern)
+  //
+  //   expect(!!matchers.matches(ctx, built)).toBe(true)
+  // })
+
+  it("doesn't match by path", () => {
+    const pattern = [
+      {
+        route: "/api/tests",
+        method: "POST",
+      },
+    ]
+    const ctx = structures.koa.newContext()
+    ctx.request.url = "/api/unknown"
+    ctx.request.method = "POST"
+
+    const built = matchers.buildMatcherRegex(pattern)
+
+    expect(!!matchers.matches(ctx, built)).toBe(false)
+  })
+
+  it("doesn't match by method", () => {
+    const pattern = [
+      {
+        route: "/api/tests",
+        method: "POST",
+      },
+    ]
+    const ctx = structures.koa.newContext()
+    ctx.request.url = "/api/tests"
+    ctx.request.method = "GET"
+
+    const built = matchers.buildMatcherRegex(pattern)
+
+    expect(!!matchers.matches(ctx, built)).toBe(false)
+  })
+
+  it("matches by path and wildcard method", () => {
+    const pattern = [
+      {
+        route: "/api/tests",
+        method: "ALL",
+      },
+    ]
+    const ctx = structures.koa.newContext()
+    ctx.request.url = "/api/tests"
+    ctx.request.method = "GET"
+
+    const built = matchers.buildMatcherRegex(pattern)
+
+    expect(!!matchers.matches(ctx, built)).toBe(true)
+  })
+})

packages/backend-core/src/migrations/migrations.ts

@@ -12,6 +12,7 @@ import {
   MigrationOptions,
   MigrationType,
   MigrationNoOpOptions,
+  App,
 } from "@budibase/types"
 
 export const getMigrationsDoc = async (db: any) => {
@@ -41,7 +42,7 @@ export const runMigration = async (
   options: MigrationOptions = {}
 ) => {
   const migrationType = migration.type
-  let tenantId: string
+  let tenantId: string | undefined
   if (migrationType !== MigrationType.INSTALLATION) {
     tenantId = getTenantId()
   }
@@ -55,14 +56,17 @@ export const runMigration = async (
   }
 
   // get the db to store the migration in
-  let dbNames
+  let dbNames: string[]
   if (migrationType === MigrationType.GLOBAL) {
     dbNames = [getGlobalDBName()]
   } else if (migrationType === MigrationType.APP) {
     if (options.noOp) {
+      if (!options.noOp.appId) {
+        throw new Error("appId is required for noOp app migration")
+      }
       dbNames = [options.noOp.appId]
     } else {
-      const apps = await getAllApps(migration.appOpts)
+      const apps = (await getAllApps(migration.appOpts)) as App[]
       dbNames = apps.map(app => app.appId)
     }
   } else if (migrationType === MigrationType.INSTALLATION) {

packages/backend-core/src/migrations/tests/index.spec.js

@@ -1,6 +1,6 @@
-require("../../../tests/utilities/TestConfiguration")
+require("../../../tests")
 const { runMigrations, getMigrationsDoc } = require("../index")
-const { dangerousGetDB } = require("../../db")
+const { getDB } = require("../../db")
 const {
   StaticDatabases,
 } = require("../../db/utils")
@@ -18,7 +18,7 @@ describe("migrations", () => {
   }]
 
   beforeEach(() => {
-    db = dangerousGetDB(StaticDatabases.GLOBAL.name)
+    db = getDB(StaticDatabases.GLOBAL.name)
   })
 
   afterEach(async () => {

packages/backend-core/src/pkg/cache.ts

@@ -3,9 +3,11 @@
 import * as generic from "../cache/generic"
 import * as user from "../cache/user"
 import * as app from "../cache/appMetadata"
+import * as writethrough from "../cache/writethrough"
 
 export = {
   app,
   user,
+  writethrough,
   ...generic,
 }

packages/backend-core/src/pkg/db.ts

@@ -1,7 +0,0 @@
-// Mimic the outer package export for usage in index.ts
-// The outer exports can't be used as they now reference dist directly
-export * from "../db"
-export * from "../db/utils"
-export * from "../db/views"
-export * from "../db/pouch"
-export * from "../db/constants"

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -29,6 +29,7 @@ class InMemoryQueue {
   _messages: any[]
   _emitter: EventEmitter
   _runCount: number
+  _addCount: number
   /**
    * The constructor the queue, exactly the same as that of Bulls.
    * @param {string} name The name of the queue which is being configured.
@@ -41,6 +42,7 @@ class InMemoryQueue {
     this._messages = []
     this._emitter = new events.EventEmitter()
     this._runCount = 0
+    this._addCount = 0
   }
 
   /**
@@ -81,6 +83,7 @@ class InMemoryQueue {
       throw "Queue only supports carrying JSON."
     }
     this._messages.push(newJob(this._name, msg))
+    this._addCount++
     this._emitter.emit("message")
   }
 
@@ -128,12 +131,9 @@ class InMemoryQueue {
   }
 
   async waitForCompletion() {
-    const currentCount = this._runCount
-    let increased = false
     do {
       await timeout(50)
-      increased = this._runCount > currentCount
+    } while (this._addCount < this._runCount)
-    } while (!increased)
   }
 }
 

packages/backend-core/src/security/permissions.js

@@ -1,164 +0,0 @@
-const { flatten } = require("lodash")
-const { cloneDeep } = require("lodash/fp")
-
-const PermissionLevels = {
-  READ: "read",
-  WRITE: "write",
-  EXECUTE: "execute",
-  ADMIN: "admin",
-}
-
-// these are the global types, that govern the underlying default behaviour
-const PermissionTypes = {
-  APP: "app",
-  TABLE: "table",
-  USER: "user",
-  AUTOMATION: "automation",
-  WEBHOOK: "webhook",
-  BUILDER: "builder",
-  VIEW: "view",
-  QUERY: "query",
-}
-
-function Permission(type, level) {
-  this.level = level
-  this.type = type
-}
-
-function levelToNumber(perm) {
-  switch (perm) {
-    // not everything has execute privileges
-    case PermissionLevels.EXECUTE:
-      return 0
-    case PermissionLevels.READ:
-      return 1
-    case PermissionLevels.WRITE:
-      return 2
-    case PermissionLevels.ADMIN:
-      return 3
-    default:
-      return -1
-  }
-}
-
-/**
- * Given the specified permission level for the user return the levels they are allowed to carry out.
- * @param {string} userPermLevel The permission level of the user.
- * @return {string[]} All the permission levels this user is allowed to carry out.
- */
-function getAllowedLevels(userPermLevel) {
-  switch (userPermLevel) {
-    case PermissionLevels.EXECUTE:
-      return [PermissionLevels.EXECUTE]
-    case PermissionLevels.READ:
-      return [PermissionLevels.EXECUTE, PermissionLevels.READ]
-    case PermissionLevels.WRITE:
-    case PermissionLevels.ADMIN:
-      return [
-        PermissionLevels.READ,
-        PermissionLevels.WRITE,
-        PermissionLevels.EXECUTE,
-      ]
-    default:
-      return []
-  }
-}
-
-exports.BUILTIN_PERMISSION_IDS = {
-  PUBLIC: "public",
-  READ_ONLY: "read_only",
-  WRITE: "write",
-  ADMIN: "admin",
-  POWER: "power",
-}
-
-const BUILTIN_PERMISSIONS = {
-  PUBLIC: {
-    _id: exports.BUILTIN_PERMISSION_IDS.PUBLIC,
-    name: "Public",
-    permissions: [
-      new Permission(PermissionTypes.WEBHOOK, PermissionLevels.EXECUTE),
-    ],
-  },
-  READ_ONLY: {
-    _id: exports.BUILTIN_PERMISSION_IDS.READ_ONLY,
-    name: "Read only",
-    permissions: [
-      new Permission(PermissionTypes.QUERY, PermissionLevels.READ),
-      new Permission(PermissionTypes.TABLE, PermissionLevels.READ),
-      new Permission(PermissionTypes.VIEW, PermissionLevels.READ),
-    ],
-  },
-  WRITE: {
-    _id: exports.BUILTIN_PERMISSION_IDS.WRITE,
-    name: "Read/Write",
-    permissions: [
-      new Permission(PermissionTypes.QUERY, PermissionLevels.WRITE),
-      new Permission(PermissionTypes.TABLE, PermissionLevels.WRITE),
-      new Permission(PermissionTypes.VIEW, PermissionLevels.READ),
-      new Permission(PermissionTypes.AUTOMATION, PermissionLevels.EXECUTE),
-    ],
-  },
-  POWER: {
-    _id: exports.BUILTIN_PERMISSION_IDS.POWER,
-    name: "Power",
-    permissions: [
-      new Permission(PermissionTypes.TABLE, PermissionLevels.WRITE),
-      new Permission(PermissionTypes.USER, PermissionLevels.READ),
-      new Permission(PermissionTypes.AUTOMATION, PermissionLevels.EXECUTE),
-      new Permission(PermissionTypes.VIEW, PermissionLevels.READ),
-      new Permission(PermissionTypes.WEBHOOK, PermissionLevels.READ),
-    ],
-  },
-  ADMIN: {
-    _id: exports.BUILTIN_PERMISSION_IDS.ADMIN,
-    name: "Admin",
-    permissions: [
-      new Permission(PermissionTypes.TABLE, PermissionLevels.ADMIN),
-      new Permission(PermissionTypes.USER, PermissionLevels.ADMIN),
-      new Permission(PermissionTypes.AUTOMATION, PermissionLevels.ADMIN),
-      new Permission(PermissionTypes.VIEW, PermissionLevels.ADMIN),
-      new Permission(PermissionTypes.WEBHOOK, PermissionLevels.READ),
-      new Permission(PermissionTypes.QUERY, PermissionLevels.ADMIN),
-    ],
-  },
-}
-
-exports.getBuiltinPermissions = () => {
-  return cloneDeep(BUILTIN_PERMISSIONS)
-}
-
-exports.getBuiltinPermissionByID = id => {
-  const perms = Object.values(BUILTIN_PERMISSIONS)
-  return perms.find(perm => perm._id === id)
-}
-
-exports.doesHaveBasePermission = (permType, permLevel, rolesHierarchy) => {
-  const basePermissions = [
-    ...new Set(rolesHierarchy.map(role => role.permissionId)),
-  ]
-  const builtins = Object.values(BUILTIN_PERMISSIONS)
-  let permissions = flatten(
-    builtins
-      .filter(builtin => basePermissions.indexOf(builtin._id) !== -1)
-      .map(builtin => builtin.permissions)
-  )
-  for (let permission of permissions) {
-    if (
-      permission.type === permType &&
-      getAllowedLevels(permission.level).indexOf(permLevel) !== -1
-    ) {
-      return true
-    }
-  }
-  return false
-}
-
-exports.isPermissionLevelHigherThanRead = level => {
-  return levelToNumber(level) > 1
-}
-
-// utility as a lot of things need simply the builder permission
-exports.BUILDER = PermissionTypes.BUILDER
-exports.PermissionTypes = PermissionTypes
-exports.PermissionLevels = PermissionLevels

packages/backend-core/src/security/permissions.ts

@@ -0,0 +1,175 @@
+const { flatten } = require("lodash")
+const { cloneDeep } = require("lodash/fp")
+
+export type RoleHierarchy = {
+  permissionId: string
+}[]
+
+export enum PermissionLevel {
+  READ = "read",
+  WRITE = "write",
+  EXECUTE = "execute",
+  ADMIN = "admin",
+}
+
+// these are the global types, that govern the underlying default behaviour
+export enum PermissionType {
+  APP = "app",
+  TABLE = "table",
+  USER = "user",
+  AUTOMATION = "automation",
+  WEBHOOK = "webhook",
+  BUILDER = "builder",
+  VIEW = "view",
+  QUERY = "query",
+}
+
+class Permission {
+  type: PermissionType
+  level: PermissionLevel
+
+  constructor(type: PermissionType, level: PermissionLevel) {
+    this.type = type
+    this.level = level
+  }
+}
+
+function levelToNumber(perm: PermissionLevel) {
+  switch (perm) {
+    // not everything has execute privileges
+    case PermissionLevel.EXECUTE:
+      return 0
+    case PermissionLevel.READ:
+      return 1
+    case PermissionLevel.WRITE:
+      return 2
+    case PermissionLevel.ADMIN:
+      return 3
+    default:
+      return -1
+  }
+}
+
+/**
+ * Given the specified permission level for the user return the levels they are allowed to carry out.
+ * @param {string} userPermLevel The permission level of the user.
+ * @return {string[]} All the permission levels this user is allowed to carry out.
+ */
+function getAllowedLevels(userPermLevel: PermissionLevel) {
+  switch (userPermLevel) {
+    case PermissionLevel.EXECUTE:
+      return [PermissionLevel.EXECUTE]
+    case PermissionLevel.READ:
+      return [PermissionLevel.EXECUTE, PermissionLevel.READ]
+    case PermissionLevel.WRITE:
+    case PermissionLevel.ADMIN:
+      return [
+        PermissionLevel.READ,
+        PermissionLevel.WRITE,
+        PermissionLevel.EXECUTE,
+      ]
+    default:
+      return []
+  }
+}
+
+export enum BuiltinPermissionID {
+  PUBLIC = "public",
+  READ_ONLY = "read_only",
+  WRITE = "write",
+  ADMIN = "admin",
+  POWER = "power",
+}
+
+const BUILTIN_PERMISSIONS = {
+  PUBLIC: {
+    _id: BuiltinPermissionID.PUBLIC,
+    name: "Public",
+    permissions: [
+      new Permission(PermissionType.WEBHOOK, PermissionLevel.EXECUTE),
+    ],
+  },
+  READ_ONLY: {
+    _id: BuiltinPermissionID.READ_ONLY,
+    name: "Read only",
+    permissions: [
+      new Permission(PermissionType.QUERY, PermissionLevel.READ),
+      new Permission(PermissionType.TABLE, PermissionLevel.READ),
+      new Permission(PermissionType.VIEW, PermissionLevel.READ),
+    ],
+  },
+  WRITE: {
+    _id: BuiltinPermissionID.WRITE,
+    name: "Read/Write",
+    permissions: [
+      new Permission(PermissionType.QUERY, PermissionLevel.WRITE),
+      new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
+      new Permission(PermissionType.VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
+    ],
+  },
+  POWER: {
+    _id: BuiltinPermissionID.POWER,
+    name: "Power",
+    permissions: [
+      new Permission(PermissionType.TABLE, PermissionLevel.WRITE),
+      new Permission(PermissionType.USER, PermissionLevel.READ),
+      new Permission(PermissionType.AUTOMATION, PermissionLevel.EXECUTE),
+      new Permission(PermissionType.VIEW, PermissionLevel.READ),
+      new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
+    ],
+  },
+  ADMIN: {
+    _id: BuiltinPermissionID.ADMIN,
+    name: "Admin",
+    permissions: [
+      new Permission(PermissionType.TABLE, PermissionLevel.ADMIN),
+      new Permission(PermissionType.USER, PermissionLevel.ADMIN),
+      new Permission(PermissionType.AUTOMATION, PermissionLevel.ADMIN),
+      new Permission(PermissionType.VIEW, PermissionLevel.ADMIN),
+      new Permission(PermissionType.WEBHOOK, PermissionLevel.READ),
+      new Permission(PermissionType.QUERY, PermissionLevel.ADMIN),
+    ],
+  },
+}
+
+export function getBuiltinPermissions() {
+  return cloneDeep(BUILTIN_PERMISSIONS)
+}
+
+export function getBuiltinPermissionByID(id: string) {
+  const perms = Object.values(BUILTIN_PERMISSIONS)
+  return perms.find(perm => perm._id === id)
+}
+
+export function doesHaveBasePermission(
+  permType: PermissionType,
+  permLevel: PermissionLevel,
+  rolesHierarchy: RoleHierarchy
+) {
+  const basePermissions = [
+    ...new Set(rolesHierarchy.map(role => role.permissionId)),
+  ]
+  const builtins = Object.values(BUILTIN_PERMISSIONS)
+  let permissions = flatten(
+    builtins
+      .filter(builtin => basePermissions.indexOf(builtin._id) !== -1)
+      .map(builtin => builtin.permissions)
+  )
+  for (let permission of permissions) {
+    if (
+      permission.type === permType &&
+      getAllowedLevels(permission.level).indexOf(permLevel) !== -1
+    ) {
+      return true
+    }
+  }
+  return false
+}
+
+export function isPermissionLevelHigherThanRead(level: PermissionLevel) {
+  return levelToNumber(level) > 1
+}
+
+// utility as a lot of things need simply the builder permission
+export const BUILDER = PermissionType.BUILDER

packages/backend-core/src/security/roles.ts

@@ -1,4 +1,4 @@
-import { BUILTIN_PERMISSION_IDS, PermissionLevels } from "./permissions"
+import { BuiltinPermissionID, PermissionLevel } from "./permissions"
 import {
   generateRoleID,
   getRoleParams,
@@ -54,19 +54,19 @@ export class Role {
 
 const BUILTIN_ROLES = {
   ADMIN: new Role(BUILTIN_IDS.ADMIN, "Admin")
-    .addPermission(BUILTIN_PERMISSION_IDS.ADMIN)
+    .addPermission(BuiltinPermissionID.ADMIN)
     .addInheritance(BUILTIN_IDS.POWER),
   POWER: new Role(BUILTIN_IDS.POWER, "Power")
-    .addPermission(BUILTIN_PERMISSION_IDS.POWER)
+    .addPermission(BuiltinPermissionID.POWER)
     .addInheritance(BUILTIN_IDS.BASIC),
   BASIC: new Role(BUILTIN_IDS.BASIC, "Basic")
-    .addPermission(BUILTIN_PERMISSION_IDS.WRITE)
+    .addPermission(BuiltinPermissionID.WRITE)
     .addInheritance(BUILTIN_IDS.PUBLIC),
   PUBLIC: new Role(BUILTIN_IDS.PUBLIC, "Public").addPermission(
-    BUILTIN_PERMISSION_IDS.PUBLIC
+    BuiltinPermissionID.PUBLIC
   ),
   BUILDER: new Role(BUILTIN_IDS.BUILDER, "Builder").addPermission(
-    BUILTIN_PERMISSION_IDS.ADMIN
+    BuiltinPermissionID.ADMIN
   ),
 }
 
@@ -147,9 +147,9 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string) {
  * @param {string|null} roleId The level ID to lookup.
  * @returns {Promise<Role|object|null>} The role object, which may contain an "inherits" property.
  */
-export async function getRole(roleId?: string) {
+export async function getRole(roleId?: string): Promise<RoleDoc | undefined> {
   if (!roleId) {
-    return null
+    return undefined
   }
   let role: any = {}
   // built in roles mostly come from the in-code implementation,
@@ -193,7 +193,9 @@ async function getAllUserRoles(userRoleId?: string): Promise<RoleDoc[]> {
   ) {
     roleIds.push(currentRole.inherits)
     currentRole = await getRole(currentRole.inherits)
-    roles.push(currentRole)
+    if (currentRole) {
+      roles.push(currentRole)
+    }
   }
   return roles
 }
@@ -225,8 +227,8 @@ export function checkForRoleResourceArray(
   if (rolePerms && !Array.isArray(rolePerms[resourceId])) {
     const permLevel = rolePerms[resourceId] as any
     rolePerms[resourceId] = [permLevel]
-    if (permLevel === PermissionLevels.WRITE) {
+    if (permLevel === PermissionLevel.WRITE) {
-      rolePerms[resourceId].push(PermissionLevels.READ)
+      rolePerms[resourceId].push(PermissionLevel.READ)
     }
   }
   return rolePerms

packages/backend-core/src/tenancy/tenancy.ts

@@ -1,20 +1,29 @@
-import { doWithDB } from "../db"
-import { queryPlatformView } from "../db/views"
-import { StaticDatabases, ViewName } from "../db/constants"
-import { getGlobalDBName } from "../db/tenancy"
 import {
-  getTenantId,
+  doWithDB,
+  queryPlatformView,
+  StaticDatabases,
+  getGlobalDBName,
+  ViewName,
+} from "../db"
+import {
   DEFAULT_TENANT_ID,
-  isMultiTenant,
+  getTenantId,
   getTenantIDFromAppID,
+  isMultiTenant,
 } from "../context"
 import env from "../environment"
-import { PlatformUser } from "@budibase/types"
+import {
+  BBContext,
+  PlatformUser,
+  TenantResolutionStrategy,
+  GetTenantIdOptions,
+} from "@budibase/types"
+import { Header } from "../constants"
 
 const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
 const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
 
-export const addTenantToUrl = (url: string) => {
+export function addTenantToUrl(url: string) {
   const tenantId = getTenantId()
 
   if (isMultiTenant()) {
@@ -25,7 +34,7 @@ export const addTenantToUrl = (url: string) => {
   return url
 }
 
-export const doesTenantExist = async (tenantId: string) => {
+export async function doesTenantExist(tenantId: string) {
   return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
     let tenants
     try {
@@ -42,12 +51,12 @@ export const doesTenantExist = async (tenantId: string) => {
   })
 }
 
-export const tryAddTenant = async (
+export async function tryAddTenant(
   tenantId: string,
   userId: string,
   email: string,
   afterCreateTenant: () => Promise<void>
-) => {
+) {
   return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
     const getDoc = async (id: string) => {
       if (!id) {
@@ -89,11 +98,11 @@ export const tryAddTenant = async (
   })
 }
 
-export const doWithGlobalDB = (tenantId: string, cb: any) => {
+export function doWithGlobalDB(tenantId: string, cb: any) {
   return doWithDB(getGlobalDBName(tenantId), cb)
 }
 
-export const lookupTenantId = async (userId: string) => {
+export async function lookupTenantId(userId: string) {
   return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: any) => {
     let tenantId = env.MULTI_TENANCY ? DEFAULT_TENANT_ID : null
     try {
@@ -109,19 +118,26 @@ export const lookupTenantId = async (userId: string) => {
 }
 
 // lookup, could be email or userId, either will return a doc
-export const getTenantUser = async (
+export async function getTenantUser(
   identifier: string
-): Promise<PlatformUser | null> => {
+): Promise<PlatformUser | undefined> {
   // use the view here and allow to find anyone regardless of casing
-  // Use lowercase to ensure email login is case insensitive
+  // Use lowercase to ensure email login is case-insensitive
-  const response = queryPlatformView(ViewName.PLATFORM_USERS_LOWERCASE, {
+  const users = await queryPlatformView<PlatformUser>(
-    keys: [identifier.toLowerCase()],
+    ViewName.PLATFORM_USERS_LOWERCASE,
-    include_docs: true,
+    {
-  }) as Promise<PlatformUser>
+      keys: [identifier.toLowerCase()],
-  return response
+      include_docs: true,
+    }
+  )
+  if (Array.isArray(users)) {
+    return users[0]
+  } else {
+    return users
+  }
 }
 
-export const isUserInAppTenant = (appId: string, user?: any) => {
+export function isUserInAppTenant(appId: string, user?: any) {
   let userTenantId
   if (user) {
     userTenantId = user.tenantId || DEFAULT_TENANT_ID
@@ -132,7 +148,7 @@ export const isUserInAppTenant = (appId: string, user?: any) => {
   return tenantId === userTenantId
 }
 
-export const getTenantIds = async () => {
+export async function getTenantIds() {
   return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
     let tenants
     try {
@@ -144,3 +160,108 @@ export const getTenantIds = async () => {
     return (tenants && tenants.tenantIds) || []
   })
 }
+
+const ALL_STRATEGIES = Object.values(TenantResolutionStrategy)
+
+export const getTenantIDFromCtx = (
+  ctx: BBContext,
+  opts: GetTenantIdOptions
+): string | null => {
+  // exit early if not multi-tenant
+  if (!isMultiTenant()) {
+    return DEFAULT_TENANT_ID
+  }
+
+  // opt defaults
+  if (opts.allowNoTenant === undefined) {
+    opts.allowNoTenant = false
+  }
+  if (!opts.includeStrategies) {
+    opts.includeStrategies = ALL_STRATEGIES
+  }
+  if (!opts.excludeStrategies) {
+    opts.excludeStrategies = []
+  }
+
+  const isAllowed = (strategy: TenantResolutionStrategy) => {
+    // excluded takes precedence
+    if (opts.excludeStrategies?.includes(strategy)) {
+      return false
+    }
+    if (opts.includeStrategies?.includes(strategy)) {
+      return true
+    }
+  }
+
+  // always use user first
+  if (isAllowed(TenantResolutionStrategy.USER)) {
+    const userTenantId = ctx.user?.tenantId
+    if (userTenantId) {
+      return userTenantId
+    }
+  }
+
+  // header
+  if (isAllowed(TenantResolutionStrategy.HEADER)) {
+    const headerTenantId = ctx.request.headers[Header.TENANT_ID]
+    if (headerTenantId) {
+      return headerTenantId as string
+    }
+  }
+
+  // query param
+  if (isAllowed(TenantResolutionStrategy.QUERY)) {
+    const queryTenantId = ctx.request.query.tenantId
+    if (queryTenantId) {
+      return queryTenantId as string
+    }
+  }
+
+  // subdomain
+  if (isAllowed(TenantResolutionStrategy.SUBDOMAIN)) {
+    // e.g. budibase.app or local.com:10000
+    const platformHost = new URL(env.PLATFORM_URL).host.split(":")[0]
+    // e.g. tenant.budibase.app or tenant.local.com
+    const requestHost = ctx.host
+    // parse the tenant id from the difference
+    if (requestHost.includes(platformHost)) {
+      const tenantId = requestHost.substring(
+        0,
+        requestHost.indexOf(`.${platformHost}`)
+      )
+      if (tenantId) {
+        return tenantId
+      }
+    }
+  }
+
+  // path
+  if (isAllowed(TenantResolutionStrategy.PATH)) {
+    // params - have to parse manually due to koa-router not run yet
+    const match = ctx.matched.find(
+      (m: any) => !!m.paramNames.find((p: any) => p.name === "tenantId")
+    )
+
+    // get the raw path url - without any query params
+    const ctxUrl = ctx.originalUrl
+    let url
+    if (ctxUrl.includes("?")) {
+      url = ctxUrl.split("?")[0]
+    } else {
+      url = ctxUrl
+    }
+
+    if (match) {
+      const params = match.params(url, match.captures(url), {})
+      if (params.tenantId) {
+        return params.tenantId
+      }
+    }
+  }
+
+  if (!opts.allowNoTenant) {
+    ctx.throw(403, "Tenant id not set")
+  }
+
+  return null
+}

packages/backend-core/src/tests/utils.spec.js

@@ -1,5 +1,4 @@
-require("../../tests/utilities/TestConfiguration")
+const { structures } = require("../../tests")
-const { structures } = require("../../tests/utilities")
 const utils = require("../utils")
 const events = require("../events")
 const { doInTenant, DEFAULT_TENANT_ID }= require("../context")

packages/backend-core/src/users.ts

@@ -3,15 +3,14 @@ import {
   getUsersByAppParams,
   getProdAppID,
   generateAppUserID,
-} from "./db/utils"
+  queryGlobalView,
-import { queryGlobalView } from "./db/views"
+  UNICODE_MAX,
-import { UNICODE_MAX } from "./db/constants"
+} from "./db"
 import { BulkDocsResponse, User } from "@budibase/types"
 import { getGlobalDB } from "./context"
-import PouchDB from "pouchdb"
 
 export const bulkGetGlobalUsersById = async (userIds: string[]) => {
-  const db = getGlobalDB() as PouchDB.Database
+  const db = getGlobalDB()
   return (
     await db.allDocs({
       keys: userIds,
@@ -21,7 +20,7 @@ export const bulkGetGlobalUsersById = async (userIds: string[]) => {
 }
 
 export const bulkUpdateGlobalUsers = async (users: User[]) => {
-  const db = getGlobalDB() as PouchDB.Database
+  const db = getGlobalDB()
   return (await db.bulkDocs(users)) as BulkDocsResponse
 }
 
@@ -69,7 +68,7 @@ export const getGlobalUserByAppPage = (appId: string, user: User) => {
   if (!user) {
     return
   }
-  return generateAppUserID(getProdAppID(appId), user._id!)
+  return generateAppUserID(getProdAppID(appId)!, user._id!)
 }
 
 /**

packages/backend-core/src/utils.ts

@@ -1,38 +1,51 @@
-const { DocumentType, SEPARATOR, ViewName, getAllApps } = require("./db/utils")
+import {
+  DocumentType,
+  SEPARATOR,
+  ViewName,
+  getAllApps,
+  queryGlobalView,
+} from "./db"
+import { options } from "./middleware/passport/jwt"
+import { Header, Cookie, MAX_VALID_DATE } from "./constants"
+import env from "./environment"
+import userCache from "./cache/user"
+import { getSessionsForUser, invalidateSessions } from "./security/sessions"
+import * as events from "./events"
+import tenancy from "./tenancy"
+import {
+  App,
+  BBContext,
+  PlatformLogoutOpts,
+  TenantResolutionStrategy,
+} from "@budibase/types"
+import { SetOption } from "cookies"
 const jwt = require("jsonwebtoken")
-const { options } = require("./middleware/passport/jwt")
-const { queryGlobalView } = require("./db/views")
-const { Headers, Cookies, MAX_VALID_DATE } = require("./constants")
-const env = require("./environment")
-const userCache = require("./cache/user")
-const {
-  getSessionsForUser,
-  invalidateSessions,
-} = require("./security/sessions")
-const events = require("./events")
-const tenancy = require("./tenancy")
 
 const APP_PREFIX = DocumentType.APP + SEPARATOR
 const PROD_APP_PREFIX = "/app/"
 
-function confirmAppId(possibleAppId) {
+function confirmAppId(possibleAppId: string | undefined) {
   return possibleAppId && possibleAppId.startsWith(APP_PREFIX)
     ? possibleAppId
     : undefined
 }
 
-async function resolveAppUrl(ctx) {
+async function resolveAppUrl(ctx: BBContext) {
   const appUrl = ctx.path.split("/")[2]
   let possibleAppUrl = `/${appUrl.toLowerCase()}`
 
-  let tenantId = tenancy.getTenantId()
+  let tenantId: string | null = tenancy.getTenantId()
-  if (!env.SELF_HOSTED && ctx.subdomains.length) {
+  if (env.MULTI_TENANCY) {
-    // always use the tenant id from the url in cloud
+    // always use the tenant id from the subdomain in multi tenancy
-    tenantId = ctx.subdomains[0]
+    // this ensures the logged-in user tenant id doesn't overwrite
+    // e.g. in the case of viewing a public app while already logged-in to another tenant
+    tenantId = tenancy.getTenantIDFromCtx(ctx, {
+      includeStrategies: [TenantResolutionStrategy.SUBDOMAIN],
+    })
   }
 
   // search prod apps for a url that matches
-  const apps = await tenancy.doInTenant(tenantId, () =>
+  const apps: App[] = await tenancy.doInTenant(tenantId, () =>
     getAllApps({ dev: false })
   )
   const app = apps.filter(
@@ -42,7 +55,7 @@ async function resolveAppUrl(ctx) {
   return app && app.appId ? app.appId : undefined
 }
 
-exports.isServingApp = ctx => {
+export function isServingApp(ctx: BBContext) {
   // dev app
   if (ctx.path.startsWith(`/${APP_PREFIX}`)) {
     return true
@@ -59,12 +72,12 @@ exports.isServingApp = ctx => {
  * @param {object} ctx The main request body to look through.
  * @returns {string|undefined} If an appId was found it will be returned.
  */
-exports.getAppIdFromCtx = async ctx => {
+export async function getAppIdFromCtx(ctx: BBContext) {
   // look in headers
-  const options = [ctx.headers[Headers.APP_ID]]
+  const options = [ctx.headers[Header.APP_ID]]
   let appId
   for (let option of options) {
-    appId = confirmAppId(option)
+    appId = confirmAppId(option as string)
     if (appId) {
       break
     }
@@ -95,7 +108,7 @@ exports.getAppIdFromCtx = async ctx => {
  * opens the contents of the specified encrypted JWT.
  * @return {object} the contents of the token.
  */
-exports.openJwt = token => {
+export function openJwt(token: string) {
   if (!token) {
     return token
   }
@@ -107,14 +120,14 @@ exports.openJwt = token => {
  * @param {object} ctx The request which is to be manipulated.
  * @param {string} name The name of the cookie to get.
  */
-exports.getCookie = (ctx, name) => {
+export function getCookie(ctx: BBContext, name: string) {
   const cookie = ctx.cookies.get(name)
 
   if (!cookie) {
     return cookie
   }
 
-  return exports.openJwt(cookie)
+  return openJwt(cookie)
 }
 
 /**
@@ -124,12 +137,17 @@ exports.getCookie = (ctx, name) => {
  * @param {string|object} value The value of cookie which will be set.
  * @param {object} opts options like whether to sign.
  */
-exports.setCookie = (ctx, value, name = "builder", opts = { sign: true }) => {
+export function setCookie(
+  ctx: BBContext,
+  value: any,
+  name = "builder",
+  opts = { sign: true }
+) {
   if (value && opts && opts.sign) {
     value = jwt.sign(value, options.secretOrKey)
   }
 
-  const config = {
+  const config: SetOption = {
     expires: MAX_VALID_DATE,
     path: "/",
     httpOnly: false,
@@ -146,8 +164,8 @@ exports.setCookie = (ctx, value, name = "builder", opts = { sign: true }) => {
 /**
  * Utility function, simply calls setCookie with an empty string for value
  */
-exports.clearCookie = (ctx, name) => {
+export function clearCookie(ctx: BBContext, name: string) {
-  exports.setCookie(ctx, null, name)
+  setCookie(ctx, null, name)
 }
 
 /**
@@ -156,11 +174,11 @@ exports.clearCookie = (ctx, name) => {
  * @param {object} ctx The koa context object to be tested.
  * @return {boolean} returns true if the call is from the client lib (a built app rather than the builder).
  */
-exports.isClient = ctx => {
+export function isClient(ctx: BBContext) {
-  return ctx.headers[Headers.TYPE] === "client"
+  return ctx.headers[Header.TYPE] === "client"
 }
 
-const getBuilders = async () => {
+async function getBuilders() {
   const builders = await queryGlobalView(ViewName.USER_BY_BUILDERS, {
     include_docs: false,
   })
@@ -176,7 +194,7 @@ const getBuilders = async () => {
   }
 }
 
-exports.getBuildersCount = async () => {
+export async function getBuildersCount() {
   const builders = await getBuilders()
   return builders.length
 }
@@ -184,10 +202,14 @@ exports.getBuildersCount = async () => {
 /**
  * Logs a user out from budibase. Re-used across account portal and builder.
  */
-exports.platformLogout = async ({ ctx, userId, keepActiveSession }) => {
+export async function platformLogout(opts: PlatformLogoutOpts) {
+  const ctx = opts.ctx
+  const userId = opts.userId
+  const keepActiveSession = opts.keepActiveSession
+
   if (!ctx) throw new Error("Koa context must be supplied to logout.")
 
-  const currentSession = exports.getCookie(ctx, Cookies.Auth)
+  const currentSession = getCookie(ctx, Cookie.Auth)
   let sessions = await getSessionsForUser(userId)
 
   if (keepActiveSession) {
@@ -196,8 +218,8 @@ exports.platformLogout = async ({ ctx, userId, keepActiveSession }) => {
     )
   } else {
     // clear cookies
-    exports.clearCookie(ctx, Cookies.Auth)
+    clearCookie(ctx, Cookie.Auth)
-    exports.clearCookie(ctx, Cookies.CurrentApp)
+    clearCookie(ctx, Cookie.CurrentApp)
   }
 
   const sessionIds = sessions.map(({ sessionId }) => sessionId)
@@ -206,6 +228,6 @@ exports.platformLogout = async ({ ctx, userId, keepActiveSession }) => {
   await userCache.invalidateUser(userId)
 }
 
-exports.timeout = timeMs => {
+export function timeout(timeMs: number) {
   return new Promise(resolve => setTimeout(resolve, timeMs))
 }

packages/backend-core/tests/jestSetup.ts

@@ -1,17 +1,21 @@
-const env = require("../src/environment")
+import env from "../src/environment"
+import { mocks } from "./utilities"
+
+// must explicitly enable fetch mock
+mocks.fetch.enable()
+
+// mock all dates to 2020-01-01T00:00:00.000Z
+// use tk.reset() to use real dates in individual tests
+import tk from "timekeeper"
+tk.freeze(mocks.date.MOCK_DATE)
 
 env._set("SELF_HOSTED", "1")
 env._set("NODE_ENV", "jest")
 env._set("JWT_SECRET", "test-jwtsecret")
 env._set("LOG_LEVEL", "silent")
-env._set("MULTI_TENANCY", true)
+env._set("MINIO_URL", "http://localhost")
-
+env._set("MINIO_ACCESS_KEY", "test")
-const { mocks } = require("@budibase/backend-core/tests")
+env._set("MINIO_SECRET_KEY", "test")
-
-// mock all dates to 2020-01-01T00:00:00.000Z
-// use tk.reset() to use real dates in individual tests
-const tk = require("timekeeper")
-tk.freeze(mocks.date.MOCK_DATE)
 
 global.console.log = jest.fn() // console.log are ignored in tests
 

packages/backend-core/tests/utilities/TestConfiguration.js

@@ -1 +0,0 @@
-require("./db")

packages/backend-core/tests/utilities/db.js

@@ -1,6 +0,0 @@
-const core = require("../../src/index")
-const dbConfig = {
-  inMemory: true,
-  allDbs: true,
-}
-core.init({ db: dbConfig })

packages/backend-core/tests/utilities/db.ts

@@ -0,0 +1,9 @@
+import * as db from "../../src/db"
+
+const dbConfig = {
+  inMemory: true,
+}
+
+export const init = () => {
+  db.init(dbConfig)
+}

packages/backend-core/tests/utilities/index.ts

@@ -1,2 +1,6 @@
 export * as mocks from "./mocks"
 export * as structures from "./structures"
+export { generator } from "./structures"
+
+import * as dbConfig from "./db"
+dbConfig.init()

packages/backend-core/tests/utilities/mocks/accounts.ts

@@ -1,7 +1,9 @@
 export const getAccount = jest.fn()
 export const getAccountByTenantId = jest.fn()
+export const getStatus = jest.fn()
 
 jest.mock("../../../src/cloud/accounts", () => ({
   getAccount,
   getAccountByTenantId,
+  getStatus,
 }))

packages/backend-core/tests/utilities/mocks/fetch.ts

@@ -0,0 +1,10 @@
+const mockFetch = jest.fn()
+
+const enable = () => {
+  jest.mock("node-fetch", () => mockFetch)
+}
+
+export default {
+  ...mockFetch,
+  enable,
+}

packages/backend-core/tests/utilities/mocks/index.ts

@@ -2,3 +2,4 @@ import "./posthog"
 import "./events"
 export * as accounts from "./accounts"
 export * as date from "./date"
+export { default as fetch } from "./fetch"

packages/backend-core/tests/utilities/structures/accounts.ts

@@ -1,23 +1,29 @@
 import { generator, uuid } from "."
-import { AuthType, CloudAccount, Hosting } from "@budibase/types"
 import * as db from "../../../src/db/utils"
+import { Account, AuthType, CloudAccount, Hosting } from "@budibase/types"
 
-export const cloudAccount = (): CloudAccount => {
+export const account = (): Account => {
   return {
     accountId: uuid(),
+    tenantId: generator.word(),
+    email: generator.email(),
+    tenantName: generator.word(),
+    hosting: Hosting.SELF,
     createdAt: Date.now(),
     verified: true,
     verificationSent: true,
-    tier: "",
+    tier: "FREE", // DEPRECATED
-    email: generator.email(),
-    tenantId: generator.word(),
-    hosting: Hosting.CLOUD,
     authType: AuthType.PASSWORD,
-    password: generator.word(),
-    tenantName: generator.word(),
     name: generator.name(),
     size: "10+",
     profession: "Software Engineer",
+  }
+}
+
+export const cloudAccount = (): CloudAccount => {
+  return {
+    ...account(),
+    hosting: Hosting.CLOUD,
     budibaseUserId: db.generateGlobalUserID(),
   }
 }

packages/backend-core/tests/utilities/structures/common.ts

@@ -1 +1,7 @@
+import { v4 as uuid } from "uuid"
+
 export { v4 as uuid } from "uuid"
+
+export const email = () => {
+  return `${uuid()}@test.com`
+}

packages/backend-core/tests/utilities/structures/koa.ts

@@ -1,5 +1,14 @@
-import { createMockContext } from "@shopify/jest-koa-mocks"
+import { createMockContext, createMockCookies } from "@shopify/jest-koa-mocks"
+import { BBContext } from "@budibase/types"
 
-export const newContext = () => {
+export const newContext = (): BBContext => {
-  return createMockContext()
+  const ctx = createMockContext()
+  return {
+    ...ctx,
+    cookies: createMockCookies(),
+    request: {
+      ...ctx.request,
+      body: {},
+    },
+  }
 }

packages/backend-core/tsconfig.build.json

@@ -12,7 +12,7 @@
     "sourceMap": true,
     "declaration": true,
     "types": [ "node", "jest" ],
-    "outDir": "dist"
+    "outDir": "dist",
   },
   "include": [
     "**/*.js",

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.1.22-alpha.2",
+  "version": "2.1.22-alpha.8",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "^1.2.1",
-    "@budibase/string-templates": "2.1.22-alpha.2",
+    "@budibase/string-templates": "2.1.22-alpha.8",
     "@spectrum-css/actionbutton": "^1.0.1",
     "@spectrum-css/actiongroup": "^1.0.1",
     "@spectrum-css/avatar": "^3.0.2",

packages/builder/cypress/integration/createApp.spec.js

@@ -10,7 +10,7 @@ filterTests(['smoke', 'all'], () => {
     })
 
     if (!(Cypress.env("TEST_ENV"))) {
-      it("should show the new user UI/UX", () => {
+      it.skip("should show the new user UI/UX", () => {
         cy.visit(`${Cypress.config().baseUrl}/builder/portal/apps/create`, { timeout: 5000 }) //added /portal/apps/create
         cy.wait(1000)
         cy.get(interact.CREATE_APP_BUTTON, { timeout: 10000 }).contains('Start from scratch').should("exist")
@@ -83,7 +83,7 @@ filterTests(['smoke', 'all'], () => {
       })
     })
 
-    it("should create the first application from scratch", () => {
+    it.skip("should create the first application from scratch", () => {
       const appName = "Cypress Tests"
       cy.createApp(appName, false)
 
@@ -93,7 +93,7 @@ filterTests(['smoke', 'all'], () => {
       cy.deleteApp(appName)
     })
 
-    it("should create the first application from scratch with a default name", () => {
+    it.skip("should create the first application from scratch with a default name", () => {
       cy.updateUserInformation("", "")
       cy.createApp("", false)
       cy.applicationInAppTable("My app")

packages/builder/cypress/integration/createAutomation.spec.js

@@ -12,7 +12,7 @@ filterTests(['smoke', 'all'], () => {
       cy.createTestTableWithData()
       cy.wait(2000)
       cy.contains("Automate").click()
-      cy.get(interact.ADD_BUTTON_SPECTRUM).click()
+      cy.get(interact.SPECTRUM_BUTTON_TEMPLATE).contains("Add automation").click({ force: true })
       cy.get(interact.MODAL_INNER_WRAPPER).within(() => {
         cy.get("input").type("Add Row")
         cy.contains("Row Created").click({ force: true })
@@ -24,7 +24,7 @@ filterTests(['smoke', 'all'], () => {
       cy.wait(500)
       cy.contains("dog").click()
       // Create action
-      cy.get('[aria-label="AddCircle"]', { timeout: 2000 }).eq(1).click()
+      cy.get('[aria-label="AddCircle"]', { timeout: 2000 }).click()
       cy.get(interact.MODAL_INNER_WRAPPER).within(() => {
         cy.wait(1000)
         cy.contains("Create Row").trigger('mouseover').click().click()

packages/builder/cypress/integration/createScreen.spec.js

@@ -9,7 +9,7 @@ filterTests(["smoke", "all"], () => {
       cy.navigateToFrontend()
     })
 
-    it("Should successfully create a screen", () => {
+    it.skip("Should successfully create a screen", () => {
       cy.createScreen("test")
       cy.get(interact.BODY).within(() => {
         cy.contains("/test").should("exist")
@@ -23,7 +23,7 @@ filterTests(["smoke", "all"], () => {
       })
     })
 
-    it("should delete all screens then create first screen via button", () => {
+    it.skip("should delete all screens then create first screen via button", () => {
       cy.deleteAllScreens()
       
       cy.contains("Create first screen").click()

packages/builder/cypress/integration/createTable.spec.js

@@ -2,7 +2,7 @@ import filterTests from "../support/filterTests"
 const interact = require('../support/interact')
 
 filterTests(["smoke", "all"], () => {
-  context("Create a Table", () => {
+  xcontext("Create a Table", () => {
     before(() => {
       cy.login()
       cy.createTestApp()

packages/builder/cypress/integration/revertApp.spec.js

@@ -2,7 +2,7 @@ import filterTests from "../support/filterTests"
 const interact = require('../support/interact')
 
 filterTests(['smoke', 'all'], () => {
-    context("Revert apps", () => {
+    xcontext("Revert apps", () => {
         before(() => {
         cy.login()
         cy.createTestApp()

packages/builder/cypress/support/commands.js

@@ -440,7 +440,7 @@ Cypress.Commands.add("createTable", (tableName, initialTable) => {
   // Creates an internal Budibase DB table
   if (!initialTable) {
     cy.navigateToDataSection()
-    cy.get(`[data-cy="new-table"]`, { timeout: 2000 }).click()
+    cy.get(`[data-cy="new-datasource"]`, { timeout: 2000 }).click()
   }
   cy.wait(2000)
   cy.get(".item", { timeout: 2000 })

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.1.22-alpha.2",
+  "version": "2.1.22-alpha.8",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -71,10 +71,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.1.22-alpha.2",
+    "@budibase/bbui": "2.1.22-alpha.8",
-    "@budibase/client": "2.1.22-alpha.2",
+    "@budibase/client": "2.1.22-alpha.8",
-    "@budibase/frontend-core": "2.1.22-alpha.2",
+    "@budibase/frontend-core": "2.1.22-alpha.8",
-    "@budibase/string-templates": "2.1.22-alpha.2",
+    "@budibase/string-templates": "2.1.22-alpha.8",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
@@ -117,9 +117,9 @@
     "start-server-and-test": "^1.12.1",
     "svelte": "^3.48.0",
     "svelte-jester": "^1.3.2",
-    "ts-node": "^10.4.0",
+    "ts-node": "10.8.1",
     "tsconfig-paths": "4.0.0",
-    "typescript": "^4.5.5",
+    "typescript": "4.7.3",
     "vite": "^3.0.8"
   },
   "gitHead": "115189f72a850bfb52b65ec61d932531bf327072"

packages/builder/src/pages/builder/portal/manage/auth/index.svelte

@@ -20,11 +20,12 @@
     Toggle,
     Tag,
     Tags,
+    Icon,
+    Helpers,
   } from "@budibase/bbui"
   import { onMount } from "svelte"
   import { API } from "api"
   import { organisation, admin } from "stores/portal"
-  import { Helpers } from "@budibase/bbui"
 
   const ConfigTypes = {
     Google: "google",
@@ -40,7 +41,9 @@
 
   // Indicate to user that callback is based on platform url
   // If there is an existing value, indicate that it may be removed to return to default behaviour
-  $: googleCallbackTooltip = googleCallbackReadonly
+  $: googleCallbackTooltip = $admin.cloud
+    ? null
+    : googleCallbackReadonly
     ? "Vist the organisation page to update the platform URL"
     : "Leave blank to use the default callback URL"
 
@@ -54,6 +57,7 @@
         readonly: googleCallbackReadonly,
         tooltip: googleCallbackTooltip,
         placeholder: $organisation.googleCallbackUrl,
+        copyButton: true,
       },
     ],
   }
@@ -66,9 +70,12 @@
       {
         name: "callbackURL",
         readonly: true,
-        tooltip: "Vist the organisation page to update the platform URL",
+        tooltip: $admin.cloud
+          ? null
+          : "Vist the organisation page to update the platform URL",
         label: "Callback URL",
         placeholder: $organisation.oidcCallbackUrl,
+        copyButton: true,
       },
     ],
   }
@@ -231,6 +238,11 @@
     },
   ]
 
+  const copyToClipboard = async value => {
+    await Helpers.copyToClipboard(value)
+    notifications.success("Copied")
+  }
+
   onMount(async () => {
     try {
       await organisation.init()
@@ -336,11 +348,23 @@
       {#each GoogleConfigFields.Google as field}
         <div class="form-row">
           <Label size="L" tooltip={field.tooltip}>{field.label}</Label>
-          <Input
+          <div class="inputContainer">
-            bind:value={providers.google.config[field.name]}
+            <div class="input">
-            readonly={field.readonly}
+              <Input
-            placeholder={field.placeholder}
+                bind:value={providers.google.config[field.name]}
-          />
+                readonly={field.readonly}
+                placeholder={field.placeholder}
+              />
+            </div>
+            {#if field.copyButton}
+              <div
+                class="copy"
+                on:click={() => copyToClipboard(field.placeholder)}
+              >
+                <Icon size="S" name="Copy" />
+              </div>
+            {/if}
+          </div>
         </div>
       {/each}
       <div class="form-row">
@@ -375,12 +399,24 @@
       {#each OIDCConfigFields.Oidc as field}
         <div class="form-row">
           <Label size="L" tooltip={field.tooltip}>{field.label}</Label>
-          <Input
+          <div class="inputContainer">
-            bind:value={providers.oidc.config.configs[0][field.name]}
+            <div class="input">
-            readonly={field.readonly}
+              <Input
-            placeholder={field.placeholder}
+                bind:value={providers.oidc.config.configs[0][field.name]}
-            dataCy={field.name}
+                readonly={field.readonly}
-          />
+                placeholder={field.placeholder}
+                dataCy={field.name}
+              />
+            </div>
+            {#if field.copyButton}
+              <div
+                class="copy"
+                on:click={() => copyToClipboard(field.placeholder)}
+              >
+                <Icon size="S" name="Copy" />
+              </div>
+            {/if}
+          </div>
         </div>
       {/each}
     </Layout>
@@ -557,4 +593,16 @@
   .provider-title span {
     flex: 1 1 auto;
   }
+  .inputContainer {
+    display: flex;
+    flex-direction: row;
+  }
+  .input {
+    flex: 1;
+  }
+  .copy {
+    display: flex;
+    align-items: center;
+    margin-left: 10px;
+  }
 </style>

packages/builder/yarn.lock

@@ -923,17 +923,12 @@
     exec-sh "^0.3.2"
     minimist "^1.2.0"
 
-"@cspotcode/source-map-consumer@0.8.0":
+"@cspotcode/source-map-support@^0.8.0":
-  version "0.8.0"
+  version "0.8.1"
-  resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b"
+  resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1"
-  integrity sha512-41qniHzTU8yAGbCp04ohlmSrZf8bkf/iJsl3V0dRGsQN/5GFfx+LbCSsCpp2gqrqjTVg/K6O8ycoV35JIwAzAg==
+  integrity sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==
-
-"@cspotcode/source-map-support@0.7.0":
-  version "0.7.0"
-  resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.7.0.tgz#4789840aa859e46d2f3173727ab707c66bf344f5"
-  integrity sha512-X4xqRHqN8ACt2aHVe51OxeA2HjbcL4MqFqXkrmQszJ1NOUuUu5u6Vqx/0lZSVNku7velL5FC/s5uEAj1lsBMhA==
   dependencies:
-    "@cspotcode/source-map-consumer" "0.8.0"
+    "@jridgewell/trace-mapping" "0.3.9"
 
 "@cypress/request@^2.88.10":
   version "2.88.10"
@@ -1182,6 +1177,24 @@
     "@types/yargs" "^16.0.0"
     chalk "^4.0.0"
 
+"@jridgewell/resolve-uri@^3.0.3":
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78"
+  integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==
+
+"@jridgewell/sourcemap-codec@^1.4.10":
+  version "1.4.14"
+  resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24"
+  integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==
+
+"@jridgewell/trace-mapping@0.3.9":
+  version "0.3.9"
+  resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz#6534fd5933a53ba7cbf3a17615e273a0d1273ff9"
+  integrity sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==
+  dependencies:
+    "@jridgewell/resolve-uri" "^3.0.3"
+    "@jridgewell/sourcemap-codec" "^1.4.10"
+
 "@nodelib/fs.scandir@2.1.5":
   version "2.1.5"
   resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
@@ -5935,12 +5948,12 @@ tr46@~0.0.3:
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
   integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=
 
-ts-node@^10.4.0:
+ts-node@10.8.1:
-  version "10.4.0"
+  version "10.8.1"
-  resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.4.0.tgz#680f88945885f4e6cf450e7f0d6223dd404895f7"
+  resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.8.1.tgz#ea2bd3459011b52699d7e88daa55a45a1af4f066"
-  integrity sha512-g0FlPvvCXSIO1JDF6S232P5jPYqBkRL9qly81ZgAOSU7rwI0stphCgd2kLiCrU9DjQCrJMWEqcNSjQL02s6d8A==
+  integrity sha512-Wwsnao4DQoJsN034wePSg5nZiw4YKXf56mPIAeD6wVmiv+RytNSWqc2f3fKvcUoV+Yn2+yocD71VOfQHbmVX4g==
   dependencies:
-    "@cspotcode/source-map-support" "0.7.0"
+    "@cspotcode/source-map-support" "^0.8.0"
     "@tsconfig/node10" "^1.0.7"
     "@tsconfig/node12" "^1.0.7"
     "@tsconfig/node14" "^1.0.0"
@@ -5951,6 +5964,7 @@ ts-node@^10.4.0:
     create-require "^1.1.0"
     diff "^4.0.1"
     make-error "^1.1.1"
+    v8-compile-cache-lib "^3.0.1"
     yn "3.1.1"
 
 tsconfig-paths@4.0.0:
@@ -6023,10 +6037,10 @@ typedarray-to-buffer@^3.1.5:
   dependencies:
     is-typedarray "^1.0.0"
 
-typescript@^4.5.5:
+typescript@4.7.3:
-  version "4.5.5"
+  version "4.7.3"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.5.tgz#d8c953832d28924a9e3d37c73d729c846c5896f3"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.3.tgz#8364b502d5257b540f9de4c40be84c98e23a129d"
-  integrity sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA==
+  integrity sha512-WOkT3XYvrpXx4vMMqlD+8R8R37fZkjyLGlxavMc4iB8lrl8L0DeTcHbYgw/v0N/z9wAFsgBhcsF0ruoySS22mA==
 
 unicode-canonical-property-names-ecmascript@^2.0.0:
   version "2.0.0"
@@ -6111,6 +6125,11 @@ uuid@^8.3.0, uuid@^8.3.2:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
   integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
 
+v8-compile-cache-lib@^3.0.1:
+  version "3.0.1"
+  resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf"
+  integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==
+
 v8-to-istanbul@^7.0.0:
   version "7.1.2"
   resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-7.1.2.tgz#30898d1a7fa0c84d225a2c1434fb958f290883c1"

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "2.1.22-alpha.2",
+  "version": "2.1.22-alpha.8",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -26,9 +26,9 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "2.1.22-alpha.2",
+    "@budibase/backend-core": "2.1.22-alpha.8",
-    "@budibase/string-templates": "2.1.22-alpha.2",
+    "@budibase/string-templates": "2.1.22-alpha.8",
-    "@budibase/types": "2.1.22-alpha.2",
+    "@budibase/types": "2.1.22-alpha.8",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "2.1.22-alpha.2",
+  "version": "2.1.22-alpha.8",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "2.1.22-alpha.2",
+    "@budibase/bbui": "2.1.22-alpha.8",
-    "@budibase/frontend-core": "2.1.22-alpha.2",
+    "@budibase/frontend-core": "2.1.22-alpha.8",
-    "@budibase/string-templates": "2.1.22-alpha.2",
+    "@budibase/string-templates": "2.1.22-alpha.8",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "2.1.22-alpha.2",
+  "version": "2.1.22-alpha.8",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "2.1.22-alpha.2",
+    "@budibase/bbui": "2.1.22-alpha.8",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/sdk",
-  "version": "2.1.22-alpha.2",
+  "version": "2.1.22-alpha.8",
   "description": "Budibase Public API SDK",
   "author": "Budibase",
   "license": "MPL-2.0",