Merge branch 'feature/audit-logs' of github.com:Budibase/budibase into feature/audit-logs

.github/workflows/budibase_ci.yml

@@ -11,7 +11,6 @@ on:
     branches:
       - master
       - develop 
-      - release 
  workflow_dispatch:
 
 env:
@@ -20,9 +19,53 @@ env:
   PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
 
 jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Use Node.js 14.x
+      uses: actions/setup-node@v1
+      with:
+        node-version: 14.x
+    - run: yarn
+    - run: yarn lint
+
   build:
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - name: Install Pro
+        run: yarn install:pro $BRANCH $BASE_BRANCH
+      - run: yarn
+      - run: yarn bootstrap
+      - run: yarn build
 
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - name: Install Pro
+        run: yarn install:pro $BRANCH $BASE_BRANCH
+      - run: yarn
+      - run: yarn bootstrap
+      - run: yarn test
+      - uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
+          files: ./packages/server/coverage/clover.xml,./packages/worker/coverage/clover.xml,./packages/backend-core/coverage/clover.xml
+          name: codecov-umbrella
+          verbose: true
+
+  integration-test:
+    runs-on: ubuntu-latest
     services:
       couchdb:
         image: ibmcom/couchdb3
@@ -31,39 +74,18 @@ jobs:
           COUCHDB_USER: budibase
         ports:
           - 4567:5984
-
-    strategy:
-      matrix:
-        node-version: [14.x]
-
     steps:
-    - uses: actions/checkout@v2
+      - uses: actions/checkout@v2
-
+      - name: Use Node.js 14.x
-    - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
-      uses: actions/setup-node@v1
+        with:
-      with:
+          node-version: 14.x
-        node-version: ${{ matrix.node-version }}
+      - name: Install Pro
-
+        run: yarn install:pro $BRANCH $BASE_BRANCH
-    - name: Install Pro
+      - run: yarn
-      run: yarn install:pro $BRANCH $BASE_BRANCH
+      - run: yarn bootstrap
-
+      - run: yarn build
-    - run: yarn
+      - run: |
-    - run: yarn bootstrap
+          cd qa-core
-    - run: yarn lint
+          yarn
-    - run: yarn build
+          yarn api:test:ci
-    - run: yarn test
-      env:
-        CI: true
-        name: Budibase CI
-    - uses: codecov/codecov-action@v1
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
-        files: ./packages/server/coverage/clover.xml,./packages/worker/coverage/clover.xml,./packages/backend-core/coverage/clover.xml
-        name: codecov-umbrella
-        verbose: true 
-
-    - name: QA Core Integration Tests
-      run: | 
-        cd qa-core
-        yarn
-        yarn api:test:ci

.github/workflows/release-develop.yml

@@ -45,10 +45,9 @@ jobs:
 
       - run: yarn 
       - run: yarn bootstrap
-      - run: yarn lint 
       - run: yarn build
       - run: yarn build:sdk
-      - run: yarn test
+#      - run: yarn test
 
       - name: Publish budibase packages to NPM
         env:
@@ -194,5 +193,5 @@ jobs:
           PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }}
         with:
           repository: budibase/budibase-deploys
-          event: deploy-budibase-develop-to-qa
+          event: budicloud-qa-deploy
           github_pat: ${{ secrets.GH_ACCESS_TOKEN }}

.github/workflows/smoke_test.yaml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   nightly:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, qa]
 
     steps:
       - uses: actions/checkout@v2
@@ -15,30 +15,17 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: 14.x
-      - run: yarn
+      - name: QA Core Integration Tests
-      - run: yarn bootstrap
-      - run: yarn build
-      - name: Pull  from budibase-infra
         run: | 
-          curl -H "Authorization: token ${{ secrets.GH_PERSONAL_TOKEN }}" \
+          cd qa-core
-          -H 'Accept: application/vnd.github.v3.raw' \
+          yarn
-          -o 
+          yarn api:test:ci
-          -L 
+        env:
-          wc -l 
+          BUDIBASE_HOST: budicloud.qa.budibase.net
-      
+          BUDIBASE_ACCOUNTS_URL: https://account-portal.budicloud.qa.budibase.net
-      - uses: actions/upload-artifact@v3
-        with:
-          name: Test Reports
-          path: 
 
-      # TODO: enable once running in QA test env
+      - name: Cypress Discord Notify
-      # - name: Configure AWS Credentials
+        run: yarn test:notify
-      #   uses: aws-actions/configure-aws-credentials@v1
+        env:
-      #   with:
+          WEBHOOK_URL: ${{ secrets.BUDI_QA_WEBHOOK }}
-      #     aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          GITHUB_RUN_URL: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID
-      #     aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      #     aws-region: eu-west-1
-      
-      # - name: Upload test results HTML
-      #   uses: aws-actions/configure-aws-credentials@v1
-      #   run: aws s3 cp packages/builder/cypress/reports/testReport.html s3://{{ secrets.BUDI_QA_REPORTS_BUCKET_NAME }}/$GITHUB_RUN_ID/index.html

.husky/pre-commit

@@ -1,4 +1,2 @@
 #!/bin/sh
 . "$(dirname "$0")/_/husky.sh"
-
-yarn run lint

charts/budibase/templates/app-service-deployment.yaml

@@ -4,9 +4,15 @@ metadata:
   annotations:
     kompose.cmd: kompose convert
     kompose.version: 1.21.0 (992df58d8)
+{{ if .Values.services.apps.deploymentAnnotations }}
+{{- toYaml .Values.services.apps.deploymentAnnotations | indent 4 -}}
+{{ end }}
   creationTimestamp: null
   labels:
     io.kompose.service: app-service
+{{ if .Values.services.apps.deploymentLabels }}
+{{- toYaml .Values.services.apps.deploymentLabels | indent 4 -}}
+{{ end }}
   name: app-service
 spec:
   replicas: {{ .Values.services.apps.replicaCount }}
@@ -20,12 +26,15 @@ spec:
       annotations:
         kompose.cmd: kompose convert
         kompose.version: 1.21.0 (992df58d8)
-{{ if .Values.services.apps.annotations }}
+{{ if .Values.services.apps.templateAnnotations }}
-{{- toYaml .Values.services.apps.annotations | indent 8 -}}
+{{- toYaml .Values.services.apps.templateAnnotations | indent 8 -}}
 {{ end }}
       creationTimestamp: null
       labels:
         io.kompose.service: app-service
+{{ if .Values.services.apps.templateLabels }}
+{{- toYaml .Values.services.apps.templateLabels | indent 8 -}}
+{{ end }}
     spec:
       containers:
       - env:
@@ -157,6 +166,14 @@ spec:
         - name: NODE_DEBUG
           value: {{ .Values.services.apps.nodeDebug | quote }}
         {{ end }}
+        {{ if .Values.globals.datadogApmEnabled }}
+        - name: DD_LOGS_INJECTION
+          value: {{ .Values.globals.datadogApmEnabled | quote }}
+        - name: DD_APM_ENABLED
+          value: {{ .Values.globals.datadogApmEnabled | quote }}
+        - name: DD_APM_DD_URL
+          value: https://trace.agent.datadoghq.eu
+        {{ end }}
         {{ if .Values.globals.elasticApmEnabled }}
         - name: ELASTIC_APM_ENABLED
           value: {{ .Values.globals.elasticApmEnabled | quote }}

charts/budibase/templates/proxy-service-deployment.yaml

@@ -4,9 +4,15 @@ metadata:
   annotations:
     kompose.cmd: kompose convert
     kompose.version: 1.21.0 (992df58d8)
+{{ if .Values.services.proxy.deploymentAnnotations }}
+{{- toYaml .Values.services.proxy.deploymentAnnotations | indent 4 -}}
+{{ end }}
   creationTimestamp: null
   labels:
     app.kubernetes.io/name: budibase-proxy
+{{ if .Values.services.proxy.deploymentLabels }}
+{{- toYaml .Values.services.proxy.deploymentLabels | indent 4 -}}
+{{ end }}
   name: proxy-service
 spec:
   replicas: {{ .Values.services.proxy.replicaCount }}
@@ -20,12 +26,15 @@ spec:
       annotations:
         kompose.cmd: kompose convert
         kompose.version: 1.21.0 (992df58d8)
-{{ if .Values.services.proxy.annotations }}
+{{ if .Values.services.proxy.templateAnnotations }}
-{{- toYaml .Values.services.proxy.annotations | indent 8 -}}
+{{- toYaml .Values.services.proxy.templateAnnotations | indent 8 -}}
 {{ end }}
       creationTimestamp: null
       labels:
         app.kubernetes.io/name: budibase-proxy
+{{ if .Values.services.proxy.templateLabels }}
+{{- toYaml .Values.services.proxy.templateLabels | indent 8 -}}
+{{ end }}
     spec:
       containers:
       - image: budibase/proxy:{{ .Values.globals.appVersion }}

charts/budibase/templates/worker-service-deployment.yaml

@@ -4,13 +4,18 @@ metadata:
   annotations:
     kompose.cmd: kompose convert
     kompose.version: 1.21.0 (992df58d8)
+{{ if .Values.services.worker.deploymentAnnotations }}
+{{- toYaml .Values.services.worker.deploymentAnnotations | indent 4 -}}
+{{ end }}
   creationTimestamp: null
   labels:
     io.kompose.service: worker-service
+{{ if .Values.services.worker.deploymentLabels }}
+{{- toYaml .Values.services.worker.deploymentLabels | indent 4 -}}
+{{ end }}
   name: worker-service
 spec:
   replicas: {{ .Values.services.worker.replicaCount }}
-
   selector:
     matchLabels:
       io.kompose.service: worker-service
@@ -21,12 +26,15 @@ spec:
       annotations:
         kompose.cmd: kompose convert
         kompose.version: 1.21.0 (992df58d8)
-{{ if .Values.services.worker.annotations }}
+{{ if .Values.services.worker.templateAnnotations }}
-{{- toYaml .Values.services.worker.annotations | indent 8 -}}
+{{- toYaml .Values.services.worker.templateAnnotations | indent 8 -}}
 {{ end }}
       creationTimestamp: null
       labels:
         io.kompose.service: worker-service
+{{ if .Values.services.worker.templateLabels }}
+{{- toYaml .Values.services.worker.templateLabels | indent 8 -}}
+{{ end }}
     spec:
       containers:
       - env:
@@ -148,6 +156,14 @@ spec:
           value: {{ .Values.globals.tenantFeatureFlags | quote }}
         - name: ENCRYPTION_KEY
           value: {{ .Values.globals.bbEncryptionKey | quote }}
+        {{ if .Values.globals.datadogApmEnabled }}
+        - name: DD_LOGS_INJECTION
+          value: {{ .Values.globals.datadogApmEnabled | quote }}
+        - name: DD_APM_ENABLED
+          value: {{ .Values.globals.datadogApmEnabled | quote }}
+        - name: DD_APM_DD_URL
+          value: https://trace.agent.datadoghq.eu
+        {{ end }}
         {{ if .Values.globals.elasticApmEnabled }}
         - name: ELASTIC_APM_ENABLED
           value: {{ .Values.globals.elasticApmEnabled | quote }}

hosting/proxy/nginx.prod.conf

@@ -55,7 +55,7 @@ http {
     set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
     set $csp_object "object-src 'none'";
     set $csp_base_uri "base-uri 'self'";
-    set $csp_connect "connect-src 'self' https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com";
+    set $csp_connect "connect-src 'self' https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.s3.*.amazonaws.com https://s3.*.amazonaws.com https://api.github.com";
     set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com  https://js.intercomcdn.com  https://fonts.intercomcdn.com";
     set $csp_frame "frame-src 'self' https:";
     set $csp_img "img-src http: https: data: blob:";

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.3.14-alpha.0",
+  "version": "2.3.18-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/jest.config.ts

@@ -9,15 +9,9 @@ const baseConfig: Config.InitialProjectOptions = {
   transform: {
     "^.+\\.ts?$": "@swc/jest",
   },
-}
+  moduleNameMapper: {
-
-if (!process.env.CI) {
-  // use sources when not in CI
-  baseConfig.moduleNameMapper = {
     "@budibase/types": "<rootDir>/../types/src",
-  }
+  },
-} else {
-  console.log("Running tests with compiled dependency sources")
 }
 
 const config: Config.InitialOptions = {

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.3.14-alpha.0",
+  "version": "2.3.18-alpha.0",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -18,13 +18,13 @@
     "build:pro": "../../scripts/pro/build.sh",
     "postbuild": "yarn run build:pro",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
-    "test": "jest --coverage --maxWorkers=2",
+    "test": "jest --coverage --runInBand",
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
     "@budibase/nano": "10.1.1",
     "@budibase/pouchdb-replication-stream": "1.2.10",
-    "@budibase/types": "2.3.14-alpha.0",
+    "@budibase/types": "2.3.18-alpha.0",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",
@@ -62,7 +62,7 @@
     "@trendyol/jest-testcontainers": "^2.1.1",
     "@types/chance": "1.1.3",
     "@types/ioredis": "4.28.0",
-    "@types/jest": "27.5.1",
+    "@types/jest": "28.1.1",
     "@types/koa": "2.13.4",
     "@types/koa-pino-logger": "3.0.0",
     "@types/lodash": "4.14.180",

packages/backend-core/src/accounts/accounts.ts

@@ -1,13 +1,24 @@
 import API from "./api"
 import env from "../environment"
 import { Header } from "../constants"
-import { CloudAccount } from "@budibase/types"
+import { CloudAccount, HealthStatusResponse } from "@budibase/types"
 
 const api = new API(env.ACCOUNT_PORTAL_URL)
 
+/**
+ * This client is intended to be used in a cloud hosted deploy only.
+ * Rather than relying on each consumer to perform the necessary environmental checks
+ * we use the following check to exit early with a undefined response which should be
+ * handled by the caller.
+ */
+const EXIT_EARLY = env.SELF_HOSTED || env.DISABLE_ACCOUNT_PORTAL
+
 export const getAccount = async (
   email: string
 ): Promise<CloudAccount | undefined> => {
+  if (EXIT_EARLY) {
+    return
+  }
   const payload = {
     email,
   }
@@ -29,6 +40,9 @@ export const getAccount = async (
 export const getAccountByTenantId = async (
   tenantId: string
 ): Promise<CloudAccount | undefined> => {
+  if (EXIT_EARLY) {
+    return
+  }
   const payload = {
     tenantId,
   }
@@ -47,7 +61,12 @@ export const getAccountByTenantId = async (
   return json[0]
 }
 
-export const getStatus = async () => {
+export const getStatus = async (): Promise<
+  HealthStatusResponse | undefined
+> => {
+  if (EXIT_EARLY) {
+    return
+  }
   const response = await api.get(`/api/status`, {
     headers: {
       [Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,

packages/backend-core/src/accounts/index.ts

@@ -0,0 +1 @@
+export * from "./accounts"

packages/backend-core/src/auth/auth.ts

@@ -1,10 +1,11 @@
 const _passport = require("koa-passport")
 const LocalStrategy = require("passport-local").Strategy
 const JwtStrategy = require("passport-jwt").Strategy
-import { getGlobalDB } from "../tenancy"
+import { getGlobalDB } from "../context"
 const refresh = require("passport-oauth2-refresh")
-import { Config } from "../constants"
+import { Config, Cookie } from "../constants"
 import { getScopedConfig } from "../db"
+import { getSessionsForUser, invalidateSessions } from "../security/sessions"
 import {
   jwt as jwtPassport,
   local,
@@ -15,8 +16,11 @@ import {
   google,
 } from "../middleware"
 import { invalidateUser } from "../cache/user"
-import { User } from "@budibase/types"
+import { PlatformLogoutOpts, User } from "@budibase/types"
 import { logAlert } from "../logging"
+import * as events from "../events"
+import * as userCache from "../cache/user"
+import { clearCookie, getCookie } from "../utils"
 export {
   auditLog,
   authError,
@@ -29,6 +33,7 @@ export {
   google,
   oidc,
 } from "../middleware"
+import { ssoSaveUserNoOp } from "../middleware/passport/sso/sso"
 export const buildAuthMiddleware = authenticated
 export const buildTenancyMiddleware = tenancy
 export const buildCsrfMiddleware = csrf
@@ -71,7 +76,7 @@ async function refreshOIDCAccessToken(
     if (!enrichedConfig) {
       throw new Error("OIDC Config contents invalid")
     }
-    strategy = await oidc.strategyFactory(enrichedConfig)
+    strategy = await oidc.strategyFactory(enrichedConfig, ssoSaveUserNoOp)
   } catch (err) {
     console.error(err)
     throw new Error("Could not refresh OAuth Token")
@@ -103,7 +108,11 @@ async function refreshGoogleAccessToken(
 
   let strategy
   try {
-    strategy = await google.strategyFactory(config, callbackUrl)
+    strategy = await google.strategyFactory(
+      config,
+      callbackUrl,
+      ssoSaveUserNoOp
+    )
   } catch (err: any) {
     console.error(err)
     throw new Error(
@@ -161,6 +170,8 @@ export async function refreshOAuthToken(
   return refreshResponse
 }
 
+// TODO: Refactor to use user save function instead to prevent the need for
+// manually saving and invalidating on callback
 export async function updateUserOAuth(userId: string, oAuthConfig: any) {
   const details = {
     accessToken: oAuthConfig.accessToken,
@@ -188,3 +199,32 @@ export async function updateUserOAuth(userId: string, oAuthConfig: any) {
     console.error("Could not update OAuth details for current user", e)
   }
 }
+
+/**
+ * Logs a user out from budibase. Re-used across account portal and builder.
+ */
+export async function platformLogout(opts: PlatformLogoutOpts) {
+  const ctx = opts.ctx
+  const userId = opts.userId
+  const keepActiveSession = opts.keepActiveSession
+
+  if (!ctx) throw new Error("Koa context must be supplied to logout.")
+
+  const currentSession = getCookie(ctx, Cookie.Auth)
+  let sessions = await getSessionsForUser(userId)
+
+  if (keepActiveSession) {
+    sessions = sessions.filter(
+      session => session.sessionId !== currentSession.sessionId
+    )
+  } else {
+    // clear cookies
+    clearCookie(ctx, Cookie.Auth)
+    clearCookie(ctx, Cookie.CurrentApp)
+  }
+
+  const sessionIds = sessions.map(({ sessionId }) => sessionId)
+  await invalidateSessions(userId, { sessionIds, reason: "logout" })
+  await events.auth.logout()
+  await userCache.invalidateUser(userId)
+}

packages/backend-core/src/auth/tests/auth.spec.ts

@@ -0,0 +1,13 @@
+import { structures, testEnv } from "../../../tests"
+import * as auth from "../auth"
+import * as events from "../../events"
+
+describe("platformLogout", () => {
+  it("should call platform logout", async () => {
+    await testEnv.withTenant(async () => {
+      const ctx = structures.koa.newContext()
+      await auth.platformLogout({ ctx, userId: "test" })
+      expect(events.auth.logout).toBeCalledTimes(1)
+    })
+  })
+})

packages/backend-core/src/cache/tests/writethrough.spec.js

@@ -1,61 +0,0 @@
-require("../../../tests")
-const { Writethrough } = require("../writethrough")
-const { getDB } = require("../../db")
-const tk = require("timekeeper")
-const { structures } = require("../../../tests")
-
-const START_DATE = Date.now()
-tk.freeze(START_DATE)
-
-
-const DELAY = 5000
-
-const db = getDB(structures.db.id())
-const db2 = getDB(structures.db.id())
-const writethrough = new Writethrough(db, DELAY), writethrough2 = new Writethrough(db2, DELAY)
-
-describe("writethrough", () => {
-  describe("put", () => {
-    let first
-    it("should be able to store, will go to DB", async () => {
-      const response = await writethrough.put({ _id: "test", value: 1 })
-      const output = await db.get(response.id)
-      first = output
-      expect(output.value).toBe(1)
-    })
-
-    it("second put shouldn't update DB", async () => {
-      const response = await writethrough.put({ ...first, value: 2 })
-      const output = await db.get(response.id)
-      expect(first._rev).toBe(output._rev)
-      expect(output.value).toBe(1)
-    })
-
-    it("should put it again after delay period", async () => {
-      tk.freeze(START_DATE + DELAY + 1)
-      const response = await writethrough.put({ ...first, value: 3 })
-      const output = await db.get(response.id)
-      expect(response.rev).not.toBe(first._rev)
-      expect(output.value).toBe(3)
-    })
-  })
-
-  describe("get", () => {
-    it("should be able to retrieve", async () => {
-      const response = await writethrough.get("test")
-      expect(response.value).toBe(3)
-    })
-  })
-
-  describe("same doc, different databases (tenancy)", () => {
-    it("should be able to two different databases", async () => {
-      const resp1 = await writethrough.put({ _id: "db1", value: "first" })
-      const resp2 = await writethrough2.put({ _id: "db1", value: "second" })
-      expect(resp1.rev).toBeDefined()
-      expect(resp2.rev).toBeDefined()
-      expect((await db.get("db1")).value).toBe("first")
-      expect((await db2.get("db1")).value).toBe("second")
-    })
-  })
-})
-

packages/backend-core/src/cache/tests/writethrough.spec.ts

@@ -0,0 +1,73 @@
+import { structures, DBTestConfiguration } from "../../../tests"
+import { Writethrough } from "../writethrough"
+import { getDB } from "../../db"
+import tk from "timekeeper"
+
+const START_DATE = Date.now()
+tk.freeze(START_DATE)
+
+const DELAY = 5000
+
+describe("writethrough", () => {
+  const config = new DBTestConfiguration()
+
+  const db = getDB(structures.db.id())
+  const db2 = getDB(structures.db.id())
+
+  const writethrough = new Writethrough(db, DELAY)
+  const writethrough2 = new Writethrough(db2, DELAY)
+
+  describe("put", () => {
+    let first: any
+
+    it("should be able to store, will go to DB", async () => {
+      await config.doInTenant(async () => {
+        const response = await writethrough.put({ _id: "test", value: 1 })
+        const output = await db.get(response.id)
+        first = output
+        expect(output.value).toBe(1)
+      })
+    })
+
+    it("second put shouldn't update DB", async () => {
+      await config.doInTenant(async () => {
+        const response = await writethrough.put({ ...first, value: 2 })
+        const output = await db.get(response.id)
+        expect(first._rev).toBe(output._rev)
+        expect(output.value).toBe(1)
+      })
+    })
+
+    it("should put it again after delay period", async () => {
+      await config.doInTenant(async () => {
+        tk.freeze(START_DATE + DELAY + 1)
+        const response = await writethrough.put({ ...first, value: 3 })
+        const output = await db.get(response.id)
+        expect(response.rev).not.toBe(first._rev)
+        expect(output.value).toBe(3)
+      })
+    })
+  })
+
+  describe("get", () => {
+    it("should be able to retrieve", async () => {
+      await config.doInTenant(async () => {
+        const response = await writethrough.get("test")
+        expect(response.value).toBe(3)
+      })
+    })
+  })
+
+  describe("same doc, different databases (tenancy)", () => {
+    it("should be able to two different databases", async () => {
+      await config.doInTenant(async () => {
+        const resp1 = await writethrough.put({ _id: "db1", value: "first" })
+        const resp2 = await writethrough2.put({ _id: "db1", value: "second" })
+        expect(resp1.rev).toBeDefined()
+        expect(resp2.rev).toBeDefined()
+        expect((await db.get("db1")).value).toBe("first")
+        expect((await db2.get("db1")).value).toBe("second")
+      })
+    })
+  })
+})

packages/backend-core/src/cache/user.ts

@@ -1,8 +1,9 @@
 import * as redis from "../redis/init"
-import { getTenantId, lookupTenantId, doWithGlobalDB } from "../tenancy"
+import * as tenancy from "../tenancy"
+import * as context from "../context"
+import * as platform from "../platform"
 import env from "../environment"
-import * as accounts from "../cloud/accounts"
+import * as accounts from "../accounts"
-import { Database } from "@budibase/types"
 
 const EXPIRY_SECONDS = 3600
 
@@ -10,7 +11,8 @@ const EXPIRY_SECONDS = 3600
  * The default populate user function
  */
 async function populateFromDB(userId: string, tenantId: string) {
-  const user = await doWithGlobalDB(tenantId, (db: Database) => db.get(userId))
+  const db = tenancy.getTenantDB(tenantId)
+  const user = await db.get(userId)
   user.budibaseAccess = true
   if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     const account = await accounts.getAccount(user.email)
@@ -42,9 +44,9 @@ export async function getUser(
   }
   if (!tenantId) {
     try {
-      tenantId = getTenantId()
+      tenantId = context.getTenantId()
     } catch (err) {
-      tenantId = await lookupTenantId(userId)
+      tenantId = await platform.users.lookupTenantId(userId)
     }
   }
   const client = await redis.getUserClient()

packages/backend-core/src/context/deprovision.ts

@@ -1,108 +0,0 @@
-import {
-  getGlobalUserParams,
-  getAllApps,
-  doWithDB,
-  StaticDatabases,
-} from "../db"
-import { doWithGlobalDB } from "../tenancy"
-import { App, Tenants, User, Database } from "@budibase/types"
-
-const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
-const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
-
-async function removeTenantFromInfoDB(tenantId: string) {
-  try {
-    await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => {
-      const tenants = (await infoDb.get(TENANT_DOC)) as Tenants
-      tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)
-
-      await infoDb.put(tenants)
-    })
-  } catch (err) {
-    console.error(`Error removing tenant ${tenantId} from info db`, err)
-    throw err
-  }
-}
-
-export async function removeUserFromInfoDB(dbUser: User) {
-  await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => {
-    const keys = [dbUser._id!, dbUser.email]
-    const userDocs = await infoDb.allDocs({
-      keys,
-      include_docs: true,
-    })
-    const toDelete = userDocs.rows.map((row: any) => {
-      return {
-        ...row.doc,
-        _deleted: true,
-      }
-    })
-    await infoDb.bulkDocs(toDelete)
-  })
-}
-
-async function removeUsersFromInfoDB(tenantId: string) {
-  return doWithGlobalDB(tenantId, async (db: any) => {
-    try {
-      const allUsers = await db.allDocs(
-        getGlobalUserParams(null, {
-          include_docs: true,
-        })
-      )
-      await doWithDB(PLATFORM_INFO_DB, async (infoDb: any) => {
-        const allEmails = allUsers.rows.map((row: any) => row.doc.email)
-        // get the id docs
-        let keys = allUsers.rows.map((row: any) => row.id)
-        // and the email docs
-        keys = keys.concat(allEmails)
-        // retrieve the docs and delete them
-        const userDocs = await infoDb.allDocs({
-          keys,
-          include_docs: true,
-        })
-        const toDelete = userDocs.rows.map((row: any) => {
-          return {
-            ...row.doc,
-            _deleted: true,
-          }
-        })
-        await infoDb.bulkDocs(toDelete)
-      })
-    } catch (err) {
-      console.error(`Error removing tenant ${tenantId} users from info db`, err)
-      throw err
-    }
-  })
-}
-
-async function removeGlobalDB(tenantId: string) {
-  return doWithGlobalDB(tenantId, async (db: Database) => {
-    try {
-      await db.destroy()
-    } catch (err) {
-      console.error(`Error removing tenant ${tenantId} users from info db`, err)
-      throw err
-    }
-  })
-}
-
-async function removeTenantApps(tenantId: string) {
-  try {
-    const apps = (await getAllApps({ all: true })) as App[]
-    const destroyPromises = apps.map(app =>
-      doWithDB(app.appId, (db: Database) => db.destroy())
-    )
-    await Promise.allSettled(destroyPromises)
-  } catch (err) {
-    console.error(`Error removing tenant ${tenantId} apps`, err)
-    throw err
-  }
-}
-
-// can't live in tenancy package due to circular dependency on db/utils
-export async function deleteTenant(tenantId: string) {
-  await removeTenantFromInfoDB(tenantId)
-  await removeUsersFromInfoDB(tenantId)
-  await removeGlobalDB(tenantId)
-  await removeTenantApps(tenantId)
-}

packages/backend-core/src/context/identity.ts

@@ -5,6 +5,8 @@ import {
   isCloudAccount,
   Account,
   AccountUserContext,
+  UserContext,
+  Ctx,
 } from "@budibase/types"
 import * as context from "."
 
@@ -16,15 +18,22 @@ export function doInIdentityContext(identity: IdentityContext, task: any) {
   return context.doInIdentityContext(identity, task)
 }
 
-export function doInUserContext(user: User, task: any) {
+// used in server/worker
-  const userContext: any = {
+export function doInUserContext(user: User, ctx: Ctx, task: any) {
+  const userContext: UserContext = {
     ...user,
     _id: user._id as string,
     type: IdentityType.USER,
+    hostInfo: {
+      ipAddress: ctx.request.ip,
+      // filled in by koa-useragent package
+      userAgent: ctx.userAgent._agent.source,
+    },
   }
   return doInIdentityContext(userContext, task)
 }
 
+// used in account portal
 export function doInAccountContext(account: Account, task: any) {
   const _id = getAccountUserId(account)
   const tenantId = account.tenantId

packages/backend-core/src/context/tests/index.spec.ts

@@ -1,11 +1,14 @@
-require("../../../tests")
+import { testEnv } from "../../../tests"
 const context = require("../")
 const { DEFAULT_TENANT_ID } = require("../../constants")
-import env from "../../environment"
 
 describe("context", () => {
   describe("doInTenant", () => {
     describe("single-tenancy", () => {
+      beforeAll(() => {
+        testEnv.singleTenant()
+      })
+
       it("defaults to the default tenant", () => {
         const tenantId = context.getTenantId()
         expect(tenantId).toBe(DEFAULT_TENANT_ID)
@@ -20,8 +23,8 @@ describe("context", () => {
     })
 
     describe("multi-tenancy", () => {
-      beforeEach(() => {
+      beforeAll(() => {
-        env._set("MULTI_TENANCY", 1)
+        testEnv.multiTenant()
       })
 
       it("fails when no tenant id is set", () => {

packages/backend-core/src/db/db.ts

@@ -1,7 +1,6 @@
 import env from "../environment"
-import { directCouchQuery, getPouchDB } from "./couch"
+import { directCouchQuery, DatabaseImpl } from "./couch"
 import { CouchFindOptions, Database } from "@budibase/types"
-import { DatabaseImpl } from "../db"
 
 const dbList = new Set()
 

packages/backend-core/src/db/tests/utils.seq.spec.ts

@@ -1,190 +0,0 @@
-require("../../../tests")
-const {
-  getDevelopmentAppID,
-  getProdAppID,
-  isDevAppID,
-  isProdAppID,
-} = require("../conversions")
-const { generateAppID, getPlatformUrl, getScopedConfig } = require("../utils")
-const tenancy = require("../../tenancy")
-const { Config, DEFAULT_TENANT_ID } = require("../../constants")
-import { generator } from "../../../tests"
-import env from "../../environment"
-
-describe("utils", () => {
-  describe("app ID manipulation", () => {
-    function getID() {
-      const appId = generateAppID()
-      const split = appId.split("_")
-      const uuid = split[split.length - 1]
-      const devAppId = `app_dev_${uuid}`
-      return { appId, devAppId, split, uuid }
-    }
-
-    it("should be able to generate a new app ID", () => {
-      expect(generateAppID().startsWith("app_")).toEqual(true)
-    })
-
-    it("should be able to convert a production app ID to development", () => {
-      const { appId, uuid } = getID()
-      expect(getDevelopmentAppID(appId)).toEqual(`app_dev_${uuid}`)
-    })
-
-    it("should be able to convert a development app ID to development", () => {
-      const { devAppId, uuid } = getID()
-      expect(getDevelopmentAppID(devAppId)).toEqual(`app_dev_${uuid}`)
-    })
-
-    it("should be able to convert a development ID to a production", () => {
-      const { devAppId, uuid } = getID()
-      expect(getProdAppID(devAppId)).toEqual(`app_${uuid}`)
-    })
-
-    it("should be able to convert a production ID to production", () => {
-      const { appId, uuid } = getID()
-      expect(getProdAppID(appId)).toEqual(`app_${uuid}`)
-    })
-
-    it("should be able to confirm dev app ID is development", () => {
-      const { devAppId } = getID()
-      expect(isDevAppID(devAppId)).toEqual(true)
-    })
-
-    it("should be able to confirm prod app ID is not development", () => {
-      const { appId } = getID()
-      expect(isDevAppID(appId)).toEqual(false)
-    })
-
-    it("should be able to confirm prod app ID is prod", () => {
-      const { appId } = getID()
-      expect(isProdAppID(appId)).toEqual(true)
-    })
-
-    it("should be able to confirm dev app ID is not prod", () => {
-      const { devAppId } = getID()
-      expect(isProdAppID(devAppId)).toEqual(false)
-    })
-  })
-})
-
-const DEFAULT_URL = "http://localhost:10000"
-const ENV_URL = "http://env.com"
-
-const setDbPlatformUrl = async (dbUrl: string) => {
-  const db = tenancy.getGlobalDB()
-  await db.put({
-    _id: "config_settings",
-    type: Config.SETTINGS,
-    config: {
-      platformUrl: dbUrl,
-    },
-  })
-}
-
-const clearSettingsConfig = async () => {
-  await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-    const db = tenancy.getGlobalDB()
-    try {
-      const config = await db.get("config_settings")
-      await db.remove("config_settings", config._rev)
-    } catch (e: any) {
-      if (e.status !== 404) {
-        throw e
-      }
-    }
-  })
-}
-
-describe("getPlatformUrl", () => {
-  describe("self host", () => {
-    beforeEach(async () => {
-      env._set("SELF_HOST", 1)
-      await clearSettingsConfig()
-    })
-
-    it("gets the default url", async () => {
-      await tenancy.doInTenant(null, async () => {
-        const url = await getPlatformUrl()
-        expect(url).toBe(DEFAULT_URL)
-      })
-    })
-
-    it("gets the platform url from the environment", async () => {
-      await tenancy.doInTenant(null, async () => {
-        env._set("PLATFORM_URL", ENV_URL)
-        const url = await getPlatformUrl()
-        expect(url).toBe(ENV_URL)
-      })
-    })
-
-    it("gets the platform url from the database", async () => {
-      await tenancy.doInTenant(null, async () => {
-        const dbUrl = generator.url()
-        await setDbPlatformUrl(dbUrl)
-        const url = await getPlatformUrl()
-        expect(url).toBe(dbUrl)
-      })
-    })
-  })
-
-  describe("cloud", () => {
-    const TENANT_AWARE_URL = "http://default.env.com"
-
-    beforeEach(async () => {
-      env._set("SELF_HOSTED", 0)
-      env._set("MULTI_TENANCY", 1)
-      env._set("PLATFORM_URL", ENV_URL)
-      await clearSettingsConfig()
-    })
-
-    it("gets the platform url from the environment without tenancy", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        const url = await getPlatformUrl({ tenantAware: false })
-        expect(url).toBe(ENV_URL)
-      })
-    })
-
-    it("gets the platform url from the environment with tenancy", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        const url = await getPlatformUrl()
-        expect(url).toBe(TENANT_AWARE_URL)
-      })
-    })
-
-    it("never gets the platform url from the database", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        await setDbPlatformUrl(generator.url())
-        const url = await getPlatformUrl()
-        expect(url).toBe(TENANT_AWARE_URL)
-      })
-    })
-  })
-})
-
-describe("getScopedConfig", () => {
-  describe("settings config", () => {
-    beforeEach(async () => {
-      env._set("SELF_HOSTED", 1)
-      env._set("PLATFORM_URL", "")
-      await clearSettingsConfig()
-    })
-
-    it("returns the platform url with an existing config", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        const dbUrl = generator.url()
-        await setDbPlatformUrl(dbUrl)
-        const db = tenancy.getGlobalDB()
-        const config = await getScopedConfig(db, { type: Config.SETTINGS })
-        expect(config.platformUrl).toBe(dbUrl)
-      })
-    })
-
-    it("returns the platform url without an existing config", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        const db = tenancy.getGlobalDB()
-        const config = await getScopedConfig(db, { type: Config.SETTINGS })
-        expect(config.platformUrl).toBe(DEFAULT_URL)
-      })
-    })
-  })
-})

packages/backend-core/src/db/tests/utils.spec.ts

@@ -0,0 +1,192 @@
+import { generator, DBTestConfiguration, testEnv } from "../../../tests"
+import {
+  getDevelopmentAppID,
+  getProdAppID,
+  isDevAppID,
+  isProdAppID,
+} from "../conversions"
+import { generateAppID, getPlatformUrl, getScopedConfig } from "../utils"
+import * as context from "../../context"
+import { Config } from "../../constants"
+import env from "../../environment"
+
+describe("utils", () => {
+  const config = new DBTestConfiguration()
+
+  describe("app ID manipulation", () => {
+    function getID() {
+      const appId = generateAppID()
+      const split = appId.split("_")
+      const uuid = split[split.length - 1]
+      const devAppId = `app_dev_${uuid}`
+      return { appId, devAppId, split, uuid }
+    }
+
+    it("should be able to generate a new app ID", () => {
+      expect(generateAppID().startsWith("app_")).toEqual(true)
+    })
+
+    it("should be able to convert a production app ID to development", () => {
+      const { appId, uuid } = getID()
+      expect(getDevelopmentAppID(appId)).toEqual(`app_dev_${uuid}`)
+    })
+
+    it("should be able to convert a development app ID to development", () => {
+      const { devAppId, uuid } = getID()
+      expect(getDevelopmentAppID(devAppId)).toEqual(`app_dev_${uuid}`)
+    })
+
+    it("should be able to convert a development ID to a production", () => {
+      const { devAppId, uuid } = getID()
+      expect(getProdAppID(devAppId)).toEqual(`app_${uuid}`)
+    })
+
+    it("should be able to convert a production ID to production", () => {
+      const { appId, uuid } = getID()
+      expect(getProdAppID(appId)).toEqual(`app_${uuid}`)
+    })
+
+    it("should be able to confirm dev app ID is development", () => {
+      const { devAppId } = getID()
+      expect(isDevAppID(devAppId)).toEqual(true)
+    })
+
+    it("should be able to confirm prod app ID is not development", () => {
+      const { appId } = getID()
+      expect(isDevAppID(appId)).toEqual(false)
+    })
+
+    it("should be able to confirm prod app ID is prod", () => {
+      const { appId } = getID()
+      expect(isProdAppID(appId)).toEqual(true)
+    })
+
+    it("should be able to confirm dev app ID is not prod", () => {
+      const { devAppId } = getID()
+      expect(isProdAppID(devAppId)).toEqual(false)
+    })
+  })
+
+  const DEFAULT_URL = "http://localhost:10000"
+  const ENV_URL = "http://env.com"
+
+  const setDbPlatformUrl = async (dbUrl: string) => {
+    const db = context.getGlobalDB()
+    await db.put({
+      _id: "config_settings",
+      type: Config.SETTINGS,
+      config: {
+        platformUrl: dbUrl,
+      },
+    })
+  }
+
+  const clearSettingsConfig = async () => {
+    await config.doInTenant(async () => {
+      const db = context.getGlobalDB()
+      try {
+        const config = await db.get("config_settings")
+        await db.remove("config_settings", config._rev)
+      } catch (e: any) {
+        if (e.status !== 404) {
+          throw e
+        }
+      }
+    })
+  }
+
+  describe("getPlatformUrl", () => {
+    describe("self host", () => {
+      beforeEach(async () => {
+        testEnv.selfHosted()
+        await clearSettingsConfig()
+      })
+
+      it("gets the default url", async () => {
+        await config.doInTenant(async () => {
+          const url = await getPlatformUrl()
+          expect(url).toBe(DEFAULT_URL)
+        })
+      })
+
+      it("gets the platform url from the environment", async () => {
+        await config.doInTenant(async () => {
+          env._set("PLATFORM_URL", ENV_URL)
+          const url = await getPlatformUrl()
+          expect(url).toBe(ENV_URL)
+        })
+      })
+
+      it("gets the platform url from the database", async () => {
+        await config.doInTenant(async () => {
+          const dbUrl = generator.url()
+          await setDbPlatformUrl(dbUrl)
+          const url = await getPlatformUrl()
+          expect(url).toBe(dbUrl)
+        })
+      })
+    })
+
+    describe("cloud", () => {
+      const TENANT_AWARE_URL = `http://${config.tenantId}.env.com`
+
+      beforeEach(async () => {
+        testEnv.cloudHosted()
+        testEnv.multiTenant()
+
+        env._set("PLATFORM_URL", ENV_URL)
+        await clearSettingsConfig()
+      })
+
+      it("gets the platform url from the environment without tenancy", async () => {
+        await config.doInTenant(async () => {
+          const url = await getPlatformUrl({ tenantAware: false })
+          expect(url).toBe(ENV_URL)
+        })
+      })
+
+      it("gets the platform url from the environment with tenancy", async () => {
+        await config.doInTenant(async () => {
+          const url = await getPlatformUrl()
+          expect(url).toBe(TENANT_AWARE_URL)
+        })
+      })
+
+      it("never gets the platform url from the database", async () => {
+        await config.doInTenant(async () => {
+          await setDbPlatformUrl(generator.url())
+          const url = await getPlatformUrl()
+          expect(url).toBe(TENANT_AWARE_URL)
+        })
+      })
+    })
+  })
+
+  describe("getScopedConfig", () => {
+    describe("settings config", () => {
+      beforeEach(async () => {
+        env._set("SELF_HOSTED", 1)
+        env._set("PLATFORM_URL", "")
+        await clearSettingsConfig()
+      })
+
+      it("returns the platform url with an existing config", async () => {
+        await config.doInTenant(async () => {
+          const dbUrl = generator.url()
+          await setDbPlatformUrl(dbUrl)
+          const db = context.getGlobalDB()
+          const config = await getScopedConfig(db, { type: Config.SETTINGS })
+          expect(config.platformUrl).toBe(dbUrl)
+        })
+      })
+
+      it("returns the platform url without an existing config", async () => {
+        await config.doInTenant(async () => {
+          const db = context.getGlobalDB()
+          const config = await getScopedConfig(db, { type: Config.SETTINGS })
+          expect(config.platformUrl).toBe(DEFAULT_URL)
+        })
+      })
+    })
+  })
+})

packages/backend-core/src/db/views.ts

@@ -1,13 +1,14 @@
 import {
-  DocumentType,
-  ViewName,
   DeprecatedViews,
+  DocumentType,
   SEPARATOR,
   StaticDatabases,
+  ViewName,
 } from "../constants"
 import { getGlobalDB } from "../context"
 import { doWithDB } from "./"
 import { Database, DatabaseQueryOpts } from "@budibase/types"
+import env from "../environment"
 
 const DESIGN_DB = "_design/database"
 
@@ -69,17 +70,6 @@ export const createNewUserEmailView = async () => {
   await createView(db, viewJs, ViewName.USER_BY_EMAIL)
 }
 
-export const createAccountEmailView = async () => {
-  const viewJs = `function(doc) {
-    if (doc._id.startsWith("${DocumentType.ACCOUNT_METADATA}${SEPARATOR}")) {
-      emit(doc.email.toLowerCase(), doc._id)
-    }
-  }`
-  await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
-    await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL)
-  })
-}
-
 export const createUserAppView = async () => {
   const db = getGlobalDB()
   const viewJs = `function(doc) {
@@ -113,17 +103,6 @@ export const createUserBuildersView = async () => {
   await createView(db, viewJs, ViewName.USER_BY_BUILDERS)
 }
 
-export const createPlatformUserView = async () => {
-  const viewJs = `function(doc) {
-    if (doc.tenantId) {
-      emit(doc._id.toLowerCase(), doc._id)
-    }
-  }`
-  await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
-    await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
-  })
-}
-
 export interface QueryViewOptions {
   arrayResponse?: boolean
 }
@@ -162,13 +141,48 @@ export const queryView = async <T>(
   }
 }
 
+// PLATFORM
+
+async function createPlatformView(viewJs: string, viewName: ViewName) {
+  try {
+    await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
+      await createView(db, viewJs, viewName)
+    })
+  } catch (e: any) {
+    if (e.status === 409 && env.isTest()) {
+      // multiple tests can try to initialise platforms views
+      // at once - safe to exit on conflict
+      return
+    }
+    throw e
+  }
+}
+
+export const createPlatformAccountEmailView = async () => {
+  const viewJs = `function(doc) {
+    if (doc._id.startsWith("${DocumentType.ACCOUNT_METADATA}${SEPARATOR}")) {
+      emit(doc.email.toLowerCase(), doc._id)
+    }
+  }`
+  await createPlatformView(viewJs, ViewName.ACCOUNT_BY_EMAIL)
+}
+
+export const createPlatformUserView = async () => {
+  const viewJs = `function(doc) {
+    if (doc.tenantId) {
+      emit(doc._id.toLowerCase(), doc._id)
+    }
+  }`
+  await createPlatformView(viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
+}
+
 export const queryPlatformView = async <T>(
   viewName: ViewName,
   params: DatabaseQueryOpts,
   opts?: QueryViewOptions
 ): Promise<T[] | T | undefined> => {
   const CreateFuncByName: any = {
-    [ViewName.ACCOUNT_BY_EMAIL]: createAccountEmailView,
+    [ViewName.ACCOUNT_BY_EMAIL]: createPlatformAccountEmailView,
     [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView,
   }
 

packages/backend-core/src/environment.ts

@@ -44,8 +44,9 @@ const environment = {
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
   GOOGLE_CLIENT_SECRET: process.env.GOOGLE_CLIENT_SECRET,
   SALT_ROUNDS: process.env.SALT_ROUNDS,
-  REDIS_URL: process.env.REDIS_URL,
+  REDIS_URL: process.env.REDIS_URL || "localhost:6379",
-  REDIS_PASSWORD: process.env.REDIS_PASSWORD,
+  REDIS_PASSWORD: process.env.REDIS_PASSWORD || "budibase",
+  MOCK_REDIS: process.env.MOCK_REDIS,
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   AWS_REGION: process.env.AWS_REGION,
@@ -82,6 +83,7 @@ const environment = {
   SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD,
   DEPLOYMENT_ENVIRONMENT:
     process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose",
+  ENABLE_4XX_HTTP_LOGGING: process.env.ENABLE_4XX_HTTP_LOGGING || true,
   _set(key: any, value: any) {
     process.env[key] = value
     // @ts-ignore

packages/backend-core/src/events/analytics.ts

@@ -1,5 +1,5 @@
 import env from "../environment"
-import * as tenancy from "../tenancy"
+import * as context from "../context"
 import * as dbUtils from "../db/utils"
 import { Config } from "../constants"
 import { withCache, TTL, CacheKey } from "../cache"
@@ -42,7 +42,7 @@ export const enabled = async () => {
 }
 
 const getSettingsDoc = async () => {
-  const db = tenancy.getGlobalDB()
+  const db = context.getGlobalDB()
   let settings
   try {
     settings = await db.get(dbUtils.generateConfigID({ type: Config.SETTINGS }))

packages/backend-core/src/events/events.ts

@@ -1,13 +1,37 @@
-import { Event, IdentityType, AuditLogFn } from "@budibase/types"
+import { AuditLogFn, Event, IdentityType, HostInfo } from "@budibase/types"
 import { processors } from "./processors"
 import identification from "./identification"
 import { getAppId } from "../context"
 import * as backfill from "./backfill"
+import { createQueue, JobQueue } from "../queue"
+import BullQueue from "bull"
 
-let writeAuditLogs: AuditLogFn | undefined
+type AuditLogEvent = {
+  event: Event
+  properties: any
+  opts: {
+    timestamp?: string | number
+    userId?: string
+    appId?: string
+    hostInfo?: HostInfo
+  }
+}
+
+let auditLogsEnabled = false
+let auditLogQueue: BullQueue.Queue<AuditLogEvent>
 
 export const configure = (fn: AuditLogFn) => {
-  writeAuditLogs = fn
+  auditLogsEnabled = true
+  const writeAuditLogs = fn
+  auditLogQueue = createQueue<AuditLogEvent>(JobQueue.AUDIT_LOG)
+  return auditLogQueue.process(async job => {
+    await writeAuditLogs(job.data.event, job.data.properties, {
+      userId: job.data.opts.userId,
+      timestamp: job.data.opts.timestamp,
+      appId: job.data.opts.appId,
+      hostInfo: job.data.opts.hostInfo,
+    })
+  })
 }
 
 export const publishEvent = async (
@@ -21,16 +45,22 @@ export const publishEvent = async (
   const backfilling = await backfill.isBackfillingEvent(event)
   // no backfill - send the event and exit
   if (!backfilling) {
-    // only audit log actual events, don't include backfills
+    await processors.processEvent(event, identity, properties, timestamp)
-    const userId = identity.type === IdentityType.USER ? identity.id : undefined
+    if (auditLogsEnabled) {
-    if (writeAuditLogs) {
+      // only audit log actual events, don't include backfills
-      await writeAuditLogs(event, properties, {
+      const userId =
-        userId,
+        identity.type === IdentityType.USER ? identity.id : undefined
-        timestamp,
+      // add to event queue, rather than just writing immediately
-        appId: getAppId(),
+      await auditLogQueue.add({
+        event,
+        properties,
+        opts: {
+          userId,
+          timestamp,
+          appId: getAppId(),
+        },
       })
     }
-    await processors.processEvent(event, identity, properties, timestamp)
     return
   }
 

packages/backend-core/src/events/identification.ts

@@ -16,6 +16,7 @@ import {
   InstallationGroup,
   UserContext,
   Group,
+  isSSOUser,
 } from "@budibase/types"
 import { processors } from "./processors"
 import * as dbUtils from "../db/utils"
@@ -88,6 +89,7 @@ const getCurrentIdentity = async (): Promise<Identity> => {
       installationId,
       tenantId,
       environment,
+      hostInfo: userContext.host,
     }
   } else {
     throw new Error("Unknown identity type")
@@ -166,7 +168,10 @@ const identifyUser = async (
   const type = IdentityType.USER
   let builder = user.builder?.global || false
   let admin = user.admin?.global || false
-  let providerType = user.providerType
+  let providerType
+  if (isSSOUser(user)) {
+    providerType = user.providerType
+  }
   const accountHolder = account?.budibaseUserId === user._id || false
   const verified =
     account && account?.budibaseUserId === user._id ? account.verified : false

packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts

@@ -1,4 +1,4 @@
-import "../../../../../tests"
+import { testEnv } from "../../../../../tests"
 import PosthogProcessor from "../PosthogProcessor"
 import { Event, IdentityType, Hosting } from "@budibase/types"
 const tk = require("timekeeper")
@@ -16,6 +16,10 @@ const newIdentity = () => {
 }
 
 describe("PosthogProcessor", () => {
+  beforeAll(() => {
+    testEnv.singleTenant()
+  })
+
   beforeEach(async () => {
     jest.clearAllMocks()
     await cache.bustCache(

packages/backend-core/src/featureFlags/index.ts

@@ -1,5 +1,5 @@
 import env from "../environment"
-import * as tenancy from "../tenancy"
+import * as context from "../context"
 
 /**
  * Read the TENANT_FEATURE_FLAGS env var and return an array of features flags for each tenant.
@@ -28,7 +28,7 @@ export function buildFeatureFlags() {
 }
 
 export function isEnabled(featureFlag: string) {
-  const tenantId = tenancy.getTenantId()
+  const tenantId = context.getTenantId()
   const flags = getTenantFeatureFlags(tenantId)
   return flags.includes(featureFlag)
 }

packages/backend-core/src/index.ts

@@ -3,12 +3,11 @@ export * as migrations from "./migrations"
 export * as users from "./users"
 export * as roles from "./security/roles"
 export * as permissions from "./security/permissions"
-export * as accounts from "./cloud/accounts"
+export * as accounts from "./accounts"
 export * as installation from "./installation"
-export * as tenancy from "./tenancy"
 export * as featureFlags from "./featureFlags"
 export * as sessions from "./security/sessions"
-export * as deprovisioning from "./context/deprovision"
+export * as platform from "./platform"
 export * as auth from "./auth"
 export * as constants from "./constants"
 export * as logging from "./logging"
@@ -21,11 +20,20 @@ export * as context from "./context"
 export * as cache from "./cache"
 export * as objectStore from "./objectStore"
 export * as redis from "./redis"
+export * as locks from "./redis/redlock"
 export * as utils from "./utils"
 export * as errors from "./errors"
 export { default as env } from "./environment"
-
 export { SearchParams } from "./db"
+// Add context to tenancy for backwards compatibility
+// only do this for external usages to prevent internal
+// circular dependencies
+import * as context from "./context"
+import * as _tenancy from "./tenancy"
+export const tenancy = {
+  ..._tenancy,
+  ...context,
+}
 
 // expose error classes directly
 export * from "./errors"
@@ -33,10 +41,6 @@ export * from "./errors"
 // expose constants directly
 export * from "./constants"
 
-// expose inner locks from redis directly
-import * as redis from "./redis"
-export const locks = redis.redlock
-
 // expose package init function
 import * as db from "./db"
 export const init = (opts: any = {}) => {

packages/backend-core/src/middleware/authenticated.ts

@@ -4,11 +4,11 @@ import { getUser } from "../cache/user"
 import { getSession, updateSessionTTL } from "../security/sessions"
 import { buildMatcherRegex, matches } from "./matchers"
 import { SEPARATOR, queryGlobalView, ViewName } from "../db"
-import { getGlobalDB, doInTenant } from "../tenancy"
+import { getGlobalDB, doInTenant } from "../context"
 import { decrypt } from "../security/encryption"
 import * as identity from "../context/identity"
 import env from "../environment"
-import { BBContext, EndpointMatcher } from "@budibase/types"
+import { Ctx, EndpointMatcher } from "@budibase/types"
 
 const ONE_MINUTE = env.SESSION_UPDATE_PERIOD
   ? parseInt(env.SESSION_UPDATE_PERIOD)
@@ -73,7 +73,7 @@ export default function (
   }
 ) {
   const noAuthOptions = noAuthPatterns ? buildMatcherRegex(noAuthPatterns) : []
-  return async (ctx: BBContext | any, next: any) => {
+  return async (ctx: Ctx | any, next: any) => {
     let publicEndpoint = false
     const version = ctx.request.headers[Header.API_VER]
     // the path is not authenticated
@@ -148,7 +148,7 @@ export default function (
       finalise(ctx, { authenticated, user, internal, version, publicEndpoint })
 
       if (user && user.email) {
-        return identity.doInUserContext(user, next)
+        return identity.doInUserContext(user, ctx, next)
       } else {
         return next()
       }

packages/backend-core/src/middleware/errorHandling.ts

@@ -0,0 +1,28 @@
+import { APIError } from "@budibase/types"
+import * as errors from "../errors"
+import env from "../environment"
+
+export async function errorHandling(ctx: any, next: any) {
+  try {
+    await next()
+  } catch (err: any) {
+    const status = err.status || err.statusCode || 500
+    ctx.status = status
+
+    if (status > 499 || env.ENABLE_4XX_HTTP_LOGGING) {
+      ctx.log.error(err)
+    }
+
+    const error = errors.getPublicError(err)
+    const body: APIError = {
+      message: err.message,
+      status: status,
+      validationErrors: err.validation,
+      error,
+    }
+
+    ctx.body = body
+  }
+}
+
+export default errorHandling

packages/backend-core/src/middleware/index.ts

@@ -1,7 +1,7 @@
 export * as jwt from "./passport/jwt"
 export * as local from "./passport/local"
-export * as google from "./passport/google"
+export * as google from "./passport/sso/google"
-export * as oidc from "./passport/oidc"
+export * as oidc from "./passport/sso/oidc"
 import * as datasourceGoogle from "./passport/datasource/google"
 export const datasource = {
   google: datasourceGoogle,
@@ -16,4 +16,5 @@ export { default as adminOnly } from "./adminOnly"
 export { default as builderOrAdmin } from "./builderOrAdmin"
 export { default as builderOnly } from "./builderOnly"
 export { default as logging } from "./logging"
+export { default as errorHandling } from "./errorHandling"
 export * as joiValidator from "./joi-validator"

packages/backend-core/src/middleware/passport/datasource/google.ts

@@ -1,10 +1,11 @@
-import * as google from "../google"
+import * as google from "../sso/google"
 import { Cookie, Config } from "../../../constants"
 import { clearCookie, getCookie } from "../../../utils"
 import { getScopedConfig, getPlatformUrl, doWithDB } from "../../../db"
 import environment from "../../../environment"
-import { getGlobalDB } from "../../../tenancy"
+import { getGlobalDB } from "../../../context"
 import { BBContext, Database, SSOProfile } from "@budibase/types"
+import { ssoSaveUserNoOp } from "../sso/sso"
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
 
 type Passport = {
@@ -36,7 +37,11 @@ export async function preAuth(
   const platformUrl = await getPlatformUrl({ tenantAware: false })
 
   let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback`
-  const strategy = await google.strategyFactory(googleConfig, callbackUrl)
+  const strategy = await google.strategyFactory(
+    googleConfig,
+    callbackUrl,
+    ssoSaveUserNoOp
+  )
 
   if (!ctx.query.appId || !ctx.query.datasourceId) {
     ctx.throw(400, "appId and datasourceId query params not present.")

packages/backend-core/src/middleware/passport/local.ts

@@ -1,15 +1,10 @@
 import { UserStatus } from "../../constants"
-import { compare, newid } from "../../utils"
+import { compare } from "../../utils"
-import env from "../../environment"
 import * as users from "../../users"
 import { authError } from "./utils"
-import { createASession } from "../../security/sessions"
-import { getTenantId } from "../../tenancy"
 import { BBContext } from "@budibase/types"
-const jwt = require("jsonwebtoken")
 
 const INVALID_ERR = "Invalid credentials"
-const SSO_NO_PASSWORD = "SSO user does not have a password set"
 const EXPIRED = "This account has expired. Please reset your password"
 
 export const options = {
@@ -35,50 +30,25 @@ export async function authenticate(
 
   const dbUser = await users.getGlobalUserByEmail(email)
   if (dbUser == null) {
-    return authError(done, `User not found: [${email}]`)
+    console.info(`user=${email} could not be found`)
-  }
-
-  // check that the user is currently inactive, if this is the case throw invalid
-  if (dbUser.status === UserStatus.INACTIVE) {
     return authError(done, INVALID_ERR)
   }
 
-  // check that the user has a stored password before proceeding
+  if (dbUser.status === UserStatus.INACTIVE) {
-  if (!dbUser.password) {
+    console.info(`user=${email} is inactive`, dbUser)
-    if (
+    return authError(done, INVALID_ERR)
-      (dbUser.account && dbUser.account.authType === "sso") || // root account sso
+  }
-      dbUser.thirdPartyProfile // internal sso
-    ) {
-      return authError(done, SSO_NO_PASSWORD)
-    }
 
-    console.error("Non SSO usser has no password set", dbUser)
+  if (!dbUser.password) {
+    console.info(`user=${email} has no password set`, dbUser)
     return authError(done, EXPIRED)
   }
 
-  // authenticate
+  if (!(await compare(password, dbUser.password))) {
-  if (await compare(password, dbUser.password)) {
-    const sessionId = newid()
-    const tenantId = getTenantId()
-
-    await createASession(dbUser._id!, { sessionId, tenantId })
-
-    const token = jwt.sign(
-      {
-        userId: dbUser._id,
-        sessionId,
-        tenantId,
-      },
-      env.JWT_SECRET
-    )
-    // Remove users password in payload
-    delete dbUser.password
-
-    return done(null, {
-      ...dbUser,
-      token,
-    })
-  } else {
     return authError(done, INVALID_ERR)
   }
+
+  // intentionally remove the users password in payload
+  delete dbUser.password
+  return done(null, dbUser)
 }

packages/backend-core/src/middleware/passport/sso/google.ts

@@ -1,18 +1,26 @@
-import { ssoCallbackUrl } from "./utils"
+import { ssoCallbackUrl } from "../utils"
-import { authenticateThirdParty, SaveUserFunction } from "./third-party-common"
+import * as sso from "./sso"
-import { ConfigType, GoogleConfig, Database, SSOProfile } from "@budibase/types"
+import {
+  ConfigType,
+  GoogleConfig,
+  Database,
+  SSOProfile,
+  SSOAuthDetails,
+  SSOProviderType,
+  SaveSSOUserFunction,
+} from "@budibase/types"
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
 
-export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
+export function buildVerifyFn(saveUserFn: SaveSSOUserFunction) {
   return (
     accessToken: string,
     refreshToken: string,
     profile: SSOProfile,
     done: Function
   ) => {
-    const thirdPartyUser = {
+    const details: SSOAuthDetails = {
-      provider: profile.provider, // should always be 'google'
+      provider: "google",
-      providerType: "google",
+      providerType: SSOProviderType.GOOGLE,
       userId: profile.id,
       profile: profile,
       email: profile._json.email,
@@ -22,8 +30,8 @@ export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
       },
     }
 
-    return authenticateThirdParty(
+    return sso.authenticate(
-      thirdPartyUser,
+      details,
       true, // require local accounts to exist
       done,
       saveUserFn
@@ -39,7 +47,7 @@ export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
 export async function strategyFactory(
   config: GoogleConfig["config"],
   callbackUrl: string,
-  saveUserFn?: SaveUserFunction
+  saveUserFn: SaveSSOUserFunction
 ) {
   try {
     const { clientID, clientSecret } = config

packages/backend-core/src/middleware/passport/sso/oidc.ts

@@ -1,22 +1,20 @@
 import fetch from "node-fetch"
-import { authenticateThirdParty, SaveUserFunction } from "./third-party-common"
+import * as sso from "./sso"
-import { ssoCallbackUrl } from "./utils"
+import { ssoCallbackUrl } from "../utils"
 import {
   ConfigType,
-  OIDCInnerCfg,
+  OIDCInnerConfig,
   Database,
   SSOProfile,
-  ThirdPartyUser,
+  OIDCStrategyConfiguration,
-  OIDCConfiguration,
+  SSOAuthDetails,
+  SSOProviderType,
+  JwtClaims,
+  SaveSSOUserFunction,
 } from "@budibase/types"
 const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy
 
-type JwtClaims = {
+export function buildVerifyFn(saveUserFn: SaveSSOUserFunction) {
-  preferred_username: string
-  email: string
-}
-
-export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
   /**
    * @param {*} issuer The identity provider base URL
    * @param {*} sub The user ID
@@ -39,10 +37,10 @@ export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
     params: any,
     done: Function
   ) => {
-    const thirdPartyUser: ThirdPartyUser = {
+    const details: SSOAuthDetails = {
       // store the issuer info to enable sync in future
       provider: issuer,
-      providerType: "oidc",
+      providerType: SSOProviderType.OIDC,
       userId: profile.id,
       profile: profile,
       email: getEmail(profile, jwtClaims),
@@ -52,8 +50,8 @@ export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
       },
     }
 
-    return authenticateThirdParty(
+    return sso.authenticate(
-      thirdPartyUser,
+      details,
       false, // don't require local accounts to exist
       done,
       saveUserFn
@@ -104,8 +102,8 @@ function validEmail(value: string) {
  * @returns Dynamically configured Passport OIDC Strategy
  */
 export async function strategyFactory(
-  config: OIDCConfiguration,
+  config: OIDCStrategyConfiguration,
-  saveUserFn?: SaveUserFunction
+  saveUserFn: SaveSSOUserFunction
 ) {
   try {
     const verify = buildVerifyFn(saveUserFn)
@@ -119,14 +117,14 @@ export async function strategyFactory(
 }
 
 export async function fetchStrategyConfig(
-  enrichedConfig: OIDCInnerCfg,
+  oidcConfig: OIDCInnerConfig,
   callbackUrl?: string
-): Promise<OIDCConfiguration> {
+): Promise<OIDCStrategyConfiguration> {
   try {
-    const { clientID, clientSecret, configUrl } = enrichedConfig
+    const { clientID, clientSecret, configUrl } = oidcConfig
 
     if (!clientID || !clientSecret || !callbackUrl || !configUrl) {
-      //check for remote config and all required elements
+      // check for remote config and all required elements
       throw new Error(
         "Configuration invalid. Must contain clientID, clientSecret, callbackUrl and configUrl"
       )

packages/backend-core/src/middleware/passport/sso/sso.ts

@@ -0,0 +1,165 @@
+import { generateGlobalUserID } from "../../../db"
+import { authError } from "../utils"
+import * as users from "../../../users"
+import * as context from "../../../context"
+import fetch from "node-fetch"
+import {
+  SaveSSOUserFunction,
+  SaveUserOpts,
+  SSOAuthDetails,
+  SSOUser,
+  User,
+} from "@budibase/types"
+
+// no-op function for user save
+// - this allows datasource auth and access token refresh to work correctly
+// - prefer no-op over an optional argument to ensure function is provided to login flows
+export const ssoSaveUserNoOp: SaveSSOUserFunction = (
+  user: SSOUser,
+  opts: SaveUserOpts
+) => Promise.resolve(user)
+
+/**
+ * Common authentication logic for third parties. e.g. OAuth, OIDC.
+ */
+export async function authenticate(
+  details: SSOAuthDetails,
+  requireLocalAccount: boolean = true,
+  done: any,
+  saveUserFn: SaveSSOUserFunction
+) {
+  if (!saveUserFn) {
+    throw new Error("Save user function must be provided")
+  }
+  if (!details.userId) {
+    return authError(done, "sso user id required")
+  }
+  if (!details.email) {
+    return authError(done, "sso user email required")
+  }
+
+  // use the third party id
+  const userId = generateGlobalUserID(details.userId)
+
+  let dbUser: User | undefined
+
+  // try to load by id
+  try {
+    dbUser = await users.getById(userId)
+  } catch (err: any) {
+    // abort when not 404 error
+    if (!err.status || err.status !== 404) {
+      return authError(
+        done,
+        "Unexpected error when retrieving existing user",
+        err
+      )
+    }
+  }
+
+  // fallback to loading by email
+  if (!dbUser) {
+    dbUser = await users.getGlobalUserByEmail(details.email)
+  }
+
+  // exit early if there is still no user and auto creation is disabled
+  if (!dbUser && requireLocalAccount) {
+    return authError(
+      done,
+      "Email does not yet exist. You must set up your local budibase account first."
+    )
+  }
+
+  // first time creation
+  if (!dbUser) {
+    // setup a blank user using the third party id
+    dbUser = {
+      _id: userId,
+      email: details.email,
+      roles: {},
+      tenantId: context.getTenantId(),
+    }
+  }
+
+  let ssoUser = await syncUser(dbUser, details)
+  // never prompt for password reset
+  ssoUser.forceResetPassword = false
+
+  try {
+    // don't try to re-save any existing password
+    delete ssoUser.password
+    // create or sync the user
+    ssoUser = (await saveUserFn(ssoUser, {
+      hashPassword: false,
+      requirePassword: false,
+    })) as SSOUser
+  } catch (err: any) {
+    return authError(done, "Error saving user", err)
+  }
+
+  return done(null, ssoUser)
+}
+
+async function getProfilePictureUrl(user: User, details: SSOAuthDetails) {
+  const pictureUrl = details.profile?._json.picture
+  if (pictureUrl) {
+    const response = await fetch(pictureUrl)
+    if (response.status === 200) {
+      const type = response.headers.get("content-type") as string
+      if (type.startsWith("image/")) {
+        return pictureUrl
+      }
+    }
+  }
+}
+
+/**
+ * @returns a user that has been sync'd with third party information
+ */
+async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
+  let firstName
+  let lastName
+  let pictureUrl
+  let oauth2
+  let thirdPartyProfile
+
+  if (details.profile) {
+    const profile = details.profile
+
+    if (profile.name) {
+      const name = profile.name
+      // first name
+      if (name.givenName) {
+        firstName = name.givenName
+      }
+      // last name
+      if (name.familyName) {
+        lastName = name.familyName
+      }
+    }
+
+    pictureUrl = await getProfilePictureUrl(user, details)
+
+    thirdPartyProfile = {
+      ...profile._json,
+    }
+  }
+
+  // oauth tokens for future use
+  if (details.oauth2) {
+    oauth2 = {
+      ...details.oauth2,
+    }
+  }
+
+  return {
+    ...user,
+    provider: details.provider,
+    providerType: details.providerType,
+    firstName,
+    lastName,
+    thirdPartyProfile,
+    pictureUrl,
+    oauth2,
+  }
+}

packages/backend-core/src/middleware/passport/sso/tests/google.spec.ts

@@ -0,0 +1,67 @@
+import { generator, structures } from "../../../../../tests"
+import { SSOProviderType } from "@budibase/types"
+
+jest.mock("passport-google-oauth")
+const mockStrategy = require("passport-google-oauth").OAuth2Strategy
+
+jest.mock("../sso")
+import * as _sso from "../sso"
+const sso = jest.mocked(_sso)
+
+const mockSaveUserFn = jest.fn()
+const mockDone = jest.fn()
+
+import * as google from "../google"
+
+describe("google", () => {
+  describe("strategyFactory", () => {
+    const googleConfig = structures.sso.googleConfig()
+    const callbackUrl = generator.url()
+
+    it("should create successfully create a google strategy", async () => {
+      await google.strategyFactory(googleConfig, callbackUrl, mockSaveUserFn)
+
+      const expectedOptions = {
+        clientID: googleConfig.clientID,
+        clientSecret: googleConfig.clientSecret,
+        callbackURL: callbackUrl,
+      }
+
+      expect(mockStrategy).toHaveBeenCalledWith(
+        expectedOptions,
+        expect.anything()
+      )
+    })
+  })
+
+  describe("authenticate", () => {
+    const details = structures.sso.authDetails()
+    details.provider = "google"
+    details.providerType = SSOProviderType.GOOGLE
+
+    const profile = details.profile!
+    profile.provider = "google"
+
+    beforeEach(() => {
+      jest.clearAllMocks()
+    })
+
+    it("delegates authentication to third party common", async () => {
+      const authenticate = await google.buildVerifyFn(mockSaveUserFn)
+
+      await authenticate(
+        details.oauth2.accessToken,
+        details.oauth2.refreshToken!,
+        profile,
+        mockDone
+      )
+
+      expect(sso.authenticate).toHaveBeenCalledWith(
+        details,
+        true,
+        mockDone,
+        mockSaveUserFn
+      )
+    })
+  })
+})

packages/backend-core/src/middleware/passport/sso/tests/oidc.spec.ts

@@ -0,0 +1,152 @@
+import { generator, mocks, structures } from "../../../../../tests"
+import {
+  JwtClaims,
+  OIDCInnerConfig,
+  SSOAuthDetails,
+  SSOProviderType,
+} from "@budibase/types"
+import * as _sso from "../sso"
+import * as oidc from "../oidc"
+
+jest.mock("@techpass/passport-openidconnect")
+const mockStrategy = require("@techpass/passport-openidconnect").Strategy
+
+jest.mock("../sso")
+const sso = jest.mocked(_sso)
+
+const mockSaveUser = jest.fn()
+const mockDone = jest.fn()
+
+describe("oidc", () => {
+  const callbackUrl = generator.url()
+  const oidcConfig: OIDCInnerConfig = structures.sso.oidcConfig()
+  const wellKnownConfig = structures.sso.oidcWellKnownConfig()
+
+  function mockRetrieveWellKnownConfig() {
+    // mock the request to retrieve the oidc configuration
+    mocks.fetch.mockReturnValue({
+      ok: true,
+      json: () => wellKnownConfig,
+    })
+  }
+
+  beforeEach(() => {
+    mockRetrieveWellKnownConfig()
+  })
+
+  describe("strategyFactory", () => {
+    it("should create successfully create an oidc strategy", async () => {
+      const strategyConfiguration = await oidc.fetchStrategyConfig(
+        oidcConfig,
+        callbackUrl
+      )
+      await oidc.strategyFactory(strategyConfiguration, mockSaveUser)
+
+      expect(mocks.fetch).toHaveBeenCalledWith(oidcConfig.configUrl)
+
+      const expectedOptions = {
+        issuer: wellKnownConfig.issuer,
+        authorizationURL: wellKnownConfig.authorization_endpoint,
+        tokenURL: wellKnownConfig.token_endpoint,
+        userInfoURL: wellKnownConfig.userinfo_endpoint,
+        clientID: oidcConfig.clientID,
+        clientSecret: oidcConfig.clientSecret,
+        callbackURL: callbackUrl,
+      }
+      expect(mockStrategy).toHaveBeenCalledWith(
+        expectedOptions,
+        expect.anything()
+      )
+    })
+  })
+
+  describe("authenticate", () => {
+    const details: SSOAuthDetails = structures.sso.authDetails()
+    details.providerType = SSOProviderType.OIDC
+    const profile = details.profile!
+    const issuer = profile.provider
+
+    const sub = generator.string()
+    const idToken = generator.string()
+    const params = {}
+
+    let authenticateFn: any
+    let jwtClaims: JwtClaims
+
+    beforeEach(async () => {
+      jest.clearAllMocks()
+      authenticateFn = await oidc.buildVerifyFn(mockSaveUser)
+    })
+
+    async function authenticate() {
+      await authenticateFn(
+        issuer,
+        sub,
+        profile,
+        jwtClaims,
+        details.oauth2.accessToken,
+        details.oauth2.refreshToken,
+        idToken,
+        params,
+        mockDone
+      )
+    }
+
+    it("passes auth details to sso module", async () => {
+      await authenticate()
+
+      expect(sso.authenticate).toHaveBeenCalledWith(
+        details,
+        false,
+        mockDone,
+        mockSaveUser
+      )
+    })
+
+    it("uses JWT email to get email", async () => {
+      delete profile._json.email
+
+      jwtClaims = {
+        email: details.email,
+      }
+
+      await authenticate()
+
+      expect(sso.authenticate).toHaveBeenCalledWith(
+        details,
+        false,
+        mockDone,
+        mockSaveUser
+      )
+    })
+
+    it("uses JWT username to get email", async () => {
+      delete profile._json.email
+
+      jwtClaims = {
+        email: details.email,
+      }
+
+      await authenticate()
+
+      expect(sso.authenticate).toHaveBeenCalledWith(
+        details,
+        false,
+        mockDone,
+        mockSaveUser
+      )
+    })
+
+    it("uses JWT invalid username to get email", async () => {
+      delete profile._json.email
+
+      jwtClaims = {
+        preferred_username: "invalidUsername",
+      }
+
+      await expect(authenticate()).rejects.toThrow(
+        "Could not determine user email from profile"
+      )
+    })
+  })
+})

packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts

@@ -0,0 +1,196 @@
+import { structures, testEnv, mocks } from "../../../../../tests"
+import { SSOAuthDetails, User } from "@budibase/types"
+
+import { HTTPError } from "../../../../errors"
+import * as sso from "../sso"
+import * as context from "../../../../context"
+
+const mockDone = jest.fn()
+const mockSaveUser = jest.fn()
+
+jest.mock("../../../../users")
+import * as _users from "../../../../users"
+const users = jest.mocked(_users)
+
+const getErrorMessage = () => {
+  return mockDone.mock.calls[0][2].message
+}
+
+describe("sso", () => {
+  describe("authenticate", () => {
+    beforeEach(() => {
+      jest.clearAllMocks()
+      testEnv.singleTenant()
+    })
+
+    describe("validation", () => {
+      const testValidation = async (
+        details: SSOAuthDetails,
+        message: string
+      ) => {
+        await sso.authenticate(details, false, mockDone, mockSaveUser)
+
+        expect(mockDone.mock.calls.length).toBe(1)
+        expect(getErrorMessage()).toContain(message)
+      }
+
+      it("user id fails", async () => {
+        const details = structures.sso.authDetails()
+        details.userId = undefined!
+
+        await testValidation(details, "sso user id required")
+      })
+
+      it("email fails", async () => {
+        const details = structures.sso.authDetails()
+        details.email = undefined!
+
+        await testValidation(details, "sso user email required")
+      })
+    })
+
+    function mockGetProfilePicture() {
+      mocks.fetch.mockReturnValueOnce(
+        Promise.resolve({
+          status: 200,
+          headers: { get: () => "image/" },
+        })
+      )
+    }
+
+    describe("when the user doesn't exist", () => {
+      let user: User
+      let details: SSOAuthDetails
+
+      beforeEach(() => {
+        users.getById.mockImplementationOnce(() => {
+          throw new HTTPError("", 404)
+        })
+        mockGetProfilePicture()
+
+        user = structures.users.user()
+        delete user._rev
+        delete user._id
+
+        details = structures.sso.authDetails(user)
+        details.userId = structures.uuid()
+      })
+
+      describe("when a local account is required", () => {
+        it("returns an error message", async () => {
+          const details = structures.sso.authDetails()
+
+          await sso.authenticate(details, true, mockDone, mockSaveUser)
+
+          expect(mockDone.mock.calls.length).toBe(1)
+          expect(getErrorMessage()).toContain(
+            "Email does not yet exist. You must set up your local budibase account first."
+          )
+        })
+      })
+
+      describe("when a local account isn't required", () => {
+        it("creates and authenticates the user", async () => {
+          const ssoUser = structures.users.ssoUser({ user, details })
+          mockSaveUser.mockReturnValueOnce(ssoUser)
+
+          await sso.authenticate(details, false, mockDone, mockSaveUser)
+
+          // default roles for new user
+          ssoUser.roles = {}
+
+          // modified external id to match user format
+          ssoUser._id = "us_" + details.userId
+
+          // new sso user won't have a password
+          delete ssoUser.password
+
+          // new user isn't saved with rev
+          delete ssoUser._rev
+
+          // tenant id added
+          ssoUser.tenantId = context.getTenantId()
+
+          expect(mockSaveUser).toBeCalledWith(ssoUser, {
+            hashPassword: false,
+            requirePassword: false,
+          })
+          expect(mockDone).toBeCalledWith(null, ssoUser)
+        })
+      })
+    })
+
+    describe("when the user exists", () => {
+      let existingUser: User
+      let details: SSOAuthDetails
+
+      beforeEach(() => {
+        existingUser = structures.users.user()
+        existingUser._id = structures.uuid()
+        details = structures.sso.authDetails(existingUser)
+        mockGetProfilePicture()
+      })
+
+      describe("exists by email", () => {
+        beforeEach(() => {
+          users.getById.mockImplementationOnce(() => {
+            throw new HTTPError("", 404)
+          })
+          users.getGlobalUserByEmail.mockReturnValueOnce(
+            Promise.resolve(existingUser)
+          )
+        })
+
+        it("syncs and authenticates the user", async () => {
+          const ssoUser = structures.users.ssoUser({
+            user: existingUser,
+            details,
+          })
+          mockSaveUser.mockReturnValueOnce(ssoUser)
+
+          await sso.authenticate(details, true, mockDone, mockSaveUser)
+
+          // roles preserved
+          ssoUser.roles = existingUser.roles
+
+          // existing id preserved
+          ssoUser._id = existingUser._id
+
+          expect(mockSaveUser).toBeCalledWith(ssoUser, {
+            hashPassword: false,
+            requirePassword: false,
+          })
+          expect(mockDone).toBeCalledWith(null, ssoUser)
+        })
+      })
+
+      describe("exists by id", () => {
+        beforeEach(() => {
+          users.getById.mockReturnValueOnce(Promise.resolve(existingUser))
+        })
+
+        it("syncs and authenticates the user", async () => {
+          const ssoUser = structures.users.ssoUser({
+            user: existingUser,
+            details,
+          })
+          mockSaveUser.mockReturnValueOnce(ssoUser)
+
+          await sso.authenticate(details, true, mockDone, mockSaveUser)
+
+          // roles preserved
+          ssoUser.roles = existingUser.roles
+
+          // existing id preserved
+          ssoUser._id = existingUser._id
+
+          expect(mockSaveUser).toBeCalledWith(ssoUser, {
+            hashPassword: false,
+            requirePassword: false,
+          })
+          expect(mockDone).toBeCalledWith(null, ssoUser)
+        })
+      })
+    })
+  })
+})

packages/backend-core/src/middleware/passport/tests/google.spec.js

@@ -1,79 +0,0 @@
-// Mock data
-
-const { data } = require("./utilities/mock-data")
-
-const TENANT_ID = "default"
-
-const googleConfig = {
-  clientID: data.clientID,
-  clientSecret: data.clientSecret,
-}
-
-const profile = {
-  id: "mockId",
-  _json: {
-    email : data.email
-  },
-  provider: "google"
-}
-
-const user = data.buildThirdPartyUser("google", "google", profile)
-
-describe("google", () => {
-  describe("strategyFactory", () => {  
-    // mock passport strategy factory
-    jest.mock("passport-google-oauth")
-    const mockStrategy = require("passport-google-oauth").OAuth2Strategy
-  
-    it("should create successfully create a google strategy", async () => {
-      const google = require("../google")
-
-      const callbackUrl = `/api/global/auth/${TENANT_ID}/google/callback`
-      await google.strategyFactory(googleConfig, callbackUrl)
-  
-      const expectedOptions = {
-        clientID: googleConfig.clientID,
-        clientSecret: googleConfig.clientSecret,
-        callbackURL: callbackUrl,
-      }
-
-      expect(mockStrategy).toHaveBeenCalledWith(
-        expectedOptions,
-        expect.anything()
-      )
-    })
-  })
-  
-  describe("authenticate", () => {    
-    afterEach(() => {
-      jest.clearAllMocks();
-    });
-
-    // mock third party common authentication
-    jest.mock("../third-party-common")
-    const authenticateThirdParty = require("../third-party-common").authenticateThirdParty
-
-    // mock the passport callback
-    const mockDone = jest.fn()
-
-    it("delegates authentication to third party common", async () => {
-      const google = require("../google")
-      const mockSaveUserFn = jest.fn()
-      const authenticate = await google.buildVerifyFn(mockSaveUserFn)
-
-      await authenticate(
-        data.accessToken,
-        data.refreshToken,
-        profile,
-        mockDone
-      )
-
-      expect(authenticateThirdParty).toHaveBeenCalledWith(
-        user,
-        true, 
-        mockDone,
-        mockSaveUserFn)
-    })
-  })
-})
-

packages/backend-core/src/middleware/passport/tests/oidc.spec.js

@@ -1,144 +0,0 @@
-// Mock data
-const mockFetch = require("node-fetch")
-const { data } = require("./utilities/mock-data")
-const issuer = "mockIssuer"
-const sub = "mockSub"
-const profile = {
-  id: "mockId",
-  _json: {
-    email : data.email
-  }
-}
-let jwtClaims = {}  
-const idToken = "mockIdToken"
-const params = {}
-
-const callbackUrl = "http://somecallbackurl"
-
-// response from .well-known/openid-configuration
-const oidcConfigUrlResponse = {
-  issuer: issuer,
-  authorization_endpoint: "mockAuthorizationEndpoint",
-  token_endpoint: "mockTokenEndpoint",
-  userinfo_endpoint: "mockUserInfoEndpoint"
-}
-
-const oidcConfig = {
-  configUrl: "http://someconfigurl",
-  clientID: data.clientID,
-  clientSecret: data.clientSecret,
-}
-
-const user = data.buildThirdPartyUser(issuer, "oidc", profile)
-
-describe("oidc", () => {
-  describe("strategyFactory", () => {  
-    // mock passport strategy factory
-    jest.mock("@techpass/passport-openidconnect")
-    const mockStrategy = require("@techpass/passport-openidconnect").Strategy
-    
-    // mock the request to retrieve the oidc configuration
-    mockFetch.mockReturnValue({
-      ok: true,
-      json: () => oidcConfigUrlResponse
-    })
-  
-    it("should create successfully create an oidc strategy", async () => {
-      const oidc = require("../oidc")
-      const enrichedConfig = await oidc.fetchStrategyConfig(oidcConfig, callbackUrl)
-      await oidc.strategyFactory(enrichedConfig, callbackUrl)
-  
-      expect(mockFetch).toHaveBeenCalledWith(oidcConfig.configUrl)
-
-      const expectedOptions = {
-        issuer: oidcConfigUrlResponse.issuer,
-        authorizationURL: oidcConfigUrlResponse.authorization_endpoint,
-        tokenURL: oidcConfigUrlResponse.token_endpoint,
-        userInfoURL: oidcConfigUrlResponse.userinfo_endpoint,
-        clientID: oidcConfig.clientID,
-        clientSecret: oidcConfig.clientSecret,
-        callbackURL: callbackUrl,
-      }
-      expect(mockStrategy).toHaveBeenCalledWith(
-        expectedOptions,
-        expect.anything()
-      )
-    })
-  })
-  
-  describe("authenticate", () => {    
-    afterEach(() => {
-      jest.clearAllMocks()
-    });
-
-    // mock third party common authentication
-    jest.mock("../third-party-common")
-    const authenticateThirdParty = require("../third-party-common").authenticateThirdParty
-    
-    // mock the passport callback
-    const mockDone = jest.fn()
-    const mockSaveUserFn = jest.fn()
-
-    async function doAuthenticate() {
-      const oidc = require("../oidc")
-      const authenticate = await oidc.buildVerifyFn(mockSaveUserFn)
-
-      await authenticate(
-        issuer,
-        sub,
-        profile,
-        jwtClaims,
-        data.accessToken,
-        data.refreshToken,
-        idToken,
-        params,
-        mockDone
-      )
-    }
-
-    async function doTest() {
-      await doAuthenticate()
-
-      expect(authenticateThirdParty).toHaveBeenCalledWith(
-        user,
-        false, 
-        mockDone,
-        mockSaveUserFn,
-      )
-    }
-
-    it("delegates authentication to third party common", async () => {
-      await doTest()
-    })
-
-    it("uses JWT email to get email", async () => {
-      delete profile._json.email
-      jwtClaims = {
-        email : "mock@budibase.com"
-      }
-
-      await doTest()
-    })
-  
-    it("uses JWT username to get email", async () => {
-      delete profile._json.email
-      jwtClaims = {
-        preferred_username : "mock@budibase.com"
-      }
-
-      await doTest()
-    })
-
-    it("uses JWT invalid username to get email", async () => {
-      delete profile._json.email
-
-      jwtClaims = {
-        preferred_username : "invalidUsername"
-      }
-
-      await expect(doAuthenticate()).rejects.toThrow("Could not determine user email from profile");
-    })
-  
-  })
-})
-

packages/backend-core/src/middleware/passport/tests/third-party-common.seq.spec.js

@@ -1,178 +0,0 @@
-require("../../../../tests")
-const { authenticateThirdParty } = require("../third-party-common")
-const { data } = require("./utilities/mock-data")
-const { DEFAULT_TENANT_ID } = require("../../../constants")
-
-const { generateGlobalUserID } = require("../../../db/utils")
-const { newid } = require("../../../utils")
-const { doWithGlobalDB, doInTenant } = require("../../../tenancy")
-
-const done = jest.fn()
-
-const getErrorMessage = () => {
-  return done.mock.calls[0][2].message
-}
-
-const saveUser = async (user) => {
-  return doWithGlobalDB(DEFAULT_TENANT_ID, async db => {
-    return await db.put(user)
-  })
-}
-
-function authenticate(user, requireLocal, saveFn) {
-  return doInTenant(DEFAULT_TENANT_ID, () => {
-    return authenticateThirdParty(user, requireLocal, done, saveFn)
-  })
-}
-
-describe("third party common", () => {
-  describe("authenticateThirdParty", () => {
-    let thirdPartyUser
-
-    beforeEach(() => {
-      thirdPartyUser = data.buildThirdPartyUser()
-    })
-
-    afterEach(async () => {
-      return doWithGlobalDB(DEFAULT_TENANT_ID, async db => {
-        jest.clearAllMocks()
-        await db.destroy()
-      })
-    })
-
-    describe("validation", () => {
-      const testValidation = async (message) => {
-        await authenticate(thirdPartyUser, false, saveUser)
-        expect(done.mock.calls.length).toBe(1)
-        expect(getErrorMessage()).toContain(message)
-      }
-
-      it("provider fails", async () => {
-        delete thirdPartyUser.provider
-        await testValidation("third party user provider required")
-      })
-
-      it("user id fails", async () => {
-        delete thirdPartyUser.userId
-        await testValidation("third party user id required")
-      })
-
-      it("email fails", async () => {
-        delete thirdPartyUser.email
-        await testValidation("third party user email required")
-      })
-    })
-
-    const expectUserIsAuthenticated = () => {
-      const user = done.mock.calls[0][1]
-      expect(user).toBeDefined()
-      expect(user._id).toBeDefined()
-      expect(user._rev).toBeDefined()
-      expect(user.token).toBeDefined()
-      return user
-    }
-
-    const expectUserIsSynced = (user, thirdPartyUser) => {
-      expect(user.provider).toBe(thirdPartyUser.provider)
-      expect(user.firstName).toBe(thirdPartyUser.profile.name.givenName)
-      expect(user.lastName).toBe(thirdPartyUser.profile.name.familyName)
-      expect(user.thirdPartyProfile).toStrictEqual(thirdPartyUser.profile._json)
-      expect(user.oauth2).toStrictEqual(thirdPartyUser.oauth2)
-    }
-
-    describe("when the user doesn't exist", () => {
-      describe("when a local account is required", () => {
-        it("returns an error message", async () => {
-          await authenticate(thirdPartyUser, true, saveUser)
-          expect(done.mock.calls.length).toBe(1)
-          expect(getErrorMessage()).toContain("Email does not yet exist. You must set up your local budibase account first.")
-        })
-      })
-
-      describe("when a local account isn't required", () => {
-        it("creates and authenticates the user", async () => {
-          await authenticate(thirdPartyUser, false, saveUser)
-          const user = expectUserIsAuthenticated()
-          expectUserIsSynced(user, thirdPartyUser)
-          expect(user.roles).toStrictEqual({})
-        })
-      })
-    })
-
-    describe("when the user exists", () => {
-      let dbUser
-      let id
-      let email
-
-      const createUser = async () => {
-        return doWithGlobalDB(DEFAULT_TENANT_ID, async db => {
-          dbUser = {
-            _id: id,
-            email: email,
-          }
-          const response = await db.put(dbUser)
-          dbUser._rev = response.rev
-          return dbUser
-        })
-      }
-
-      const expectUserIsUpdated = (user) => {
-        // id is unchanged
-        expect(user._id).toBe(id)
-        // user is updated
-        expect(user._rev).not.toBe(dbUser._rev)
-      }
-
-      describe("exists by email", () => {
-        beforeEach(async () => {
-          id = generateGlobalUserID(newid()) // random id
-          email = thirdPartyUser.email //  matching email
-          await createUser()
-        })
-
-        it("syncs and authenticates the user", async () => {
-          await authenticate(thirdPartyUser, true, saveUser)
-
-          const user = expectUserIsAuthenticated()
-          expectUserIsSynced(user, thirdPartyUser)
-          expectUserIsUpdated(user)
-        })
-      })
-
-      describe("exists by email with different casing", () => {
-        beforeEach(async () => {
-          id = generateGlobalUserID(newid()) // random id
-          email = thirdPartyUser.email.toUpperCase() //  matching email except for casing
-          await createUser()
-        })
-
-        it("syncs and authenticates the user", async () => {
-          await authenticate(thirdPartyUser, true, saveUser)
-
-          const user = expectUserIsAuthenticated()
-          expectUserIsSynced(user, thirdPartyUser)
-          expectUserIsUpdated(user)
-          expect(user.email).toBe(thirdPartyUser.email.toUpperCase())
-        })
-      })
-
-
-      describe("exists by id", () => {
-        beforeEach(async () => {
-          id = generateGlobalUserID(thirdPartyUser.userId) // matching id
-          email = "test@test.com" // random email
-          await createUser()
-        })
-
-        it("syncs and authenticates the user", async () => {
-          await authenticate(thirdPartyUser, true, saveUser)
-
-          const user = expectUserIsAuthenticated()
-          expectUserIsSynced(user, thirdPartyUser)
-          expectUserIsUpdated(user)
-        })
-      })
-    })
-  })
-})
-

packages/backend-core/src/middleware/passport/tests/utilities/mock-data.js

@@ -1,54 +0,0 @@
-// Mock Data
-
-const mockClientID = "mockClientID"
-const mockClientSecret = "mockClientSecret"
-
-const mockEmail = "mock@budibase.com"
-const mockAccessToken = "mockAccessToken"
-const mockRefreshToken = "mockRefreshToken"
-
-const mockProvider = "mockProvider"
-const mockProviderType = "mockProviderType"
-
-const mockProfile = {
-  id: "mockId",
-  name: {
-    givenName: "mockGivenName",
-    familyName: "mockFamilyName",
-  },
-  _json: {
-    email: mockEmail,
-  },
-}
-
-const buildOauth2 = (
-  accessToken = mockAccessToken,
-  refreshToken = mockRefreshToken
-) => ({
-  accessToken: accessToken,
-  refreshToken: refreshToken,
-})
-
-const buildThirdPartyUser = (
-  provider = mockProvider,
-  providerType = mockProviderType,
-  profile = mockProfile,
-  email = mockEmail,
-  oauth2 = buildOauth2()
-) => ({
-  provider: provider,
-  providerType: providerType,
-  userId: profile.id,
-  profile: profile,
-  email: email,
-  oauth2: oauth2,
-})
-
-exports.data = {
-  clientID: mockClientID,
-  clientSecret: mockClientSecret,
-  email: mockEmail,
-  accessToken: mockAccessToken,
-  refreshToken: mockRefreshToken,
-  buildThirdPartyUser,
-}

packages/backend-core/src/middleware/passport/third-party-common.ts

@@ -1,177 +0,0 @@
-import env from "../../environment"
-import { generateGlobalUserID } from "../../db"
-import { authError } from "./utils"
-import { newid } from "../../utils"
-import { createASession } from "../../security/sessions"
-import * as users from "../../users"
-import { getGlobalDB, getTenantId } from "../../tenancy"
-import fetch from "node-fetch"
-import { ThirdPartyUser } from "@budibase/types"
-const jwt = require("jsonwebtoken")
-
-type SaveUserOpts = {
-  requirePassword?: boolean
-  hashPassword?: boolean
-  currentUserId?: string
-}
-
-export type SaveUserFunction = (
-  user: ThirdPartyUser,
-  opts: SaveUserOpts
-) => Promise<any>
-
-/**
- * Common authentication logic for third parties. e.g. OAuth, OIDC.
- */
-export async function authenticateThirdParty(
-  thirdPartyUser: ThirdPartyUser,
-  requireLocalAccount: boolean = true,
-  done: Function,
-  saveUserFn?: SaveUserFunction
-) {
-  if (!saveUserFn) {
-    throw new Error("Save user function must be provided")
-  }
-  if (!thirdPartyUser.provider) {
-    return authError(done, "third party user provider required")
-  }
-  if (!thirdPartyUser.userId) {
-    return authError(done, "third party user id required")
-  }
-  if (!thirdPartyUser.email) {
-    return authError(done, "third party user email required")
-  }
-
-  // use the third party id
-  const userId = generateGlobalUserID(thirdPartyUser.userId)
-  const db = getGlobalDB()
-
-  let dbUser
-
-  // try to load by id
-  try {
-    dbUser = await db.get(userId)
-  } catch (err: any) {
-    // abort when not 404 error
-    if (!err.status || err.status !== 404) {
-      return authError(
-        done,
-        "Unexpected error when retrieving existing user",
-        err
-      )
-    }
-  }
-
-  // fallback to loading by email
-  if (!dbUser) {
-    dbUser = await users.getGlobalUserByEmail(thirdPartyUser.email)
-  }
-
-  // exit early if there is still no user and auto creation is disabled
-  if (!dbUser && requireLocalAccount) {
-    return authError(
-      done,
-      "Email does not yet exist. You must set up your local budibase account first."
-    )
-  }
-
-  // first time creation
-  if (!dbUser) {
-    // setup a blank user using the third party id
-    dbUser = {
-      _id: userId,
-      email: thirdPartyUser.email,
-      roles: {},
-    }
-  }
-
-  dbUser = await syncUser(dbUser, thirdPartyUser)
-
-  // never prompt for password reset
-  dbUser.forceResetPassword = false
-
-  // create or sync the user
-  try {
-    await saveUserFn(dbUser, { hashPassword: false, requirePassword: false })
-  } catch (err: any) {
-    return authError(done, err)
-  }
-
-  // now that we're sure user exists, load them from the db
-  dbUser = await db.get(dbUser._id)
-
-  // authenticate
-  const sessionId = newid()
-  const tenantId = getTenantId()
-  await createASession(dbUser._id, { sessionId, tenantId })
-
-  dbUser.token = jwt.sign(
-    {
-      userId: dbUser._id,
-      sessionId,
-    },
-    env.JWT_SECRET
-  )
-
-  return done(null, dbUser)
-}
-
-async function syncProfilePicture(
-  user: ThirdPartyUser,
-  thirdPartyUser: ThirdPartyUser
-) {
-  const pictureUrl = thirdPartyUser.profile?._json.picture
-  if (pictureUrl) {
-    const response = await fetch(pictureUrl)
-
-    if (response.status === 200) {
-      const type = response.headers.get("content-type") as string
-      if (type.startsWith("image/")) {
-        user.pictureUrl = pictureUrl
-      }
-    }
-  }
-
-  return user
-}
-
-/**
- * @returns a user that has been sync'd with third party information
- */
-async function syncUser(user: ThirdPartyUser, thirdPartyUser: ThirdPartyUser) {
-  // provider
-  user.provider = thirdPartyUser.provider
-  user.providerType = thirdPartyUser.providerType
-
-  if (thirdPartyUser.profile) {
-    const profile = thirdPartyUser.profile
-
-    if (profile.name) {
-      const name = profile.name
-      // first name
-      if (name.givenName) {
-        user.firstName = name.givenName
-      }
-      // last name
-      if (name.familyName) {
-        user.lastName = name.familyName
-      }
-    }
-
-    user = await syncProfilePicture(user, thirdPartyUser)
-
-    // profile
-    user.thirdPartyProfile = {
-      ...profile._json,
-    }
-  }
-
-  // oauth tokens for future use
-  if (thirdPartyUser.oauth2) {
-    user.oauth2 = {
-      ...thirdPartyUser.oauth2,
-    }
-  }
-
-  return user
-}

packages/backend-core/src/middleware/passport/utils.ts

@@ -1,6 +1,6 @@
-import { isMultiTenant, getTenantId } from "../../tenancy"
+import { isMultiTenant, getTenantId } from "../../context"
 import { getScopedConfig } from "../../db"
-import { ConfigType, Database, Config } from "@budibase/types"
+import { ConfigType, Database } from "@budibase/types"
 
 /**
  * Utility to handle authentication errors.

packages/backend-core/src/middleware/tenancy.ts

@@ -1,4 +1,5 @@
-import { doInTenant, getTenantIDFromCtx } from "../tenancy"
+import { doInTenant } from "../context"
+import { getTenantIDFromCtx } from "../tenancy"
 import { buildMatcherRegex, matches } from "./matchers"
 import { Header } from "../constants"
 import {

packages/backend-core/src/migrations/migrations.ts

@@ -4,10 +4,10 @@ import {
   StaticDatabases,
   getAllApps,
   getGlobalDBName,
-  doWithDB,
+  getDB,
 } from "../db"
 import environment from "../environment"
-import { doInTenant, getTenantIds, getTenantId } from "../tenancy"
+import * as platform from "../platform"
 import * as context from "../context"
 import { DEFINITIONS } from "."
 import {
@@ -47,7 +47,7 @@ export const runMigration = async (
   const migrationType = migration.type
   let tenantId: string | undefined
   if (migrationType !== MigrationType.INSTALLATION) {
-    tenantId = getTenantId()
+    tenantId = context.getTenantId()
   }
   const migrationName = migration.name
   const silent = migration.silent
@@ -86,66 +86,65 @@ export const runMigration = async (
     count++
     const lengthStatement = length > 1 ? `[${count}/${length}]` : ""
 
-    await doWithDB(dbName, async (db: any) => {
+    const db = getDB(dbName)
-      try {
+    try {
-        const doc = await getMigrationsDoc(db)
+      const doc = await getMigrationsDoc(db)
 
-        // the migration has already been run
+      // the migration has already been run
-        if (doc[migrationName]) {
+      if (doc[migrationName]) {
-          // check for force
+        // check for force
-          if (
+        if (
-            options.force &&
+          options.force &&
-            options.force[migrationType] &&
+          options.force[migrationType] &&
-            options.force[migrationType].includes(migrationName)
+          options.force[migrationType].includes(migrationName)
-          ) {
+        ) {
-            log(
-              `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
-            )
-          } else {
-            // no force, exit
-            return
-          }
-        }
-
-        // check if the migration is not a no-op
-        if (!options.noOp) {
           log(
-            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
+            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
-          )
-
-          if (migration.preventRetry) {
-            // eagerly set the completion date
-            // so that we never run this migration twice even upon failure
-            doc[migrationName] = Date.now()
-            const response = await db.put(doc)
-            doc._rev = response.rev
-          }
-
-          // run the migration
-          if (migrationType === MigrationType.APP) {
-            await context.doInAppContext(db.name, async () => {
-              await migration.fn(db)
-            })
-          } else {
-            await migration.fn(db)
-          }
-
-          log(
-            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
           )
+        } else {
+          // no force, exit
+          return
         }
-
-        // mark as complete
-        doc[migrationName] = Date.now()
-        await db.put(doc)
-      } catch (err) {
-        console.error(
-          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
-          err
-        )
-        throw err
       }
-    })
+
+      // check if the migration is not a no-op
+      if (!options.noOp) {
+        log(
+          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
+        )
+
+        if (migration.preventRetry) {
+          // eagerly set the completion date
+          // so that we never run this migration twice even upon failure
+          doc[migrationName] = Date.now()
+          const response = await db.put(doc)
+          doc._rev = response.rev
+        }
+
+        // run the migration
+        if (migrationType === MigrationType.APP) {
+          await context.doInAppContext(db.name, async () => {
+            await migration.fn(db)
+          })
+        } else {
+          await migration.fn(db)
+        }
+
+        log(
+          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
+        )
+      }
+
+      // mark as complete
+      doc[migrationName] = Date.now()
+      await db.put(doc)
+    } catch (err) {
+      console.error(
+        `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
+        err
+      )
+      throw err
+    }
   }
 }
 
@@ -160,7 +159,7 @@ export const runMigrations = async (
       tenantIds = [options.noOp.tenantId]
     } else if (!options.tenantIds || !options.tenantIds.length) {
       // run for all tenants
-      tenantIds = await getTenantIds()
+      tenantIds = await platform.tenants.getTenantIds()
     } else {
       tenantIds = options.tenantIds
     }
@@ -185,7 +184,10 @@ export const runMigrations = async (
     // for all migrations
     for (const migration of migrations) {
       // run the migration
-      await doInTenant(tenantId, () => runMigration(migration, options))
+      await context.doInTenant(
+        tenantId,
+        async () => await runMigration(migration, options)
+      )
     }
   }
   console.log("Migrations complete")

packages/backend-core/src/migrations/tests/index.spec.js

@@ -1,57 +0,0 @@
-require("../../../tests")
-const { runMigrations, getMigrationsDoc } = require("../index")
-const { getGlobalDBName, getDB } = require("../../db")
-
-const { structures, testEnv } = require("../../../tests")
-testEnv.multiTenant()
-
-let db
-
-describe("migrations", () => {
-
-  const migrationFunction = jest.fn()
-
-  const MIGRATIONS = [{
-    type: "global",
-    name: "test",
-    fn: migrationFunction
-  }]
-
-  let tenantId
-
-  beforeEach(() => {
-    tenantId = structures.tenant.id()
-    db = getDB(getGlobalDBName(tenantId))
-  })
-
-  afterEach(async () => {
-    jest.clearAllMocks()
-    await db.destroy()
-  })
-
-  const migrate = () => {
-    return runMigrations(MIGRATIONS, { tenantIds: [tenantId]})
-  }
-
-  it("should run a new migration", async () => {
-    await migrate()
-    expect(migrationFunction).toHaveBeenCalled()
-    const doc = await getMigrationsDoc(db)
-    expect(doc.test).toBeDefined()
-  })
-
-  it("should match snapshot", async () => {
-    await migrate()
-    const doc = await getMigrationsDoc(db)
-    expect(doc).toMatchSnapshot()
-  })
-
-  it("should skip a previously run migration", async () => {
-    await migrate()
-    const previousMigrationTime = await getMigrationsDoc(db).test
-    await migrate()
-    const currentMigrationTime = await getMigrationsDoc(db).test
-    expect(migrationFunction).toHaveBeenCalledTimes(1)
-    expect(currentMigrationTime).toBe(previousMigrationTime)
-  })
-})

packages/backend-core/src/migrations/tests/migrations.spec.ts

@@ -0,0 +1,64 @@
+import { testEnv, DBTestConfiguration } from "../../../tests"
+import * as migrations from "../index"
+import * as context from "../../context"
+import { MigrationType } from "@budibase/types"
+
+testEnv.multiTenant()
+
+describe("migrations", () => {
+  const config = new DBTestConfiguration()
+
+  const migrationFunction = jest.fn()
+
+  const MIGRATIONS = [
+    {
+      type: MigrationType.GLOBAL,
+      name: "test" as any,
+      fn: migrationFunction,
+    },
+  ]
+
+  beforeEach(() => {
+    config.newTenant()
+  })
+
+  afterEach(async () => {
+    jest.clearAllMocks()
+  })
+
+  const migrate = () => {
+    return migrations.runMigrations(MIGRATIONS, {
+      tenantIds: [config.tenantId],
+    })
+  }
+
+  it("should run a new migration", async () => {
+    await config.doInTenant(async () => {
+      await migrate()
+      expect(migrationFunction).toHaveBeenCalled()
+      const db = context.getGlobalDB()
+      const doc = await migrations.getMigrationsDoc(db)
+      expect(doc.test).toBeDefined()
+    })
+  })
+
+  it("should match snapshot", async () => {
+    await config.doInTenant(async () => {
+      await migrate()
+      const doc = await migrations.getMigrationsDoc(context.getGlobalDB())
+      expect(doc).toMatchSnapshot()
+    })
+  })
+
+  it("should skip a previously run migration", async () => {
+    await config.doInTenant(async () => {
+      const db = context.getGlobalDB()
+      await migrate()
+      const previousDoc = await migrations.getMigrationsDoc(db)
+      await migrate()
+      const currentDoc = await migrations.getMigrationsDoc(db)
+      expect(migrationFunction).toHaveBeenCalledTimes(1)
+      expect(currentDoc.test).toBe(previousDoc.test)
+    })
+  })
+})

packages/backend-core/src/objectStore/buckets/global.ts

@@ -1,5 +1,5 @@
 import env from "../../environment"
-import * as tenancy from "../../tenancy"
+import * as context from "../../context"
 import * as objectStore from "../objectStore"
 import * as cloudfront from "../cloudfront"
 
@@ -22,7 +22,7 @@ export const getGlobalFileUrl = (type: string, name: string, etag?: string) => {
 export const getGlobalFileS3Key = (type: string, name: string) => {
   let file = `${type}/${name}`
   if (env.MULTI_TENANCY) {
-    const tenantId = tenancy.getTenantId()
+    const tenantId = context.getTenantId()
     file = `${tenantId}/${file}`
   }
   return file

packages/backend-core/src/objectStore/buckets/plugins.ts

@@ -1,6 +1,6 @@
 import env from "../../environment"
 import * as objectStore from "../objectStore"
-import * as tenancy from "../../tenancy"
+import * as context from "../../context"
 import * as cloudfront from "../cloudfront"
 import { Plugin } from "@budibase/types"
 
@@ -61,7 +61,7 @@ const getPluginS3Key = (plugin: Plugin, fileName: string) => {
 export const getPluginS3Dir = (pluginName: string) => {
   let s3Key = `${pluginName}`
   if (env.MULTI_TENANCY) {
-    const tenantId = tenancy.getTenantId()
+    const tenantId = context.getTenantId()
     s3Key = `${tenantId}/${s3Key}`
   }
   if (env.CLOUDFRONT_CDN) {

packages/backend-core/src/platform/index.ts

@@ -0,0 +1,3 @@
+export * as users from "./users"
+export * as tenants from "./tenants"
+export * from "./platformDb"

packages/backend-core/src/platform/platformDb.ts

@@ -0,0 +1,6 @@
+import { StaticDatabases } from "../constants"
+import { getDB } from "../db/db"
+
+export function getPlatformDB() {
+  return getDB(StaticDatabases.PLATFORM_INFO.name)
+}

packages/backend-core/src/platform/tenants.ts

@@ -0,0 +1,101 @@
+import { StaticDatabases } from "../constants"
+import { getPlatformDB } from "./platformDb"
+import { LockName, LockOptions, LockType, Tenants } from "@budibase/types"
+import * as locks from "../redis/redlock"
+
+const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
+
+export const tenacyLockOptions: LockOptions = {
+  type: LockType.DEFAULT,
+  name: LockName.UPDATE_TENANTS_DOC,
+  ttl: 10 * 1000, // auto expire after 10 seconds
+  systemLock: true,
+}
+
+// READ
+
+export async function getTenantIds(): Promise<string[]> {
+  const tenants = await getTenants()
+  return tenants.tenantIds
+}
+
+async function getTenants(): Promise<Tenants> {
+  const db = getPlatformDB()
+  let tenants: Tenants
+
+  try {
+    tenants = await db.get(TENANT_DOC)
+  } catch (e: any) {
+    // doesn't exist yet - create
+    if (e.status === 404) {
+      tenants = await createTenantsDoc()
+    } else {
+      throw e
+    }
+  }
+
+  return tenants
+}
+
+export async function exists(tenantId: string) {
+  const tenants = await getTenants()
+  return tenants.tenantIds.indexOf(tenantId) !== -1
+}
+
+// CREATE / UPDATE
+
+function newTenantsDoc(): Tenants {
+  return {
+    _id: TENANT_DOC,
+    tenantIds: [],
+  }
+}
+
+async function createTenantsDoc(): Promise<Tenants> {
+  const db = getPlatformDB()
+  let tenants = newTenantsDoc()
+
+  try {
+    const response = await db.put(tenants)
+    tenants._rev = response.rev
+  } catch (e: any) {
+    // don't throw 409 is doc has already been created
+    if (e.status === 409) {
+      return db.get(TENANT_DOC)
+    }
+    throw e
+  }
+
+  return tenants
+}
+
+export async function addTenant(tenantId: string) {
+  const db = getPlatformDB()
+
+  // use a lock as tenant creation is conflict prone
+  await locks.doWithLock(tenacyLockOptions, async () => {
+    const tenants = await getTenants()
+
+    // write the new tenant if it doesn't already exist
+    if (tenants.tenantIds.indexOf(tenantId) === -1) {
+      tenants.tenantIds.push(tenantId)
+      await db.put(tenants)
+    }
+  })
+}
+
+// DELETE
+
+export async function removeTenant(tenantId: string) {
+  try {
+    await locks.doWithLock(tenacyLockOptions, async () => {
+      const db = getPlatformDB()
+      const tenants = await getTenants()
+      tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)
+      await db.put(tenants)
+    })
+  } catch (err) {
+    console.error(`Error removing tenant ${tenantId} from info db`, err)
+    throw err
+  }
+}

packages/backend-core/src/platform/tests/tenants.spec.ts

@@ -0,0 +1,25 @@
+import { DBTestConfiguration, structures } from "../../../tests"
+import * as tenants from "../tenants"
+
+describe("tenants", () => {
+  const config = new DBTestConfiguration()
+
+  describe("addTenant", () => {
+    it("concurrently adds multiple tenants safely", async () => {
+      const tenant1 = structures.tenant.id()
+      const tenant2 = structures.tenant.id()
+      const tenant3 = structures.tenant.id()
+
+      await Promise.all([
+        tenants.addTenant(tenant1),
+        tenants.addTenant(tenant2),
+        tenants.addTenant(tenant3),
+      ])
+
+      const tenantIds = await tenants.getTenantIds()
+      expect(tenantIds.includes(tenant1)).toBe(true)
+      expect(tenantIds.includes(tenant2)).toBe(true)
+      expect(tenantIds.includes(tenant3)).toBe(true)
+    })
+  })
+})

packages/backend-core/src/platform/users.ts

@@ -0,0 +1,90 @@
+import { getPlatformDB } from "./platformDb"
+import { DEFAULT_TENANT_ID } from "../constants"
+import env from "../environment"
+import {
+  PlatformUser,
+  PlatformUserByEmail,
+  PlatformUserById,
+  User,
+} from "@budibase/types"
+
+// READ
+
+export async function lookupTenantId(userId: string) {
+  if (!env.MULTI_TENANCY) {
+    return DEFAULT_TENANT_ID
+  }
+
+  const user = await getUserDoc(userId)
+  return user.tenantId
+}
+
+async function getUserDoc(emailOrId: string): Promise<PlatformUser> {
+  const db = getPlatformDB()
+  return db.get(emailOrId)
+}
+
+// CREATE
+
+function newUserIdDoc(id: string, tenantId: string): PlatformUserById {
+  return {
+    _id: id,
+    tenantId,
+  }
+}
+
+function newUserEmailDoc(
+  userId: string,
+  email: string,
+  tenantId: string
+): PlatformUserByEmail {
+  return {
+    _id: email,
+    userId,
+    tenantId,
+  }
+}
+
+/**
+ * Add a new user id or email doc if it doesn't exist.
+ */
+async function addUserDoc(emailOrId: string, newDocFn: () => PlatformUser) {
+  const db = getPlatformDB()
+  let user: PlatformUser
+
+  try {
+    await db.get(emailOrId)
+  } catch (e: any) {
+    if (e.status === 404) {
+      user = newDocFn()
+      await db.put(user)
+    } else {
+      throw e
+    }
+  }
+}
+
+export async function addUser(tenantId: string, userId: string, email: string) {
+  await Promise.all([
+    addUserDoc(userId, () => newUserIdDoc(userId, tenantId)),
+    addUserDoc(email, () => newUserEmailDoc(userId, email, tenantId)),
+  ])
+}
+
+// DELETE
+
+export async function removeUser(user: User) {
+  const db = getPlatformDB()
+  const keys = [user._id!, user.email]
+  const userDocs = await db.allDocs({
+    keys,
+    include_docs: true,
+  })
+  const toDelete = userDocs.rows.map((row: any) => {
+    return {
+      ...row.doc,
+      _deleted: true,
+    }
+  })
+  await db.bulkDocs(toDelete)
+}

packages/backend-core/src/queue/constants.ts

@@ -1,4 +1,5 @@
 export enum JobQueue {
   AUTOMATION = "automationQueue",
   APP_BACKUP = "appBackupQueue",
+  AUDIT_LOG = "auditLogQueue",
 }

packages/backend-core/src/queue/queue.ts

@@ -4,7 +4,6 @@ import { JobQueue } from "./constants"
 import InMemoryQueue from "./inMemoryQueue"
 import BullQueue from "bull"
 import { addListeners, StalledFn } from "./listeners"
-const { opts: redisOpts, redisProtocolUrl } = getRedisOptions()
 
 const CLEANUP_PERIOD_MS = 60 * 1000
 let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
@@ -20,6 +19,7 @@ export function createQueue<T>(
   jobQueue: JobQueue,
   opts: { removeStalledCb?: StalledFn } = {}
 ): BullQueue.Queue<T> {
+  const { opts: redisOpts, redisProtocolUrl } = getRedisOptions()
   const queueConfig: any = redisProtocolUrl || { redis: redisOpts }
   let queue: any
   if (!env.isTest()) {

packages/backend-core/src/redis/index.ts

@@ -3,4 +3,4 @@
 export { default as Client } from "./redis"
 export * as utils from "./utils"
 export * as clients from "./init"
-export * as redlock from "./redlock"
+export * as locks from "./redlock"

packages/backend-core/src/redis/init.ts

@@ -20,13 +20,17 @@ async function init() {
   ).init()
 }
 
-process.on("exit", async () => {
+export async function shutdown() {
   if (userClient) await userClient.finish()
   if (sessionClient) await sessionClient.finish()
   if (appClient) await appClient.finish()
   if (cacheClient) await cacheClient.finish()
   if (writethroughClient) await writethroughClient.finish()
   if (lockClient) await lockClient.finish()
+}
+
+process.on("exit", async () => {
+  await shutdown()
 })
 
 export async function getUserClient() {

packages/backend-core/src/redis/redis.ts

@@ -1,6 +1,6 @@
 import env from "../environment"
 // ioredis mock is all in memory
-const Redis = env.isTest() ? require("ioredis-mock") : require("ioredis")
+const Redis = env.MOCK_REDIS ? require("ioredis-mock") : require("ioredis")
 import {
   addDbPrefix,
   removeDbPrefix,
@@ -17,8 +17,13 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT
 // for testing just generate the client once
 let CLOSED = false
 let CLIENTS: { [key: number]: any } = {}
-// if in test always connected
+
-let CONNECTED = env.isTest()
+let CONNECTED = false
+
+// mock redis always connected
+if (env.MOCK_REDIS) {
+  CONNECTED = true
+}
 
 function pickClient(selectDb: number): any {
   return CLIENTS[selectDb]
@@ -57,7 +62,7 @@ function init(selectDb = DEFAULT_SELECT_DB) {
     return
   }
   // testing uses a single in memory client
-  if (env.isTest()) {
+  if (env.MOCK_REDIS) {
     CLIENTS[selectDb] = new Redis(getRedisOptions())
   }
   // start the timer - only allowed 5 seconds to connect
@@ -86,6 +91,11 @@ function init(selectDb = DEFAULT_SELECT_DB) {
   }
   // attach handlers
   client.on("end", (err: Error) => {
+    if (env.isTest()) {
+      // don't try to re-connect in test env
+      // allow the process to exit
+      return
+    }
     connectionError(selectDb, timeout, err)
   })
   client.on("error", (err: Error) => {

packages/backend-core/src/redis/redlock.ts

@@ -1,29 +1,22 @@
 import Redlock, { Options } from "redlock"
 import { getLockClient } from "./init"
 import { LockOptions, LockType } from "@budibase/types"
-import * as tenancy from "../tenancy"
+import * as context from "../context"
-
+import env from "../environment"
-let noRetryRedlock: Redlock | undefined
 
 const getClient = async (type: LockType): Promise<Redlock> => {
+  if (env.isTest() && type !== LockType.TRY_ONCE) {
+    return newRedlock(OPTIONS.TEST)
+  }
   switch (type) {
     case LockType.TRY_ONCE: {
-      if (!noRetryRedlock) {
+      return newRedlock(OPTIONS.TRY_ONCE)
-        noRetryRedlock = await newRedlock(OPTIONS.TRY_ONCE)
-      }
-      return noRetryRedlock
     }
     case LockType.DEFAULT: {
-      if (!noRetryRedlock) {
+      return newRedlock(OPTIONS.DEFAULT)
-        noRetryRedlock = await newRedlock(OPTIONS.DEFAULT)
-      }
-      return noRetryRedlock
     }
     case LockType.DELAY_500: {
-      if (!noRetryRedlock) {
+      return newRedlock(OPTIONS.DELAY_500)
-        noRetryRedlock = await newRedlock(OPTIONS.DELAY_500)
-      }
-      return noRetryRedlock
     }
     default: {
       throw new Error(`Could not get redlock client: ${type}`)
@@ -36,6 +29,11 @@ export const OPTIONS = {
     // immediately throws an error if the lock is already held
     retryCount: 0,
   },
+  TEST: {
+    // higher retry count in unit tests
+    // due to high contention.
+    retryCount: 100,
+  },
   DEFAULT: {
     // the expected clock drift; for more details
     // see http://redis.io/topics/distlock
@@ -69,12 +67,19 @@ export const doWithLock = async (opts: LockOptions, task: any) => {
   const redlock = await getClient(opts.type)
   let lock
   try {
-    // aquire lock
+    // determine lock name
-    let name: string = `lock:${tenancy.getTenantId()}_${opts.name}`
+    // by default use the tenantId for uniqueness, unless using a system lock
+    const prefix = opts.systemLock ? "system" : context.getTenantId()
+    let name: string = `lock:${prefix}_${opts.name}`
+
+    // add additional unique name if required
     if (opts.nameSuffix) {
       name = name + `_${opts.nameSuffix}`
     }
+
+    // create the lock
     lock = await redlock.lock(name, opts.ttl)
+
     // perform locked task
     // need to await to ensure completion before unlocking
     const result = await task()

packages/backend-core/src/redis/utils.ts

@@ -2,8 +2,6 @@ import env from "../environment"
 
 const SLOT_REFRESH_MS = 2000
 const CONNECT_TIMEOUT_MS = 10000
-const REDIS_URL = !env.REDIS_URL ? "localhost:6379" : env.REDIS_URL
-const REDIS_PASSWORD = !env.REDIS_PASSWORD ? "budibase" : env.REDIS_PASSWORD
 export const SEPARATOR = "-"
 
 /**
@@ -60,8 +58,8 @@ export enum SelectableDatabase {
 }
 
 export function getRedisOptions(clustered = false) {
-  let password = REDIS_PASSWORD
+  let password = env.REDIS_PASSWORD
-  let url: string[] | string = REDIS_URL.split("//")
+  let url: string[] | string = env.REDIS_URL.split("//")
   // get rid of the protocol
   url = url.length > 1 ? url[1] : url[0]
   // check for a password etc
@@ -78,8 +76,8 @@ export function getRedisOptions(clustered = false) {
   let redisProtocolUrl
 
   // fully qualified redis URL
-  if (/rediss?:\/\//.test(REDIS_URL)) {
+  if (/rediss?:\/\//.test(env.REDIS_URL)) {
-    redisProtocolUrl = REDIS_URL
+    redisProtocolUrl = env.REDIS_URL
   }
 
   const opts: any = {

packages/backend-core/src/tenancy/db.ts

@@ -0,0 +1,6 @@
+import { getDB } from "../db/db"
+import { getGlobalDBName } from "../context"
+
+export function getTenantDB(tenantId: string) {
+  return getDB(getGlobalDBName(tenantId))
+}

packages/backend-core/src/tenancy/index.ts

@@ -1,2 +1,2 @@
-export * from "../context"
+export * from "./db"
 export * from "./tenancy"

packages/backend-core/src/tenancy/tenancy.ts

@@ -1,4 +1,3 @@
-import { doWithDB, getGlobalDBName } from "../db"
 import {
   DEFAULT_TENANT_ID,
   getTenantId,
@@ -11,10 +10,7 @@ import {
   TenantResolutionStrategy,
   GetTenantIdOptions,
 } from "@budibase/types"
-import { Header, StaticDatabases } from "../constants"
+import { Header } from "../constants"
-
-const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
-const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
 
 export function addTenantToUrl(url: string) {
   const tenantId = getTenantId()
@@ -27,89 +23,6 @@ export function addTenantToUrl(url: string) {
   return url
 }
 
-export async function doesTenantExist(tenantId: string) {
-  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
-    let tenants
-    try {
-      tenants = await db.get(TENANT_DOC)
-    } catch (err) {
-      // if theres an error the doc doesn't exist, no tenants exist
-      return false
-    }
-    return (
-      tenants &&
-      Array.isArray(tenants.tenantIds) &&
-      tenants.tenantIds.indexOf(tenantId) !== -1
-    )
-  })
-}
-
-export async function tryAddTenant(
-  tenantId: string,
-  userId: string,
-  email: string,
-  afterCreateTenant: () => Promise<void>
-) {
-  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
-    const getDoc = async (id: string) => {
-      if (!id) {
-        return null
-      }
-      try {
-        return await db.get(id)
-      } catch (err) {
-        return { _id: id }
-      }
-    }
-    let [tenants, userIdDoc, emailDoc] = await Promise.all([
-      getDoc(TENANT_DOC),
-      getDoc(userId),
-      getDoc(email),
-    ])
-    if (!Array.isArray(tenants.tenantIds)) {
-      tenants = {
-        _id: TENANT_DOC,
-        tenantIds: [],
-      }
-    }
-    let promises = []
-    if (userIdDoc) {
-      userIdDoc.tenantId = tenantId
-      promises.push(db.put(userIdDoc))
-    }
-    if (emailDoc) {
-      emailDoc.tenantId = tenantId
-      emailDoc.userId = userId
-      promises.push(db.put(emailDoc))
-    }
-    if (tenants.tenantIds.indexOf(tenantId) === -1) {
-      tenants.tenantIds.push(tenantId)
-      promises.push(db.put(tenants))
-      await afterCreateTenant()
-    }
-    await Promise.all(promises)
-  })
-}
-
-export function doWithGlobalDB(tenantId: string, cb: any) {
-  return doWithDB(getGlobalDBName(tenantId), cb)
-}
-
-export async function lookupTenantId(userId: string) {
-  return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: any) => {
-    let tenantId = env.MULTI_TENANCY ? DEFAULT_TENANT_ID : null
-    try {
-      const doc = await db.get(userId)
-      if (doc && doc.tenantId) {
-        tenantId = doc.tenantId
-      }
-    } catch (err) {
-      // just return the default
-    }
-    return tenantId
-  })
-}
-
 export const isUserInAppTenant = (appId: string, user?: any) => {
   let userTenantId
   if (user) {
@@ -121,19 +34,6 @@ export const isUserInAppTenant = (appId: string, user?: any) => {
   return tenantId === userTenantId
 }
 
-export async function getTenantIds() {
-  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
-    let tenants
-    try {
-      tenants = await db.get(TENANT_DOC)
-    } catch (err) {
-      // if theres an error the doc doesn't exist, no tenants exist
-      return []
-    }
-    return (tenants && tenants.tenantIds) || []
-  })
-}
-
 const ALL_STRATEGIES = Object.values(TenantResolutionStrategy)
 
 export const getTenantIDFromCtx = (

packages/backend-core/src/users.ts

@@ -8,6 +8,7 @@ import {
 } from "./db"
 import { BulkDocsResponse, User } from "@budibase/types"
 import { getGlobalDB } from "./context"
+import * as context from "./context"
 
 export const bulkGetGlobalUsersById = async (userIds: string[]) => {
   const db = getGlobalDB()
@@ -24,6 +25,11 @@ export const bulkUpdateGlobalUsers = async (users: User[]) => {
   return (await db.bulkDocs(users)) as BulkDocsResponse
 }
 
+export async function getById(id: string): Promise<User> {
+  const db = context.getGlobalDB()
+  return db.get(id)
+}
+
 /**
  * Given an email address this will use a view to search through
  * all the users to find one with this email address.

packages/backend-core/src/utils/tests/utils.spec.ts

@@ -1,21 +1,12 @@
-import { structures } from "../../../tests"
+import { structures, DBTestConfiguration } from "../../../tests"
 import * as utils from "../../utils"
-import * as events from "../../events"
 import * as db from "../../db"
 import { Header } from "../../constants"
-import { doInTenant } from "../../context"
 import { newid } from "../../utils"
+import env from "../../environment"
 
 describe("utils", () => {
-  describe("platformLogout", () => {
+  const config = new DBTestConfiguration()
-    it("should call platform logout", async () => {
-      await doInTenant(structures.tenant.id(), async () => {
-        const ctx = structures.koa.newContext()
-        await utils.platformLogout({ ctx, userId: "test" })
-        expect(events.auth.logout).toBeCalledTimes(1)
-      })
-    })
-  })
 
   describe("getAppIdFromCtx", () => {
     it("gets appId from header", async () => {
@@ -50,21 +41,28 @@ describe("utils", () => {
     })
 
     it("gets appId from url", async () => {
-      const ctx = structures.koa.newContext()
+      await config.doInTenant(async () => {
-      const expected = db.generateAppID()
+        const url = "http://test.com"
-      const app = structures.apps.app(expected)
+        env._set("PLATFORM_URL", url)
 
-      // set custom url
+        const ctx = structures.koa.newContext()
-      const appUrl = newid()
+        ctx.host = `${config.tenantId}.test.com`
-      app.url = `/${appUrl}`
-      ctx.path = `/app/${appUrl}`
 
-      // save the app
+        const expected = db.generateAppID(config.tenantId)
-      const database = db.getDB(expected)
+        const app = structures.apps.app(expected)
-      await database.put(app)
 
-      const actual = await utils.getAppIdFromCtx(ctx)
+        // set custom url
-      expect(actual).toBe(expected)
+        const appUrl = newid()
+        app.url = `/${appUrl}`
+        ctx.path = `/app/${appUrl}`
+
+        // save the app
+        const database = db.getDB(expected)
+        await database.put(app)
+
+        const actual = await utils.getAppIdFromCtx(ctx)
+        expect(actual).toBe(expected)
+      })
     })
 
     it("doesn't get appId from url when previewing", async () => {

packages/backend-core/src/utils/utils.ts

@@ -2,23 +2,15 @@ import { getAllApps, queryGlobalView } from "../db"
 import { options } from "../middleware/passport/jwt"
 import {
   Header,
-  Cookie,
   MAX_VALID_DATE,
   DocumentType,
   SEPARATOR,
   ViewName,
 } from "../constants"
 import env from "../environment"
-import * as userCache from "../cache/user"
-import { getSessionsForUser, invalidateSessions } from "../security/sessions"
-import * as events from "../events"
 import * as tenancy from "../tenancy"
-import {
+import * as context from "../context"
-  App,
+import { App, Ctx, TenantResolutionStrategy } from "@budibase/types"
-  Ctx,
-  PlatformLogoutOpts,
-  TenantResolutionStrategy,
-} from "@budibase/types"
 import { SetOption } from "cookies"
 const jwt = require("jsonwebtoken")
 
@@ -38,7 +30,7 @@ export async function resolveAppUrl(ctx: Ctx) {
   const appUrl = ctx.path.split("/")[2]
   let possibleAppUrl = `/${appUrl.toLowerCase()}`
 
-  let tenantId: string | null = tenancy.getTenantId()
+  let tenantId: string | null = context.getTenantId()
   if (env.MULTI_TENANCY) {
     // always use the tenant id from the subdomain in multi tenancy
     // this ensures the logged-in user tenant id doesn't overwrite
@@ -49,7 +41,7 @@ export async function resolveAppUrl(ctx: Ctx) {
   }
 
   // search prod apps for a url that matches
-  const apps: App[] = await tenancy.doInTenant(tenantId, () =>
+  const apps: App[] = await context.doInTenant(tenantId, () =>
     getAllApps({ dev: false })
   )
   const app = apps.filter(

packages/backend-core/tests/jestEnv.ts

@@ -1,23 +1,6 @@
-import env from "../src/environment"
+process.env.SELF_HOSTED = "1"
-import { mocks } from "./utilities"
+process.env.MULTI_TENANCY = "1"
-
+process.env.NODE_ENV = "jest"
-// must explicitly enable fetch mock
+process.env.MOCK_REDIS = "1"
-mocks.fetch.enable()
+process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
-
+process.env.ENABLE_4XX_HTTP_LOGGING = "0"
-// mock all dates to 2020-01-01T00:00:00.000Z
-// use tk.reset() to use real dates in individual tests
-import tk from "timekeeper"
-tk.freeze(mocks.date.MOCK_DATE)
-
-env._set("SELF_HOSTED", "1")
-env._set("NODE_ENV", "jest")
-
-if (!process.env.DEBUG) {
-  global.console.log = jest.fn() // console.log are ignored in tests
-}
-
-if (!process.env.CI) {
-  // set a longer timeout in dev for debugging
-  // 100 seconds
-  jest.setTimeout(100000)
-}

packages/backend-core/tests/jestSetup.ts

@@ -1,4 +1,23 @@
+import "./logging"
 import env from "../src/environment"
-import { testContainerUtils } from "./utilities"
+import { mocks, testContainerUtils } from "./utilities"
+
+// must explicitly enable fetch mock
+mocks.fetch.enable()
+
+// mock all dates to 2020-01-01T00:00:00.000Z
+// use tk.reset() to use real dates in individual tests
+import tk from "timekeeper"
+tk.freeze(mocks.date.MOCK_DATE)
+
+if (!process.env.DEBUG) {
+  console.log = jest.fn() // console.log are ignored in tests
+}
+
+if (!process.env.CI) {
+  // set a longer timeout in dev for debugging
+  // 100 seconds
+  jest.setTimeout(100000)
+}
 
 testContainerUtils.setupEnv(env)

packages/backend-core/tests/logging.ts

@@ -0,0 +1,34 @@
+export enum LogLevel {
+  TRACE = "trace",
+  DEBUG = "debug",
+  INFO = "info",
+  WARN = "warn",
+  ERROR = "error",
+}
+
+const LOG_INDEX: { [key in LogLevel]: number } = {
+  [LogLevel.TRACE]: 1,
+  [LogLevel.DEBUG]: 2,
+  [LogLevel.INFO]: 3,
+  [LogLevel.WARN]: 4,
+  [LogLevel.ERROR]: 5,
+}
+
+const setIndex = LOG_INDEX[process.env.LOG_LEVEL as LogLevel]
+
+if (setIndex > LOG_INDEX.trace) {
+  global.console.trace = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.debug) {
+  global.console.debug = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.info) {
+  global.console.info = jest.fn()
+  global.console.log = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.warn) {
+  global.console.warn = jest.fn()
+}

packages/backend-core/tests/utilities/DBTestConfiguration.ts

@@ -0,0 +1,36 @@
+import "./mocks"
+import * as structures from "./structures"
+import * as testEnv from "./testEnv"
+import * as context from "../../src/context"
+
+class DBTestConfiguration {
+  tenantId: string
+
+  constructor() {
+    // db tests need to be multi tenant to prevent conflicts
+    testEnv.multiTenant()
+    this.tenantId = structures.tenant.id()
+  }
+
+  newTenant() {
+    this.tenantId = structures.tenant.id()
+  }
+
+  // TENANCY
+
+  doInTenant(task: any) {
+    return context.doInTenant(this.tenantId, () => {
+      return task()
+    })
+  }
+
+  getTenantId() {
+    try {
+      return context.getTenantId()
+    } catch (e) {
+      return this.tenantId!
+    }
+  }
+}
+
+export default DBTestConfiguration

packages/backend-core/tests/utilities/db.ts

@@ -1,9 +0,0 @@
-import * as db from "../../src/db"
-
-const dbConfig = {
-  inMemory: true,
-}
-
-export const init = () => {
-  db.init(dbConfig)
-}

packages/backend-core/tests/utilities/index.ts

@@ -4,5 +4,4 @@ export { generator } from "./structures"
 export * as testEnv from "./testEnv"
 export * as testContainerUtils from "./testContainerUtils"
 
-import * as dbConfig from "./db"
+export { default as DBTestConfiguration } from "./DBTestConfiguration"
-dbConfig.init()

packages/backend-core/tests/utilities/mocks/accounts.ts

@@ -1,13 +0,0 @@
-const mockGetAccount = jest.fn()
-const mockGetAccountByTenantId = jest.fn()
-const mockGetStatus = jest.fn()
-
-jest.mock("../../../src/cloud/accounts", () => ({
-  getAccount: mockGetAccount,
-  getAccountByTenantId: mockGetAccountByTenantId,
-  getStatus: mockGetStatus,
-}))
-
-export const getAccount = mockGetAccount
-export const getAccountByTenantId = mockGetAccountByTenantId
-export const getStatus = mockGetStatus

packages/backend-core/tests/utilities/mocks/index.ts

@@ -1,4 +1,7 @@
-export * as accounts from "./accounts"
+jest.mock("../../../src/accounts")
+import * as _accounts from "../../../src/accounts"
+export const accounts = jest.mocked(_accounts)
+
 export * as date from "./date"
 export * as licenses from "./licenses"
 export { default as fetch } from "./fetch"

packages/backend-core/tests/utilities/structures/accounts.ts

@@ -1,6 +1,15 @@
 import { generator, uuid } from "."
 import * as db from "../../../src/db/utils"
-import { Account, AuthType, CloudAccount, Hosting } from "@budibase/types"
+import {
+  Account,
+  AccountSSOProvider,
+  AccountSSOProviderType,
+  AuthType,
+  CloudAccount,
+  Hosting,
+  SSOAccount,
+} from "@budibase/types"
+import _ from "lodash"
 
 export const account = (): Account => {
   return {
@@ -27,3 +36,28 @@ export const cloudAccount = (): CloudAccount => {
     budibaseUserId: db.generateGlobalUserID(),
   }
 }
+
+function providerType(): AccountSSOProviderType {
+  return _.sample(
+    Object.values(AccountSSOProviderType)
+  ) as AccountSSOProviderType
+}
+
+function provider(): AccountSSOProvider {
+  return _.sample(Object.values(AccountSSOProvider)) as AccountSSOProvider
+}
+
+export function ssoAccount(): SSOAccount {
+  return {
+    ...cloudAccount(),
+    authType: AuthType.SSO,
+    oauth2: {
+      accessToken: generator.string(),
+      refreshToken: generator.string(),
+    },
+    pictureUrl: generator.url(),
+    provider: provider(),
+    providerType: providerType(),
+    thirdPartyProfile: {},
+  }
+}

packages/backend-core/tests/utilities/structures/index.ts

@@ -5,8 +5,10 @@ export const generator = new Chance()
 
 export * as accounts from "./accounts"
 export * as apps from "./apps"
+export * as db from "./db"
 export * as koa from "./koa"
 export * as licenses from "./licenses"
 export * as plugins from "./plugins"
+export * as sso from "./sso"
 export * as tenant from "./tenants"
-export * as db from "./db"
+export * as users from "./users"

packages/backend-core/tests/utilities/structures/sso.ts

@@ -0,0 +1,100 @@
+import {
+  GoogleInnerConfig,
+  JwtClaims,
+  OIDCInnerConfig,
+  OIDCWellKnownConfig,
+  SSOAuthDetails,
+  SSOProfile,
+  SSOProviderType,
+  User,
+} from "@budibase/types"
+import { uuid, generator, users, email } from "./index"
+import _ from "lodash"
+
+export function providerType(): SSOProviderType {
+  return _.sample(Object.values(SSOProviderType)) as SSOProviderType
+}
+
+export function ssoProfile(user?: User): SSOProfile {
+  if (!user) {
+    user = users.user()
+  }
+  return {
+    id: user._id!,
+    name: {
+      givenName: user.firstName,
+      familyName: user.lastName,
+    },
+    _json: {
+      email: user.email,
+      picture: "http://test.com",
+    },
+    provider: generator.string(),
+  }
+}
+
+export function authDetails(user?: User): SSOAuthDetails {
+  if (!user) {
+    user = users.user()
+  }
+
+  const userId = user._id || uuid()
+  const provider = generator.string()
+
+  const profile = ssoProfile(user)
+  profile.provider = provider
+  profile.id = userId
+
+  return {
+    email: user.email,
+    oauth2: {
+      refreshToken: generator.string(),
+      accessToken: generator.string(),
+    },
+    profile,
+    provider,
+    providerType: providerType(),
+    userId,
+  }
+}
+
+// OIDC
+
+export function oidcConfig(): OIDCInnerConfig {
+  return {
+    uuid: uuid(),
+    activated: true,
+    logo: "",
+    name: generator.string(),
+    configUrl: "http://someconfigurl",
+    clientID: generator.string(),
+    clientSecret: generator.string(),
+  }
+}
+
+// response from .well-known/openid-configuration
+export function oidcWellKnownConfig(): OIDCWellKnownConfig {
+  return {
+    issuer: generator.string(),
+    authorization_endpoint: generator.url(),
+    token_endpoint: generator.url(),
+    userinfo_endpoint: generator.url(),
+  }
+}
+
+export function jwtClaims(): JwtClaims {
+  return {
+    email: email(),
+    preferred_username: email(),
+  }
+}
+
+// GOOGLE
+
+export function googleConfig(): GoogleInnerConfig {
+  return {
+    activated: true,
+    clientID: generator.string(),
+    clientSecret: generator.string(),
+  }
+}

packages/backend-core/tests/utilities/structures/users.ts

@@ -0,0 +1,70 @@
+import { generator } from "../"
+import {
+  AdminUser,
+  BuilderUser,
+  SSOAuthDetails,
+  SSOUser,
+  User,
+} from "@budibase/types"
+import { v4 as uuid } from "uuid"
+import * as sso from "./sso"
+
+export const newEmail = () => {
+  return `${uuid()}@test.com`
+}
+
+export const user = (userProps?: any): User => {
+  return {
+    email: newEmail(),
+    password: "test",
+    roles: { app_test: "admin" },
+    firstName: generator.first(),
+    lastName: generator.last(),
+    pictureUrl: "http://test.com",
+    ...userProps,
+  }
+}
+
+export const adminUser = (userProps?: any): AdminUser => {
+  return {
+    ...user(userProps),
+    admin: {
+      global: true,
+    },
+    builder: {
+      global: true,
+    },
+  }
+}
+
+export const builderUser = (userProps?: any): BuilderUser => {
+  return {
+    ...user(userProps),
+    builder: {
+      global: true,
+    },
+  }
+}
+
+export function ssoUser(
+  opts: { user?: any; details?: SSOAuthDetails } = {}
+): SSOUser {
+  const base = user(opts.user)
+  delete base.password
+
+  if (!opts.details) {
+    opts.details = sso.authDetails(base)
+  }
+
+  return {
+    ...base,
+    forceResetPassword: false,
+    oauth2: opts.details?.oauth2,
+    provider: opts.details?.provider!,
+    providerType: opts.details?.providerType!,
+    thirdPartyProfile: {
+      email: base.email,
+      picture: base.pictureUrl,
+    },
+  }
+}

packages/backend-core/tests/utilities/testContainerUtils.ts

@@ -34,12 +34,17 @@ function getMinioConfig() {
   return getContainerInfo("minio-service", 9000)
 }
 
+function getRedisConfig() {
+  return getContainerInfo("redis-service", 6379)
+}
+
 export function setupEnv(...envs: any[]) {
   const configs = [
     { key: "COUCH_DB_PORT", value: getCouchConfig().port },
     { key: "COUCH_DB_URL", value: getCouchConfig().url },
     { key: "MINIO_PORT", value: getMinioConfig().port },
     { key: "MINIO_URL", value: getMinioConfig().url },
+    { key: "REDIS_URL", value: getRedisConfig().url },
   ]
 
   for (const config of configs.filter(x => !!x.value)) {

packages/backend-core/tests/utilities/testEnv.ts

@@ -1,12 +1,12 @@
 import env from "../../src/environment"
-import * as tenancy from "../../src/tenancy"
+import * as context from "../../src/context"
-import { newid } from "../../src/utils"
+import * as structures from "./structures"
 
 // TENANCY
 
 export async function withTenant(task: (tenantId: string) => any) {
-  const tenantId = newid()
+  const tenantId = structures.tenant.id()
-  return tenancy.doInTenant(tenantId, async () => {
+  return context.doInTenant(tenantId, async () => {
     await task(tenantId)
   })
 }
@@ -19,6 +19,14 @@ export function multiTenant() {
   env._set("MULTI_TENANCY", 1)
 }
 
+export function selfHosted() {
+  env._set("SELF_HOSTED", 1)
+}
+
+export function cloudHosted() {
+  env._set("SELF_HOSTED", 0)
+}
+
 // NODE
 
 export function nodeDev() {

packages/backend-core/yarn.lock

@@ -1197,10 +1197,10 @@
   dependencies:
     "@types/istanbul-lib-report" "*"
 
-"@types/jest@27.5.1":
+"@types/jest@28.1.1":
-  version "27.5.1"
+  version "28.1.1"
-  resolved "https://registry.yarnpkg.com/@types/jest/-/jest-27.5.1.tgz#2c8b6dc6ff85c33bcd07d0b62cb3d19ddfdb3ab9"
+  resolved "https://registry.yarnpkg.com/@types/jest/-/jest-28.1.1.tgz#8c9ba63702a11f8c386ee211280e8b68cb093cd1"
-  integrity sha512-fUy7YRpT+rHXto1YlL+J9rs0uLGyiqVt3ZOTQR+4ROc47yNl8WLdVLgUloBRhOxP1PZvguHl44T3H0wAWxahYQ==
+  integrity sha512-C2p7yqleUKtCkVjlOur9BWVA4HgUQmEj/HWCt5WzZ5mLXrWnyIfl0wGuArc+kBXsy0ZZfLp+7dywB4HtSVYGVA==
   dependencies:
     jest-matcher-utils "^27.0.0"
     pretty-format "^27.0.0"

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.3.14-alpha.0",
+  "version": "2.3.18-alpha.0",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/string-templates": "2.3.14-alpha.0",
+    "@budibase/string-templates": "2.3.18-alpha.0",
     "@spectrum-css/accordion": "3.0.24",
     "@spectrum-css/actionbutton": "1.0.1",
     "@spectrum-css/actiongroup": "1.0.1",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.3.14-alpha.0",
+  "version": "2.3.18-alpha.0",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -58,10 +58,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.3.14-alpha.0",
+    "@budibase/bbui": "2.3.18-alpha.0",
-    "@budibase/client": "2.3.14-alpha.0",
+    "@budibase/client": "2.3.18-alpha.0",
-    "@budibase/frontend-core": "2.3.14-alpha.0",
+    "@budibase/frontend-core": "2.3.18-alpha.0",
-    "@budibase/string-templates": "2.3.14-alpha.0",
+    "@budibase/string-templates": "2.3.18-alpha.0",
     "@fortawesome/fontawesome-svg-core": "^6.2.1",
     "@fortawesome/free-brands-svg-icons": "^6.2.1",
     "@fortawesome/free-solid-svg-icons": "^6.2.1",

packages/builder/setup.js

@@ -19,6 +19,7 @@ process.env.COUCH_DB_USER = "budibase"
 process.env.COUCH_DB_PASSWORD = "budibase"
 process.env.INTERNAL_API_KEY = "budibase"
 process.env.ALLOW_DEV_AUTOMATIONS = 1
+process.env.MOCK_REDIS = 1
 
 // Stop info logs polluting test outputs
 process.env.LOG_LEVEL = "error"

packages/builder/src/builderStore/store/frontend.js

@@ -312,7 +312,7 @@ export const getFrontendStore = () => {
         const screensToDelete = Array.isArray(screens) ? screens : [screens]
 
         // Build array of promises to speed up bulk deletions
-        const promises = []
+        let promises = []
         let deleteUrls = []
         screensToDelete.forEach(screen => {
           // Delete the screen
@@ -326,8 +326,8 @@ export const getFrontendStore = () => {
           deleteUrls.push(screen.routing.route)
         })
 
-        promises.push(store.actions.links.delete(deleteUrls))
         await Promise.all(promises)
+        await store.actions.links.delete(deleteUrls)
         const deletedIds = screensToDelete.map(screen => screen._id)
         const routesResponse = await API.fetchAppRoutes()
         store.update(state => {

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -18,7 +18,6 @@
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
   import {
     FIELDS,
-    AUTO_COLUMN_SUB_TYPES,
     RelationshipTypes,
     ALLOWABLE_STRING_OPTIONS,
     ALLOWABLE_NUMBER_OPTIONS,
@@ -132,12 +131,6 @@
     : availableAutoColumns
 
   // used to select what different options can be displayed for column type
-  $: canBeSearched =
-    editableColumn?.type !== LINK_TYPE &&
-    editableColumn?.type !== JSON_TYPE &&
-    editableColumn?.subtype !== AUTO_COLUMN_SUB_TYPES.CREATED_BY &&
-    editableColumn?.subtype !== AUTO_COLUMN_SUB_TYPES.UPDATED_BY &&
-    editableColumn?.type !== FORMULA_TYPE
   $: canBeDisplay =
     editableColumn?.type !== LINK_TYPE &&
     editableColumn?.type !== AUTO_TYPE &&
@@ -254,18 +247,6 @@
     }
   }
 
-  function onChangePrimaryIndex(e) {
-    indexes = e.detail ? [editableColumn.name] : []
-  }
-
-  function onChangeSecondaryIndex(e) {
-    if (e.detail) {
-      indexes[1] = editableColumn.name
-    } else {
-      indexes = indexes.slice(0, 1)
-    }
-  }
-
   function openJsonSchemaEditor() {
     jsonSchemaModal.show()
   }
@@ -460,24 +441,6 @@
     </div>
   {/if}
 
-  {#if canBeSearched && !external}
-    <div>
-      <Label>Search Indexes</Label>
-      <Toggle
-        value={indexes[0] === editableColumn.name}
-        disabled={indexes[1] === editableColumn.name}
-        on:change={onChangePrimaryIndex}
-        text="Primary"
-      />
-      <Toggle
-        value={indexes[1] === editableColumn.name}
-        disabled={!indexes[0] || indexes[0] === editableColumn.name}
-        on:change={onChangeSecondaryIndex}
-        text="Secondary"
-      />
-    </div>
-  {/if}
-
   {#if editableColumn.type === "string"}
     <Input
       type="number"

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -183,6 +183,7 @@
     bind:this={popover}
     anchor={popoverAnchor}
     maxWidth={300}
+    dismissible={false}
   >
     <Layout gap="S">
       <div class="helper">

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/FetchRow.svelte

@@ -0,0 +1,40 @@
+<script>
+  import { Select, Label } from "@budibase/bbui"
+  import { tables } from "stores/backend"
+  import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
+
+  export let parameters
+  export let bindings = []
+
+  $: tableOptions = $tables.list || []
+</script>
+
+<div class="root">
+  <Label>Table</Label>
+  <Select
+    bind:value={parameters.tableId}
+    options={tableOptions}
+    getOptionLabel={table => table.name}
+    getOptionValue={table => table._id}
+  />
+
+  <Label small>Row ID</Label>
+  <DrawerBindableInput
+    {bindings}
+    title="Row ID to Fetch"
+    value={parameters.rowId}
+    on:change={value => (parameters.rowId = value.detail)}
+  />
+</div>
+
+<style>
+  .root {
+    display: grid;
+    column-gap: var(--spacing-l);
+    row-gap: var(--spacing-s);
+    grid-template-columns: 60px 1fr;
+    align-items: center;
+    max-width: 800px;
+    margin: 0 auto;
+  }
+</style>

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/index.js

@@ -1,3 +1,4 @@
+export { default as FetchRow } from "./FetchRow.svelte"
 export { default as NavigateTo } from "./NavigateTo.svelte"
 export { default as SaveRow } from "./SaveRow.svelte"
 export { default as DeleteRow } from "./DeleteRow.svelte"

packages/builder/src/components/design/settings/controls/ButtonActionEditor/manifest.json

@@ -27,6 +27,17 @@
       "type": "data",
       "component": "DeleteRow"
     },
+    {
+      "name": "Fetch Row",
+      "type": "data",
+      "component": "FetchRow",
+      "context": [
+        {
+          "label": "Fetched row",
+          "value": "row"
+        }
+      ]
+    },
     {
       "name": "Navigate To",
       "type": "application",
@@ -135,4 +146,4 @@
       "dependsOnFeature": "sidePanel"
     }
   ]
-}
+}

packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte

@@ -101,7 +101,7 @@
   }
 
   const getSchema = filter => {
-    return schemaFields.find(field => field.name === filter.field)
+    return enrichedSchemaFields.find(field => field.name === filter.field)
   }
 
   const santizeTypes = filter => {
@@ -254,8 +254,8 @@
               {:else if filter.type === "datetime"}
                 <DatePicker
                   disabled={filter.noValue}
-                  enableTime={!getSchema(filter).dateOnly}
+                  enableTime={!getSchema(filter)?.dateOnly}
-                  timeOnly={getSchema(filter).timeOnly}
+                  timeOnly={getSchema(filter)?.timeOnly}
                   bind:value={filter.value}
                 />
               {:else}