Merge branch 'develop' of github.com:Budibase/budibase into cheeks-lab-day-spreadsheet

.github/workflows/budibase_ci.yml

@@ -11,7 +11,6 @@ on:
     branches:
       - master
       - develop 
-      - release 
  workflow_dispatch:
 
 env:
@@ -20,9 +19,53 @@ env:
   PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
 
 jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Use Node.js 14.x
+      uses: actions/setup-node@v1
+      with:
+        node-version: 14.x
+    - run: yarn
+    - run: yarn lint
+
   build:
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - name: Install Pro
+        run: yarn install:pro $BRANCH $BASE_BRANCH
+      - run: yarn
+      - run: yarn bootstrap
+      - run: yarn build
 
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - name: Install Pro
+        run: yarn install:pro $BRANCH $BASE_BRANCH
+      - run: yarn
+      - run: yarn bootstrap
+      - run: yarn test
+      - uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
+          files: ./packages/server/coverage/clover.xml,./packages/worker/coverage/clover.xml,./packages/backend-core/coverage/clover.xml
+          name: codecov-umbrella
+          verbose: true
+
+  integration-test:
+    runs-on: ubuntu-latest
     services:
       couchdb:
         image: ibmcom/couchdb3
@@ -31,39 +74,18 @@ jobs:
           COUCHDB_USER: budibase
         ports:
           - 4567:5984
-
-    strategy:
-      matrix:
-        node-version: [14.x]
-
     steps:
-    - uses: actions/checkout@v2
-
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-
-    - name: Install Pro
-      run: yarn install:pro $BRANCH $BASE_BRANCH
-
-    - run: yarn
-    - run: yarn bootstrap
-    - run: yarn lint
-    - run: yarn build
-    - run: yarn test
-      env:
-        CI: true
-        name: Budibase CI
-    - uses: codecov/codecov-action@v1
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
-        files: ./packages/server/coverage/clover.xml,./packages/worker/coverage/clover.xml,./packages/backend-core/coverage/clover.xml
-        name: codecov-umbrella
-        verbose: true 
-
-    - name: QA Core Integration Tests
-      run: | 
-        cd qa-core
-        yarn
-        yarn api:test:ci
+      - uses: actions/checkout@v2
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - name: Install Pro
+        run: yarn install:pro $BRANCH $BASE_BRANCH
+      - run: yarn
+      - run: yarn bootstrap
+      - run: yarn build
+      - run: |
+          cd qa-core
+          yarn
+          yarn api:test:ci

.husky/pre-commit

@@ -1,4 +1,2 @@
 #!/bin/sh
 . "$(dirname "$0")/_/husky.sh"
-
-yarn run lint

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.18-alpha.8",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/jest.config.ts

@@ -9,15 +9,9 @@ const baseConfig: Config.InitialProjectOptions = {
   transform: {
     "^.+\\.ts?$": "@swc/jest",
   },
-}
-
-if (!process.env.CI) {
-  // use sources when not in CI
-  baseConfig.moduleNameMapper = {
+  moduleNameMapper: {
     "@budibase/types": "<rootDir>/../types/src",
-  }
-} else {
-  console.log("Running tests with compiled dependency sources")
+  },
 }
 
 const config: Config.InitialOptions = {

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.18-alpha.8",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -18,13 +18,13 @@
     "build:pro": "../../scripts/pro/build.sh",
     "postbuild": "yarn run build:pro",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
-    "test": "jest --coverage --maxWorkers=2",
+    "test": "bash scripts/test.sh",
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
     "@budibase/nano": "10.1.1",
     "@budibase/pouchdb-replication-stream": "1.2.10",
-    "@budibase/types": "2.3.17-alpha.4",
+    "@budibase/types": "2.3.18-alpha.8",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",
@@ -62,7 +62,7 @@
     "@trendyol/jest-testcontainers": "^2.1.1",
     "@types/chance": "1.1.3",
     "@types/ioredis": "4.28.0",
-    "@types/jest": "27.5.1",
+    "@types/jest": "28.1.1",
     "@types/koa": "2.13.4",
     "@types/koa-pino-logger": "3.0.0",
     "@types/lodash": "4.14.180",

packages/backend-core/scripts/test.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [[ -n $CI ]]
+then
+  # --runInBand performs better in ci where resources are limited
+  echo "jest --coverage --runInBand"
+  jest --coverage --runInBand
+else
+  # --maxWorkers performs better in development
+  echo "jest --coverage"
+  jest --coverage
+fi

packages/backend-core/src/accounts/accounts.ts

@@ -1,13 +1,24 @@
 import API from "./api"
 import env from "../environment"
 import { Header } from "../constants"
-import { CloudAccount } from "@budibase/types"
+import { CloudAccount, HealthStatusResponse } from "@budibase/types"
 
 const api = new API(env.ACCOUNT_PORTAL_URL)
 
+/**
+ * This client is intended to be used in a cloud hosted deploy only.
+ * Rather than relying on each consumer to perform the necessary environmental checks
+ * we use the following check to exit early with a undefined response which should be
+ * handled by the caller.
+ */
+const EXIT_EARLY = env.SELF_HOSTED || env.DISABLE_ACCOUNT_PORTAL
+
 export const getAccount = async (
   email: string
 ): Promise<CloudAccount | undefined> => {
+  if (EXIT_EARLY) {
+    return
+  }
   const payload = {
     email,
   }
@@ -29,6 +40,9 @@ export const getAccount = async (
 export const getAccountByTenantId = async (
   tenantId: string
 ): Promise<CloudAccount | undefined> => {
+  if (EXIT_EARLY) {
+    return
+  }
   const payload = {
     tenantId,
   }
@@ -47,7 +61,12 @@ export const getAccountByTenantId = async (
   return json[0]
 }
 
-export const getStatus = async () => {
+export const getStatus = async (): Promise<
+  HealthStatusResponse | undefined
+> => {
+  if (EXIT_EARLY) {
+    return
+  }
   const response = await api.get(`/api/status`, {
     headers: {
       [Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,

packages/backend-core/src/accounts/index.ts

@@ -0,0 +1 @@
+export * from "./accounts"

packages/backend-core/src/auth/auth.ts

@@ -1,10 +1,11 @@
 const _passport = require("koa-passport")
 const LocalStrategy = require("passport-local").Strategy
 const JwtStrategy = require("passport-jwt").Strategy
-import { getGlobalDB } from "../tenancy"
+import { getGlobalDB } from "../context"
 const refresh = require("passport-oauth2-refresh")
-import { Config } from "../constants"
+import { Config, Cookie } from "../constants"
 import { getScopedConfig } from "../db"
+import { getSessionsForUser, invalidateSessions } from "../security/sessions"
 import {
   jwt as jwtPassport,
   local,
@@ -15,8 +16,11 @@ import {
   google,
 } from "../middleware"
 import { invalidateUser } from "../cache/user"
-import { User } from "@budibase/types"
+import { PlatformLogoutOpts, User } from "@budibase/types"
 import { logAlert } from "../logging"
+import * as events from "../events"
+import * as userCache from "../cache/user"
+import { clearCookie, getCookie } from "../utils"
 export {
   auditLog,
   authError,
@@ -29,6 +33,7 @@ export {
   google,
   oidc,
 } from "../middleware"
+import { ssoSaveUserNoOp } from "../middleware/passport/sso/sso"
 export const buildAuthMiddleware = authenticated
 export const buildTenancyMiddleware = tenancy
 export const buildCsrfMiddleware = csrf
@@ -71,7 +76,7 @@ async function refreshOIDCAccessToken(
     if (!enrichedConfig) {
       throw new Error("OIDC Config contents invalid")
     }
-    strategy = await oidc.strategyFactory(enrichedConfig)
+    strategy = await oidc.strategyFactory(enrichedConfig, ssoSaveUserNoOp)
   } catch (err) {
     console.error(err)
     throw new Error("Could not refresh OAuth Token")
@@ -103,7 +108,11 @@ async function refreshGoogleAccessToken(
 
   let strategy
   try {
-    strategy = await google.strategyFactory(config, callbackUrl)
+    strategy = await google.strategyFactory(
+      config,
+      callbackUrl,
+      ssoSaveUserNoOp
+    )
   } catch (err: any) {
     console.error(err)
     throw new Error(
@@ -161,6 +170,8 @@ export async function refreshOAuthToken(
   return refreshResponse
 }
 
+// TODO: Refactor to use user save function instead to prevent the need for
+// manually saving and invalidating on callback
 export async function updateUserOAuth(userId: string, oAuthConfig: any) {
   const details = {
     accessToken: oAuthConfig.accessToken,
@@ -188,3 +199,32 @@ export async function updateUserOAuth(userId: string, oAuthConfig: any) {
     console.error("Could not update OAuth details for current user", e)
   }
 }
+
+/**
+ * Logs a user out from budibase. Re-used across account portal and builder.
+ */
+export async function platformLogout(opts: PlatformLogoutOpts) {
+  const ctx = opts.ctx
+  const userId = opts.userId
+  const keepActiveSession = opts.keepActiveSession
+
+  if (!ctx) throw new Error("Koa context must be supplied to logout.")
+
+  const currentSession = getCookie(ctx, Cookie.Auth)
+  let sessions = await getSessionsForUser(userId)
+
+  if (keepActiveSession) {
+    sessions = sessions.filter(
+      session => session.sessionId !== currentSession.sessionId
+    )
+  } else {
+    // clear cookies
+    clearCookie(ctx, Cookie.Auth)
+    clearCookie(ctx, Cookie.CurrentApp)
+  }
+
+  const sessionIds = sessions.map(({ sessionId }) => sessionId)
+  await invalidateSessions(userId, { sessionIds, reason: "logout" })
+  await events.auth.logout()
+  await userCache.invalidateUser(userId)
+}

packages/backend-core/src/auth/tests/auth.spec.ts

@@ -0,0 +1,13 @@
+import { structures, testEnv } from "../../../tests"
+import * as auth from "../auth"
+import * as events from "../../events"
+
+describe("platformLogout", () => {
+  it("should call platform logout", async () => {
+    await testEnv.withTenant(async () => {
+      const ctx = structures.koa.newContext()
+      await auth.platformLogout({ ctx, userId: "test" })
+      expect(events.auth.logout).toBeCalledTimes(1)
+    })
+  })
+})

packages/backend-core/src/cache/tests/writethrough.spec.js

@@ -1,61 +0,0 @@
-require("../../../tests")
-const { Writethrough } = require("../writethrough")
-const { getDB } = require("../../db")
-const tk = require("timekeeper")
-const { structures } = require("../../../tests")
-
-const START_DATE = Date.now()
-tk.freeze(START_DATE)
-
-
-const DELAY = 5000
-
-const db = getDB(structures.db.id())
-const db2 = getDB(structures.db.id())
-const writethrough = new Writethrough(db, DELAY), writethrough2 = new Writethrough(db2, DELAY)
-
-describe("writethrough", () => {
-  describe("put", () => {
-    let first
-    it("should be able to store, will go to DB", async () => {
-      const response = await writethrough.put({ _id: "test", value: 1 })
-      const output = await db.get(response.id)
-      first = output
-      expect(output.value).toBe(1)
-    })
-
-    it("second put shouldn't update DB", async () => {
-      const response = await writethrough.put({ ...first, value: 2 })
-      const output = await db.get(response.id)
-      expect(first._rev).toBe(output._rev)
-      expect(output.value).toBe(1)
-    })
-
-    it("should put it again after delay period", async () => {
-      tk.freeze(START_DATE + DELAY + 1)
-      const response = await writethrough.put({ ...first, value: 3 })
-      const output = await db.get(response.id)
-      expect(response.rev).not.toBe(first._rev)
-      expect(output.value).toBe(3)
-    })
-  })
-
-  describe("get", () => {
-    it("should be able to retrieve", async () => {
-      const response = await writethrough.get("test")
-      expect(response.value).toBe(3)
-    })
-  })
-
-  describe("same doc, different databases (tenancy)", () => {
-    it("should be able to two different databases", async () => {
-      const resp1 = await writethrough.put({ _id: "db1", value: "first" })
-      const resp2 = await writethrough2.put({ _id: "db1", value: "second" })
-      expect(resp1.rev).toBeDefined()
-      expect(resp2.rev).toBeDefined()
-      expect((await db.get("db1")).value).toBe("first")
-      expect((await db2.get("db1")).value).toBe("second")
-    })
-  })
-})
-

packages/backend-core/src/cache/tests/writethrough.spec.ts

@@ -0,0 +1,73 @@
+import { structures, DBTestConfiguration } from "../../../tests"
+import { Writethrough } from "../writethrough"
+import { getDB } from "../../db"
+import tk from "timekeeper"
+
+const START_DATE = Date.now()
+tk.freeze(START_DATE)
+
+const DELAY = 5000
+
+describe("writethrough", () => {
+  const config = new DBTestConfiguration()
+
+  const db = getDB(structures.db.id())
+  const db2 = getDB(structures.db.id())
+
+  const writethrough = new Writethrough(db, DELAY)
+  const writethrough2 = new Writethrough(db2, DELAY)
+
+  describe("put", () => {
+    let first: any
+
+    it("should be able to store, will go to DB", async () => {
+      await config.doInTenant(async () => {
+        const response = await writethrough.put({ _id: "test", value: 1 })
+        const output = await db.get(response.id)
+        first = output
+        expect(output.value).toBe(1)
+      })
+    })
+
+    it("second put shouldn't update DB", async () => {
+      await config.doInTenant(async () => {
+        const response = await writethrough.put({ ...first, value: 2 })
+        const output = await db.get(response.id)
+        expect(first._rev).toBe(output._rev)
+        expect(output.value).toBe(1)
+      })
+    })
+
+    it("should put it again after delay period", async () => {
+      await config.doInTenant(async () => {
+        tk.freeze(START_DATE + DELAY + 1)
+        const response = await writethrough.put({ ...first, value: 3 })
+        const output = await db.get(response.id)
+        expect(response.rev).not.toBe(first._rev)
+        expect(output.value).toBe(3)
+      })
+    })
+  })
+
+  describe("get", () => {
+    it("should be able to retrieve", async () => {
+      await config.doInTenant(async () => {
+        const response = await writethrough.get("test")
+        expect(response.value).toBe(3)
+      })
+    })
+  })
+
+  describe("same doc, different databases (tenancy)", () => {
+    it("should be able to two different databases", async () => {
+      await config.doInTenant(async () => {
+        const resp1 = await writethrough.put({ _id: "db1", value: "first" })
+        const resp2 = await writethrough2.put({ _id: "db1", value: "second" })
+        expect(resp1.rev).toBeDefined()
+        expect(resp2.rev).toBeDefined()
+        expect((await db.get("db1")).value).toBe("first")
+        expect((await db2.get("db1")).value).toBe("second")
+      })
+    })
+  })
+})

packages/backend-core/src/cache/user.ts

@@ -1,8 +1,9 @@
 import * as redis from "../redis/init"
-import { getTenantId, lookupTenantId, doWithGlobalDB } from "../tenancy"
+import * as tenancy from "../tenancy"
+import * as context from "../context"
+import * as platform from "../platform"
 import env from "../environment"
-import * as accounts from "../cloud/accounts"
-import { Database } from "@budibase/types"
+import * as accounts from "../accounts"
 
 const EXPIRY_SECONDS = 3600
 
@@ -10,7 +11,8 @@ const EXPIRY_SECONDS = 3600
  * The default populate user function
  */
 async function populateFromDB(userId: string, tenantId: string) {
-  const user = await doWithGlobalDB(tenantId, (db: Database) => db.get(userId))
+  const db = tenancy.getTenantDB(tenantId)
+  const user = await db.get(userId)
   user.budibaseAccess = true
   if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     const account = await accounts.getAccount(user.email)
@@ -42,9 +44,9 @@ export async function getUser(
   }
   if (!tenantId) {
     try {
-      tenantId = getTenantId()
+      tenantId = context.getTenantId()
     } catch (err) {
-      tenantId = await lookupTenantId(userId)
+      tenantId = await platform.users.lookupTenantId(userId)
     }
   }
   const client = await redis.getUserClient()

packages/backend-core/src/context/deprovision.ts

@@ -1,108 +0,0 @@
-import {
-  getGlobalUserParams,
-  getAllApps,
-  doWithDB,
-  StaticDatabases,
-} from "../db"
-import { doWithGlobalDB } from "../tenancy"
-import { App, Tenants, User, Database } from "@budibase/types"
-
-const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
-const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
-
-async function removeTenantFromInfoDB(tenantId: string) {
-  try {
-    await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => {
-      const tenants = (await infoDb.get(TENANT_DOC)) as Tenants
-      tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)
-
-      await infoDb.put(tenants)
-    })
-  } catch (err) {
-    console.error(`Error removing tenant ${tenantId} from info db`, err)
-    throw err
-  }
-}
-
-export async function removeUserFromInfoDB(dbUser: User) {
-  await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => {
-    const keys = [dbUser._id!, dbUser.email]
-    const userDocs = await infoDb.allDocs({
-      keys,
-      include_docs: true,
-    })
-    const toDelete = userDocs.rows.map((row: any) => {
-      return {
-        ...row.doc,
-        _deleted: true,
-      }
-    })
-    await infoDb.bulkDocs(toDelete)
-  })
-}
-
-async function removeUsersFromInfoDB(tenantId: string) {
-  return doWithGlobalDB(tenantId, async (db: any) => {
-    try {
-      const allUsers = await db.allDocs(
-        getGlobalUserParams(null, {
-          include_docs: true,
-        })
-      )
-      await doWithDB(PLATFORM_INFO_DB, async (infoDb: any) => {
-        const allEmails = allUsers.rows.map((row: any) => row.doc.email)
-        // get the id docs
-        let keys = allUsers.rows.map((row: any) => row.id)
-        // and the email docs
-        keys = keys.concat(allEmails)
-        // retrieve the docs and delete them
-        const userDocs = await infoDb.allDocs({
-          keys,
-          include_docs: true,
-        })
-        const toDelete = userDocs.rows.map((row: any) => {
-          return {
-            ...row.doc,
-            _deleted: true,
-          }
-        })
-        await infoDb.bulkDocs(toDelete)
-      })
-    } catch (err) {
-      console.error(`Error removing tenant ${tenantId} users from info db`, err)
-      throw err
-    }
-  })
-}
-
-async function removeGlobalDB(tenantId: string) {
-  return doWithGlobalDB(tenantId, async (db: Database) => {
-    try {
-      await db.destroy()
-    } catch (err) {
-      console.error(`Error removing tenant ${tenantId} users from info db`, err)
-      throw err
-    }
-  })
-}
-
-async function removeTenantApps(tenantId: string) {
-  try {
-    const apps = (await getAllApps({ all: true })) as App[]
-    const destroyPromises = apps.map(app =>
-      doWithDB(app.appId, (db: Database) => db.destroy())
-    )
-    await Promise.allSettled(destroyPromises)
-  } catch (err) {
-    console.error(`Error removing tenant ${tenantId} apps`, err)
-    throw err
-  }
-}
-
-// can't live in tenancy package due to circular dependency on db/utils
-export async function deleteTenant(tenantId: string) {
-  await removeTenantFromInfoDB(tenantId)
-  await removeUsersFromInfoDB(tenantId)
-  await removeGlobalDB(tenantId)
-  await removeTenantApps(tenantId)
-}

packages/backend-core/src/context/tests/index.spec.ts

@@ -1,11 +1,14 @@
-require("../../../tests")
+import { testEnv } from "../../../tests"
 const context = require("../")
 const { DEFAULT_TENANT_ID } = require("../../constants")
-import env from "../../environment"
 
 describe("context", () => {
   describe("doInTenant", () => {
     describe("single-tenancy", () => {
+      beforeAll(() => {
+        testEnv.singleTenant()
+      })
+
       it("defaults to the default tenant", () => {
         const tenantId = context.getTenantId()
         expect(tenantId).toBe(DEFAULT_TENANT_ID)
@@ -20,8 +23,8 @@ describe("context", () => {
     })
 
     describe("multi-tenancy", () => {
-      beforeEach(() => {
-        env._set("MULTI_TENANCY", 1)
+      beforeAll(() => {
+        testEnv.multiTenant()
       })
 
       it("fails when no tenant id is set", () => {

packages/backend-core/src/db/db.ts

@@ -1,7 +1,6 @@
 import env from "../environment"
-import { directCouchQuery, getPouchDB } from "./couch"
+import { directCouchQuery, DatabaseImpl } from "./couch"
 import { CouchFindOptions, Database } from "@budibase/types"
-import { DatabaseImpl } from "../db"
 
 const dbList = new Set()
 

packages/backend-core/src/db/tests/utils.seq.spec.ts

@@ -1,190 +0,0 @@
-require("../../../tests")
-const {
-  getDevelopmentAppID,
-  getProdAppID,
-  isDevAppID,
-  isProdAppID,
-} = require("../conversions")
-const { generateAppID, getPlatformUrl, getScopedConfig } = require("../utils")
-const tenancy = require("../../tenancy")
-const { Config, DEFAULT_TENANT_ID } = require("../../constants")
-import { generator } from "../../../tests"
-import env from "../../environment"
-
-describe("utils", () => {
-  describe("app ID manipulation", () => {
-    function getID() {
-      const appId = generateAppID()
-      const split = appId.split("_")
-      const uuid = split[split.length - 1]
-      const devAppId = `app_dev_${uuid}`
-      return { appId, devAppId, split, uuid }
-    }
-
-    it("should be able to generate a new app ID", () => {
-      expect(generateAppID().startsWith("app_")).toEqual(true)
-    })
-
-    it("should be able to convert a production app ID to development", () => {
-      const { appId, uuid } = getID()
-      expect(getDevelopmentAppID(appId)).toEqual(`app_dev_${uuid}`)
-    })
-
-    it("should be able to convert a development app ID to development", () => {
-      const { devAppId, uuid } = getID()
-      expect(getDevelopmentAppID(devAppId)).toEqual(`app_dev_${uuid}`)
-    })
-
-    it("should be able to convert a development ID to a production", () => {
-      const { devAppId, uuid } = getID()
-      expect(getProdAppID(devAppId)).toEqual(`app_${uuid}`)
-    })
-
-    it("should be able to convert a production ID to production", () => {
-      const { appId, uuid } = getID()
-      expect(getProdAppID(appId)).toEqual(`app_${uuid}`)
-    })
-
-    it("should be able to confirm dev app ID is development", () => {
-      const { devAppId } = getID()
-      expect(isDevAppID(devAppId)).toEqual(true)
-    })
-
-    it("should be able to confirm prod app ID is not development", () => {
-      const { appId } = getID()
-      expect(isDevAppID(appId)).toEqual(false)
-    })
-
-    it("should be able to confirm prod app ID is prod", () => {
-      const { appId } = getID()
-      expect(isProdAppID(appId)).toEqual(true)
-    })
-
-    it("should be able to confirm dev app ID is not prod", () => {
-      const { devAppId } = getID()
-      expect(isProdAppID(devAppId)).toEqual(false)
-    })
-  })
-})
-
-const DEFAULT_URL = "http://localhost:10000"
-const ENV_URL = "http://env.com"
-
-const setDbPlatformUrl = async (dbUrl: string) => {
-  const db = tenancy.getGlobalDB()
-  await db.put({
-    _id: "config_settings",
-    type: Config.SETTINGS,
-    config: {
-      platformUrl: dbUrl,
-    },
-  })
-}
-
-const clearSettingsConfig = async () => {
-  await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-    const db = tenancy.getGlobalDB()
-    try {
-      const config = await db.get("config_settings")
-      await db.remove("config_settings", config._rev)
-    } catch (e: any) {
-      if (e.status !== 404) {
-        throw e
-      }
-    }
-  })
-}
-
-describe("getPlatformUrl", () => {
-  describe("self host", () => {
-    beforeEach(async () => {
-      env._set("SELF_HOST", 1)
-      await clearSettingsConfig()
-    })
-
-    it("gets the default url", async () => {
-      await tenancy.doInTenant(null, async () => {
-        const url = await getPlatformUrl()
-        expect(url).toBe(DEFAULT_URL)
-      })
-    })
-
-    it("gets the platform url from the environment", async () => {
-      await tenancy.doInTenant(null, async () => {
-        env._set("PLATFORM_URL", ENV_URL)
-        const url = await getPlatformUrl()
-        expect(url).toBe(ENV_URL)
-      })
-    })
-
-    it("gets the platform url from the database", async () => {
-      await tenancy.doInTenant(null, async () => {
-        const dbUrl = generator.url()
-        await setDbPlatformUrl(dbUrl)
-        const url = await getPlatformUrl()
-        expect(url).toBe(dbUrl)
-      })
-    })
-  })
-
-  describe("cloud", () => {
-    const TENANT_AWARE_URL = "http://default.env.com"
-
-    beforeEach(async () => {
-      env._set("SELF_HOSTED", 0)
-      env._set("MULTI_TENANCY", 1)
-      env._set("PLATFORM_URL", ENV_URL)
-      await clearSettingsConfig()
-    })
-
-    it("gets the platform url from the environment without tenancy", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        const url = await getPlatformUrl({ tenantAware: false })
-        expect(url).toBe(ENV_URL)
-      })
-    })
-
-    it("gets the platform url from the environment with tenancy", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        const url = await getPlatformUrl()
-        expect(url).toBe(TENANT_AWARE_URL)
-      })
-    })
-
-    it("never gets the platform url from the database", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        await setDbPlatformUrl(generator.url())
-        const url = await getPlatformUrl()
-        expect(url).toBe(TENANT_AWARE_URL)
-      })
-    })
-  })
-})
-
-describe("getScopedConfig", () => {
-  describe("settings config", () => {
-    beforeEach(async () => {
-      env._set("SELF_HOSTED", 1)
-      env._set("PLATFORM_URL", "")
-      await clearSettingsConfig()
-    })
-
-    it("returns the platform url with an existing config", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        const dbUrl = generator.url()
-        await setDbPlatformUrl(dbUrl)
-        const db = tenancy.getGlobalDB()
-        const config = await getScopedConfig(db, { type: Config.SETTINGS })
-        expect(config.platformUrl).toBe(dbUrl)
-      })
-    })
-
-    it("returns the platform url without an existing config", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        const db = tenancy.getGlobalDB()
-        const config = await getScopedConfig(db, { type: Config.SETTINGS })
-        expect(config.platformUrl).toBe(DEFAULT_URL)
-      })
-    })
-  })
-})

packages/backend-core/src/db/tests/utils.spec.ts

@@ -0,0 +1,192 @@
+import { generator, DBTestConfiguration, testEnv } from "../../../tests"
+import {
+  getDevelopmentAppID,
+  getProdAppID,
+  isDevAppID,
+  isProdAppID,
+} from "../conversions"
+import { generateAppID, getPlatformUrl, getScopedConfig } from "../utils"
+import * as context from "../../context"
+import { Config } from "../../constants"
+import env from "../../environment"
+
+describe("utils", () => {
+  const config = new DBTestConfiguration()
+
+  describe("app ID manipulation", () => {
+    function getID() {
+      const appId = generateAppID()
+      const split = appId.split("_")
+      const uuid = split[split.length - 1]
+      const devAppId = `app_dev_${uuid}`
+      return { appId, devAppId, split, uuid }
+    }
+
+    it("should be able to generate a new app ID", () => {
+      expect(generateAppID().startsWith("app_")).toEqual(true)
+    })
+
+    it("should be able to convert a production app ID to development", () => {
+      const { appId, uuid } = getID()
+      expect(getDevelopmentAppID(appId)).toEqual(`app_dev_${uuid}`)
+    })
+
+    it("should be able to convert a development app ID to development", () => {
+      const { devAppId, uuid } = getID()
+      expect(getDevelopmentAppID(devAppId)).toEqual(`app_dev_${uuid}`)
+    })
+
+    it("should be able to convert a development ID to a production", () => {
+      const { devAppId, uuid } = getID()
+      expect(getProdAppID(devAppId)).toEqual(`app_${uuid}`)
+    })
+
+    it("should be able to convert a production ID to production", () => {
+      const { appId, uuid } = getID()
+      expect(getProdAppID(appId)).toEqual(`app_${uuid}`)
+    })
+
+    it("should be able to confirm dev app ID is development", () => {
+      const { devAppId } = getID()
+      expect(isDevAppID(devAppId)).toEqual(true)
+    })
+
+    it("should be able to confirm prod app ID is not development", () => {
+      const { appId } = getID()
+      expect(isDevAppID(appId)).toEqual(false)
+    })
+
+    it("should be able to confirm prod app ID is prod", () => {
+      const { appId } = getID()
+      expect(isProdAppID(appId)).toEqual(true)
+    })
+
+    it("should be able to confirm dev app ID is not prod", () => {
+      const { devAppId } = getID()
+      expect(isProdAppID(devAppId)).toEqual(false)
+    })
+  })
+
+  const DEFAULT_URL = "http://localhost:10000"
+  const ENV_URL = "http://env.com"
+
+  const setDbPlatformUrl = async (dbUrl: string) => {
+    const db = context.getGlobalDB()
+    await db.put({
+      _id: "config_settings",
+      type: Config.SETTINGS,
+      config: {
+        platformUrl: dbUrl,
+      },
+    })
+  }
+
+  const clearSettingsConfig = async () => {
+    await config.doInTenant(async () => {
+      const db = context.getGlobalDB()
+      try {
+        const config = await db.get("config_settings")
+        await db.remove("config_settings", config._rev)
+      } catch (e: any) {
+        if (e.status !== 404) {
+          throw e
+        }
+      }
+    })
+  }
+
+  describe("getPlatformUrl", () => {
+    describe("self host", () => {
+      beforeEach(async () => {
+        testEnv.selfHosted()
+        await clearSettingsConfig()
+      })
+
+      it("gets the default url", async () => {
+        await config.doInTenant(async () => {
+          const url = await getPlatformUrl()
+          expect(url).toBe(DEFAULT_URL)
+        })
+      })
+
+      it("gets the platform url from the environment", async () => {
+        await config.doInTenant(async () => {
+          env._set("PLATFORM_URL", ENV_URL)
+          const url = await getPlatformUrl()
+          expect(url).toBe(ENV_URL)
+        })
+      })
+
+      it("gets the platform url from the database", async () => {
+        await config.doInTenant(async () => {
+          const dbUrl = generator.url()
+          await setDbPlatformUrl(dbUrl)
+          const url = await getPlatformUrl()
+          expect(url).toBe(dbUrl)
+        })
+      })
+    })
+
+    describe("cloud", () => {
+      const TENANT_AWARE_URL = `http://${config.tenantId}.env.com`
+
+      beforeEach(async () => {
+        testEnv.cloudHosted()
+        testEnv.multiTenant()
+
+        env._set("PLATFORM_URL", ENV_URL)
+        await clearSettingsConfig()
+      })
+
+      it("gets the platform url from the environment without tenancy", async () => {
+        await config.doInTenant(async () => {
+          const url = await getPlatformUrl({ tenantAware: false })
+          expect(url).toBe(ENV_URL)
+        })
+      })
+
+      it("gets the platform url from the environment with tenancy", async () => {
+        await config.doInTenant(async () => {
+          const url = await getPlatformUrl()
+          expect(url).toBe(TENANT_AWARE_URL)
+        })
+      })
+
+      it("never gets the platform url from the database", async () => {
+        await config.doInTenant(async () => {
+          await setDbPlatformUrl(generator.url())
+          const url = await getPlatformUrl()
+          expect(url).toBe(TENANT_AWARE_URL)
+        })
+      })
+    })
+  })
+
+  describe("getScopedConfig", () => {
+    describe("settings config", () => {
+      beforeEach(async () => {
+        env._set("SELF_HOSTED", 1)
+        env._set("PLATFORM_URL", "")
+        await clearSettingsConfig()
+      })
+
+      it("returns the platform url with an existing config", async () => {
+        await config.doInTenant(async () => {
+          const dbUrl = generator.url()
+          await setDbPlatformUrl(dbUrl)
+          const db = context.getGlobalDB()
+          const config = await getScopedConfig(db, { type: Config.SETTINGS })
+          expect(config.platformUrl).toBe(dbUrl)
+        })
+      })
+
+      it("returns the platform url without an existing config", async () => {
+        await config.doInTenant(async () => {
+          const db = context.getGlobalDB()
+          const config = await getScopedConfig(db, { type: Config.SETTINGS })
+          expect(config.platformUrl).toBe(DEFAULT_URL)
+        })
+      })
+    })
+  })
+})

packages/backend-core/src/db/views.ts

@@ -1,13 +1,14 @@
 import {
-  DocumentType,
-  ViewName,
   DeprecatedViews,
+  DocumentType,
   SEPARATOR,
   StaticDatabases,
+  ViewName,
 } from "../constants"
 import { getGlobalDB } from "../context"
 import { doWithDB } from "./"
 import { Database, DatabaseQueryOpts } from "@budibase/types"
+import env from "../environment"
 
 const DESIGN_DB = "_design/database"
 
@@ -69,17 +70,6 @@ export const createNewUserEmailView = async () => {
   await createView(db, viewJs, ViewName.USER_BY_EMAIL)
 }
 
-export const createAccountEmailView = async () => {
-  const viewJs = `function(doc) {
-    if (doc._id.startsWith("${DocumentType.ACCOUNT_METADATA}${SEPARATOR}")) {
-      emit(doc.email.toLowerCase(), doc._id)
-    }
-  }`
-  await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
-    await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL)
-  })
-}
-
 export const createUserAppView = async () => {
   const db = getGlobalDB()
   const viewJs = `function(doc) {
@@ -113,17 +103,6 @@ export const createUserBuildersView = async () => {
   await createView(db, viewJs, ViewName.USER_BY_BUILDERS)
 }
 
-export const createPlatformUserView = async () => {
-  const viewJs = `function(doc) {
-    if (doc.tenantId) {
-      emit(doc._id.toLowerCase(), doc._id)
-    }
-  }`
-  await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
-    await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
-  })
-}
-
 export interface QueryViewOptions {
   arrayResponse?: boolean
 }
@@ -162,13 +141,48 @@ export const queryView = async <T>(
   }
 }
 
+// PLATFORM
+
+async function createPlatformView(viewJs: string, viewName: ViewName) {
+  try {
+    await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
+      await createView(db, viewJs, viewName)
+    })
+  } catch (e: any) {
+    if (e.status === 409 && env.isTest()) {
+      // multiple tests can try to initialise platforms views
+      // at once - safe to exit on conflict
+      return
+    }
+    throw e
+  }
+}
+
+export const createPlatformAccountEmailView = async () => {
+  const viewJs = `function(doc) {
+    if (doc._id.startsWith("${DocumentType.ACCOUNT_METADATA}${SEPARATOR}")) {
+      emit(doc.email.toLowerCase(), doc._id)
+    }
+  }`
+  await createPlatformView(viewJs, ViewName.ACCOUNT_BY_EMAIL)
+}
+
+export const createPlatformUserView = async () => {
+  const viewJs = `function(doc) {
+    if (doc.tenantId) {
+      emit(doc._id.toLowerCase(), doc._id)
+    }
+  }`
+  await createPlatformView(viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
+}
+
 export const queryPlatformView = async <T>(
   viewName: ViewName,
   params: DatabaseQueryOpts,
   opts?: QueryViewOptions
 ): Promise<T[] | T | undefined> => {
   const CreateFuncByName: any = {
-    [ViewName.ACCOUNT_BY_EMAIL]: createAccountEmailView,
+    [ViewName.ACCOUNT_BY_EMAIL]: createPlatformAccountEmailView,
     [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView,
   }
 

packages/backend-core/src/environment.ts

@@ -44,8 +44,9 @@ const environment = {
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
   GOOGLE_CLIENT_SECRET: process.env.GOOGLE_CLIENT_SECRET,
   SALT_ROUNDS: process.env.SALT_ROUNDS,
-  REDIS_URL: process.env.REDIS_URL,
-  REDIS_PASSWORD: process.env.REDIS_PASSWORD,
+  REDIS_URL: process.env.REDIS_URL || "localhost:6379",
+  REDIS_PASSWORD: process.env.REDIS_PASSWORD || "budibase",
+  MOCK_REDIS: process.env.MOCK_REDIS,
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   AWS_REGION: process.env.AWS_REGION,
@@ -82,6 +83,7 @@ const environment = {
   SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD,
   DEPLOYMENT_ENVIRONMENT:
     process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose",
+  ENABLE_4XX_HTTP_LOGGING: process.env.ENABLE_4XX_HTTP_LOGGING || true,
   _set(key: any, value: any) {
     process.env[key] = value
     // @ts-ignore

packages/backend-core/src/events/analytics.ts

@@ -1,5 +1,5 @@
 import env from "../environment"
-import * as tenancy from "../tenancy"
+import * as context from "../context"
 import * as dbUtils from "../db/utils"
 import { Config } from "../constants"
 import { withCache, TTL, CacheKey } from "../cache"
@@ -42,7 +42,7 @@ export const enabled = async () => {
 }
 
 const getSettingsDoc = async () => {
-  const db = tenancy.getGlobalDB()
+  const db = context.getGlobalDB()
   let settings
   try {
     settings = await db.get(dbUtils.generateConfigID({ type: Config.SETTINGS }))

packages/backend-core/src/events/identification.ts

@@ -16,6 +16,7 @@ import {
   InstallationGroup,
   UserContext,
   Group,
+  isSSOUser,
 } from "@budibase/types"
 import { processors } from "./processors"
 import * as dbUtils from "../db/utils"
@@ -166,7 +167,10 @@ const identifyUser = async (
   const type = IdentityType.USER
   let builder = user.builder?.global || false
   let admin = user.admin?.global || false
-  let providerType = user.providerType
+  let providerType
+  if (isSSOUser(user)) {
+    providerType = user.providerType
+  }
   const accountHolder = account?.budibaseUserId === user._id || false
   const verified =
     account && account?.budibaseUserId === user._id ? account.verified : false

packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts

@@ -1,4 +1,4 @@
-import "../../../../../tests"
+import { testEnv } from "../../../../../tests"
 import PosthogProcessor from "../PosthogProcessor"
 import { Event, IdentityType, Hosting } from "@budibase/types"
 const tk = require("timekeeper")
@@ -16,6 +16,10 @@ const newIdentity = () => {
 }
 
 describe("PosthogProcessor", () => {
+  beforeAll(() => {
+    testEnv.singleTenant()
+  })
+
   beforeEach(async () => {
     jest.clearAllMocks()
     await cache.bustCache(

packages/backend-core/src/featureFlags/index.ts

@@ -1,5 +1,5 @@
 import env from "../environment"
-import * as tenancy from "../tenancy"
+import * as context from "../context"
 
 /**
  * Read the TENANT_FEATURE_FLAGS env var and return an array of features flags for each tenant.
@@ -28,7 +28,7 @@ export function buildFeatureFlags() {
 }
 
 export function isEnabled(featureFlag: string) {
-  const tenantId = tenancy.getTenantId()
+  const tenantId = context.getTenantId()
   const flags = getTenantFeatureFlags(tenantId)
   return flags.includes(featureFlag)
 }

packages/backend-core/src/index.ts

@@ -3,12 +3,11 @@ export * as migrations from "./migrations"
 export * as users from "./users"
 export * as roles from "./security/roles"
 export * as permissions from "./security/permissions"
-export * as accounts from "./cloud/accounts"
+export * as accounts from "./accounts"
 export * as installation from "./installation"
-export * as tenancy from "./tenancy"
 export * as featureFlags from "./featureFlags"
 export * as sessions from "./security/sessions"
-export * as deprovisioning from "./context/deprovision"
+export * as platform from "./platform"
 export * as auth from "./auth"
 export * as constants from "./constants"
 export * as logging from "./logging"
@@ -21,20 +20,27 @@ export * as context from "./context"
 export * as cache from "./cache"
 export * as objectStore from "./objectStore"
 export * as redis from "./redis"
+export * as locks from "./redis/redlock"
 export * as utils from "./utils"
 export * as errors from "./errors"
 export { default as env } from "./environment"
 
+// Add context to tenancy for backwards compatibility
+// only do this for external usages to prevent internal
+// circular dependencies
+import * as context from "./context"
+import * as _tenancy from "./tenancy"
+export const tenancy = {
+  ..._tenancy,
+  ...context,
+}
+
 // expose error classes directly
 export * from "./errors"
 
 // expose constants directly
 export * from "./constants"
 
-// expose inner locks from redis directly
-import * as redis from "./redis"
-export const locks = redis.redlock
-
 // expose package init function
 import * as db from "./db"
 export const init = (opts: any = {}) => {

packages/backend-core/src/middleware/authenticated.ts

@@ -4,7 +4,7 @@ import { getUser } from "../cache/user"
 import { getSession, updateSessionTTL } from "../security/sessions"
 import { buildMatcherRegex, matches } from "./matchers"
 import { SEPARATOR, queryGlobalView, ViewName } from "../db"
-import { getGlobalDB, doInTenant } from "../tenancy"
+import { getGlobalDB, doInTenant } from "../context"
 import { decrypt } from "../security/encryption"
 import * as identity from "../context/identity"
 import env from "../environment"

packages/backend-core/src/middleware/errorHandling.ts

@@ -0,0 +1,28 @@
+import { APIError } from "@budibase/types"
+import * as errors from "../errors"
+import env from "../environment"
+
+export async function errorHandling(ctx: any, next: any) {
+  try {
+    await next()
+  } catch (err: any) {
+    const status = err.status || err.statusCode || 500
+    ctx.status = status
+
+    if (status > 499 || env.ENABLE_4XX_HTTP_LOGGING) {
+      ctx.log.error(err)
+    }
+
+    const error = errors.getPublicError(err)
+    const body: APIError = {
+      message: err.message,
+      status: status,
+      validationErrors: err.validation,
+      error,
+    }
+
+    ctx.body = body
+  }
+}
+
+export default errorHandling

packages/backend-core/src/middleware/index.ts

@@ -1,7 +1,7 @@
 export * as jwt from "./passport/jwt"
 export * as local from "./passport/local"
-export * as google from "./passport/google"
-export * as oidc from "./passport/oidc"
+export * as google from "./passport/sso/google"
+export * as oidc from "./passport/sso/oidc"
 import * as datasourceGoogle from "./passport/datasource/google"
 export const datasource = {
   google: datasourceGoogle,
@@ -16,4 +16,5 @@ export { default as adminOnly } from "./adminOnly"
 export { default as builderOrAdmin } from "./builderOrAdmin"
 export { default as builderOnly } from "./builderOnly"
 export { default as logging } from "./logging"
+export { default as errorHandling } from "./errorHandling"
 export * as joiValidator from "./joi-validator"

packages/backend-core/src/middleware/passport/datasource/google.ts

@@ -1,10 +1,11 @@
-import * as google from "../google"
+import * as google from "../sso/google"
 import { Cookie, Config } from "../../../constants"
 import { clearCookie, getCookie } from "../../../utils"
 import { getScopedConfig, getPlatformUrl, doWithDB } from "../../../db"
 import environment from "../../../environment"
-import { getGlobalDB } from "../../../tenancy"
+import { getGlobalDB } from "../../../context"
 import { BBContext, Database, SSOProfile } from "@budibase/types"
+import { ssoSaveUserNoOp } from "../sso/sso"
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
 
 type Passport = {
@@ -36,7 +37,11 @@ export async function preAuth(
   const platformUrl = await getPlatformUrl({ tenantAware: false })
 
   let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback`
-  const strategy = await google.strategyFactory(googleConfig, callbackUrl)
+  const strategy = await google.strategyFactory(
+    googleConfig,
+    callbackUrl,
+    ssoSaveUserNoOp
+  )
 
   if (!ctx.query.appId || !ctx.query.datasourceId) {
     ctx.throw(400, "appId and datasourceId query params not present.")

packages/backend-core/src/middleware/passport/local.ts

@@ -1,15 +1,10 @@
 import { UserStatus } from "../../constants"
-import { compare, newid } from "../../utils"
-import env from "../../environment"
+import { compare } from "../../utils"
 import * as users from "../../users"
 import { authError } from "./utils"
-import { createASession } from "../../security/sessions"
-import { getTenantId } from "../../tenancy"
 import { BBContext } from "@budibase/types"
-const jwt = require("jsonwebtoken")
 
 const INVALID_ERR = "Invalid credentials"
-const SSO_NO_PASSWORD = "SSO user does not have a password set"
 const EXPIRED = "This account has expired. Please reset your password"
 
 export const options = {
@@ -35,50 +30,25 @@ export async function authenticate(
 
   const dbUser = await users.getGlobalUserByEmail(email)
   if (dbUser == null) {
-    return authError(done, `User not found: [${email}]`)
-  }
-
-  // check that the user is currently inactive, if this is the case throw invalid
-  if (dbUser.status === UserStatus.INACTIVE) {
+    console.info(`user=${email} could not be found`)
     return authError(done, INVALID_ERR)
   }
 
-  // check that the user has a stored password before proceeding
-  if (!dbUser.password) {
-    if (
-      (dbUser.account && dbUser.account.authType === "sso") || // root account sso
-      dbUser.thirdPartyProfile // internal sso
-    ) {
-      return authError(done, SSO_NO_PASSWORD)
-    }
+  if (dbUser.status === UserStatus.INACTIVE) {
+    console.info(`user=${email} is inactive`, dbUser)
+    return authError(done, INVALID_ERR)
+  }
 
-    console.error("Non SSO usser has no password set", dbUser)
+  if (!dbUser.password) {
+    console.info(`user=${email} has no password set`, dbUser)
     return authError(done, EXPIRED)
   }
 
-  // authenticate
-  if (await compare(password, dbUser.password)) {
-    const sessionId = newid()
-    const tenantId = getTenantId()
-
-    await createASession(dbUser._id!, { sessionId, tenantId })
-
-    const token = jwt.sign(
-      {
-        userId: dbUser._id,
-        sessionId,
-        tenantId,
-      },
-      env.JWT_SECRET
-    )
-    // Remove users password in payload
-    delete dbUser.password
-
-    return done(null, {
-      ...dbUser,
-      token,
-    })
-  } else {
+  if (!(await compare(password, dbUser.password))) {
     return authError(done, INVALID_ERR)
   }
+
+  // intentionally remove the users password in payload
+  delete dbUser.password
+  return done(null, dbUser)
 }

packages/backend-core/src/middleware/passport/sso/google.ts

@@ -1,18 +1,26 @@
-import { ssoCallbackUrl } from "./utils"
-import { authenticateThirdParty, SaveUserFunction } from "./third-party-common"
-import { ConfigType, GoogleConfig, Database, SSOProfile } from "@budibase/types"
+import { ssoCallbackUrl } from "../utils"
+import * as sso from "./sso"
+import {
+  ConfigType,
+  GoogleConfig,
+  Database,
+  SSOProfile,
+  SSOAuthDetails,
+  SSOProviderType,
+  SaveSSOUserFunction,
+} from "@budibase/types"
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
 
-export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
+export function buildVerifyFn(saveUserFn: SaveSSOUserFunction) {
   return (
     accessToken: string,
     refreshToken: string,
     profile: SSOProfile,
     done: Function
   ) => {
-    const thirdPartyUser = {
-      provider: profile.provider, // should always be 'google'
-      providerType: "google",
+    const details: SSOAuthDetails = {
+      provider: "google",
+      providerType: SSOProviderType.GOOGLE,
       userId: profile.id,
       profile: profile,
       email: profile._json.email,
@@ -22,8 +30,8 @@ export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
       },
     }
 
-    return authenticateThirdParty(
-      thirdPartyUser,
+    return sso.authenticate(
+      details,
       true, // require local accounts to exist
       done,
       saveUserFn
@@ -39,7 +47,7 @@ export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
 export async function strategyFactory(
   config: GoogleConfig["config"],
   callbackUrl: string,
-  saveUserFn?: SaveUserFunction
+  saveUserFn: SaveSSOUserFunction
 ) {
   try {
     const { clientID, clientSecret } = config

packages/backend-core/src/middleware/passport/sso/oidc.ts

@@ -1,22 +1,20 @@
 import fetch from "node-fetch"
-import { authenticateThirdParty, SaveUserFunction } from "./third-party-common"
-import { ssoCallbackUrl } from "./utils"
+import * as sso from "./sso"
+import { ssoCallbackUrl } from "../utils"
 import {
   ConfigType,
-  OIDCInnerCfg,
+  OIDCInnerConfig,
   Database,
   SSOProfile,
-  ThirdPartyUser,
-  OIDCConfiguration,
+  OIDCStrategyConfiguration,
+  SSOAuthDetails,
+  SSOProviderType,
+  JwtClaims,
+  SaveSSOUserFunction,
 } from "@budibase/types"
 const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy
 
-type JwtClaims = {
-  preferred_username: string
-  email: string
-}
-
-export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
+export function buildVerifyFn(saveUserFn: SaveSSOUserFunction) {
   /**
    * @param {*} issuer The identity provider base URL
    * @param {*} sub The user ID
@@ -39,10 +37,10 @@ export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
     params: any,
     done: Function
   ) => {
-    const thirdPartyUser: ThirdPartyUser = {
+    const details: SSOAuthDetails = {
       // store the issuer info to enable sync in future
       provider: issuer,
-      providerType: "oidc",
+      providerType: SSOProviderType.OIDC,
       userId: profile.id,
       profile: profile,
       email: getEmail(profile, jwtClaims),
@@ -52,8 +50,8 @@ export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
       },
     }
 
-    return authenticateThirdParty(
-      thirdPartyUser,
+    return sso.authenticate(
+      details,
       false, // don't require local accounts to exist
       done,
       saveUserFn
@@ -104,8 +102,8 @@ function validEmail(value: string) {
  * @returns Dynamically configured Passport OIDC Strategy
  */
 export async function strategyFactory(
-  config: OIDCConfiguration,
-  saveUserFn?: SaveUserFunction
+  config: OIDCStrategyConfiguration,
+  saveUserFn: SaveSSOUserFunction
 ) {
   try {
     const verify = buildVerifyFn(saveUserFn)
@@ -119,14 +117,14 @@ export async function strategyFactory(
 }
 
 export async function fetchStrategyConfig(
-  enrichedConfig: OIDCInnerCfg,
+  oidcConfig: OIDCInnerConfig,
   callbackUrl?: string
-): Promise<OIDCConfiguration> {
+): Promise<OIDCStrategyConfiguration> {
   try {
-    const { clientID, clientSecret, configUrl } = enrichedConfig
+    const { clientID, clientSecret, configUrl } = oidcConfig
 
     if (!clientID || !clientSecret || !callbackUrl || !configUrl) {
-      //check for remote config and all required elements
+      // check for remote config and all required elements
       throw new Error(
         "Configuration invalid. Must contain clientID, clientSecret, callbackUrl and configUrl"
       )

packages/backend-core/src/middleware/passport/sso/sso.ts

@@ -0,0 +1,165 @@
+import { generateGlobalUserID } from "../../../db"
+import { authError } from "../utils"
+import * as users from "../../../users"
+import * as context from "../../../context"
+import fetch from "node-fetch"
+import {
+  SaveSSOUserFunction,
+  SaveUserOpts,
+  SSOAuthDetails,
+  SSOUser,
+  User,
+} from "@budibase/types"
+
+// no-op function for user save
+// - this allows datasource auth and access token refresh to work correctly
+// - prefer no-op over an optional argument to ensure function is provided to login flows
+export const ssoSaveUserNoOp: SaveSSOUserFunction = (
+  user: SSOUser,
+  opts: SaveUserOpts
+) => Promise.resolve(user)
+
+/**
+ * Common authentication logic for third parties. e.g. OAuth, OIDC.
+ */
+export async function authenticate(
+  details: SSOAuthDetails,
+  requireLocalAccount: boolean = true,
+  done: any,
+  saveUserFn: SaveSSOUserFunction
+) {
+  if (!saveUserFn) {
+    throw new Error("Save user function must be provided")
+  }
+  if (!details.userId) {
+    return authError(done, "sso user id required")
+  }
+  if (!details.email) {
+    return authError(done, "sso user email required")
+  }
+
+  // use the third party id
+  const userId = generateGlobalUserID(details.userId)
+
+  let dbUser: User | undefined
+
+  // try to load by id
+  try {
+    dbUser = await users.getById(userId)
+  } catch (err: any) {
+    // abort when not 404 error
+    if (!err.status || err.status !== 404) {
+      return authError(
+        done,
+        "Unexpected error when retrieving existing user",
+        err
+      )
+    }
+  }
+
+  // fallback to loading by email
+  if (!dbUser) {
+    dbUser = await users.getGlobalUserByEmail(details.email)
+  }
+
+  // exit early if there is still no user and auto creation is disabled
+  if (!dbUser && requireLocalAccount) {
+    return authError(
+      done,
+      "Email does not yet exist. You must set up your local budibase account first."
+    )
+  }
+
+  // first time creation
+  if (!dbUser) {
+    // setup a blank user using the third party id
+    dbUser = {
+      _id: userId,
+      email: details.email,
+      roles: {},
+      tenantId: context.getTenantId(),
+    }
+  }
+
+  let ssoUser = await syncUser(dbUser, details)
+  // never prompt for password reset
+  ssoUser.forceResetPassword = false
+
+  try {
+    // don't try to re-save any existing password
+    delete ssoUser.password
+    // create or sync the user
+    ssoUser = (await saveUserFn(ssoUser, {
+      hashPassword: false,
+      requirePassword: false,
+    })) as SSOUser
+  } catch (err: any) {
+    return authError(done, "Error saving user", err)
+  }
+
+  return done(null, ssoUser)
+}
+
+async function getProfilePictureUrl(user: User, details: SSOAuthDetails) {
+  const pictureUrl = details.profile?._json.picture
+  if (pictureUrl) {
+    const response = await fetch(pictureUrl)
+    if (response.status === 200) {
+      const type = response.headers.get("content-type") as string
+      if (type.startsWith("image/")) {
+        return pictureUrl
+      }
+    }
+  }
+}
+
+/**
+ * @returns a user that has been sync'd with third party information
+ */
+async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
+  let firstName
+  let lastName
+  let pictureUrl
+  let oauth2
+  let thirdPartyProfile
+
+  if (details.profile) {
+    const profile = details.profile
+
+    if (profile.name) {
+      const name = profile.name
+      // first name
+      if (name.givenName) {
+        firstName = name.givenName
+      }
+      // last name
+      if (name.familyName) {
+        lastName = name.familyName
+      }
+    }
+
+    pictureUrl = await getProfilePictureUrl(user, details)
+
+    thirdPartyProfile = {
+      ...profile._json,
+    }
+  }
+
+  // oauth tokens for future use
+  if (details.oauth2) {
+    oauth2 = {
+      ...details.oauth2,
+    }
+  }
+
+  return {
+    ...user,
+    provider: details.provider,
+    providerType: details.providerType,
+    firstName,
+    lastName,
+    thirdPartyProfile,
+    pictureUrl,
+    oauth2,
+  }
+}

packages/backend-core/src/middleware/passport/sso/tests/google.spec.ts

@@ -0,0 +1,67 @@
+import { generator, structures } from "../../../../../tests"
+import { SSOProviderType } from "@budibase/types"
+
+jest.mock("passport-google-oauth")
+const mockStrategy = require("passport-google-oauth").OAuth2Strategy
+
+jest.mock("../sso")
+import * as _sso from "../sso"
+const sso = jest.mocked(_sso)
+
+const mockSaveUserFn = jest.fn()
+const mockDone = jest.fn()
+
+import * as google from "../google"
+
+describe("google", () => {
+  describe("strategyFactory", () => {
+    const googleConfig = structures.sso.googleConfig()
+    const callbackUrl = generator.url()
+
+    it("should create successfully create a google strategy", async () => {
+      await google.strategyFactory(googleConfig, callbackUrl, mockSaveUserFn)
+
+      const expectedOptions = {
+        clientID: googleConfig.clientID,
+        clientSecret: googleConfig.clientSecret,
+        callbackURL: callbackUrl,
+      }
+
+      expect(mockStrategy).toHaveBeenCalledWith(
+        expectedOptions,
+        expect.anything()
+      )
+    })
+  })
+
+  describe("authenticate", () => {
+    const details = structures.sso.authDetails()
+    details.provider = "google"
+    details.providerType = SSOProviderType.GOOGLE
+
+    const profile = details.profile!
+    profile.provider = "google"
+
+    beforeEach(() => {
+      jest.clearAllMocks()
+    })
+
+    it("delegates authentication to third party common", async () => {
+      const authenticate = await google.buildVerifyFn(mockSaveUserFn)
+
+      await authenticate(
+        details.oauth2.accessToken,
+        details.oauth2.refreshToken!,
+        profile,
+        mockDone
+      )
+
+      expect(sso.authenticate).toHaveBeenCalledWith(
+        details,
+        true,
+        mockDone,
+        mockSaveUserFn
+      )
+    })
+  })
+})

packages/backend-core/src/middleware/passport/sso/tests/oidc.spec.ts

@@ -0,0 +1,152 @@
+import { generator, mocks, structures } from "../../../../../tests"
+import {
+  JwtClaims,
+  OIDCInnerConfig,
+  SSOAuthDetails,
+  SSOProviderType,
+} from "@budibase/types"
+import * as _sso from "../sso"
+import * as oidc from "../oidc"
+
+jest.mock("@techpass/passport-openidconnect")
+const mockStrategy = require("@techpass/passport-openidconnect").Strategy
+
+jest.mock("../sso")
+const sso = jest.mocked(_sso)
+
+const mockSaveUser = jest.fn()
+const mockDone = jest.fn()
+
+describe("oidc", () => {
+  const callbackUrl = generator.url()
+  const oidcConfig: OIDCInnerConfig = structures.sso.oidcConfig()
+  const wellKnownConfig = structures.sso.oidcWellKnownConfig()
+
+  function mockRetrieveWellKnownConfig() {
+    // mock the request to retrieve the oidc configuration
+    mocks.fetch.mockReturnValue({
+      ok: true,
+      json: () => wellKnownConfig,
+    })
+  }
+
+  beforeEach(() => {
+    mockRetrieveWellKnownConfig()
+  })
+
+  describe("strategyFactory", () => {
+    it("should create successfully create an oidc strategy", async () => {
+      const strategyConfiguration = await oidc.fetchStrategyConfig(
+        oidcConfig,
+        callbackUrl
+      )
+      await oidc.strategyFactory(strategyConfiguration, mockSaveUser)
+
+      expect(mocks.fetch).toHaveBeenCalledWith(oidcConfig.configUrl)
+
+      const expectedOptions = {
+        issuer: wellKnownConfig.issuer,
+        authorizationURL: wellKnownConfig.authorization_endpoint,
+        tokenURL: wellKnownConfig.token_endpoint,
+        userInfoURL: wellKnownConfig.userinfo_endpoint,
+        clientID: oidcConfig.clientID,
+        clientSecret: oidcConfig.clientSecret,
+        callbackURL: callbackUrl,
+      }
+      expect(mockStrategy).toHaveBeenCalledWith(
+        expectedOptions,
+        expect.anything()
+      )
+    })
+  })
+
+  describe("authenticate", () => {
+    const details: SSOAuthDetails = structures.sso.authDetails()
+    details.providerType = SSOProviderType.OIDC
+    const profile = details.profile!
+    const issuer = profile.provider
+
+    const sub = generator.string()
+    const idToken = generator.string()
+    const params = {}
+
+    let authenticateFn: any
+    let jwtClaims: JwtClaims
+
+    beforeEach(async () => {
+      jest.clearAllMocks()
+      authenticateFn = await oidc.buildVerifyFn(mockSaveUser)
+    })
+
+    async function authenticate() {
+      await authenticateFn(
+        issuer,
+        sub,
+        profile,
+        jwtClaims,
+        details.oauth2.accessToken,
+        details.oauth2.refreshToken,
+        idToken,
+        params,
+        mockDone
+      )
+    }
+
+    it("passes auth details to sso module", async () => {
+      await authenticate()
+
+      expect(sso.authenticate).toHaveBeenCalledWith(
+        details,
+        false,
+        mockDone,
+        mockSaveUser
+      )
+    })
+
+    it("uses JWT email to get email", async () => {
+      delete profile._json.email
+
+      jwtClaims = {
+        email: details.email,
+      }
+
+      await authenticate()
+
+      expect(sso.authenticate).toHaveBeenCalledWith(
+        details,
+        false,
+        mockDone,
+        mockSaveUser
+      )
+    })
+
+    it("uses JWT username to get email", async () => {
+      delete profile._json.email
+
+      jwtClaims = {
+        email: details.email,
+      }
+
+      await authenticate()
+
+      expect(sso.authenticate).toHaveBeenCalledWith(
+        details,
+        false,
+        mockDone,
+        mockSaveUser
+      )
+    })
+
+    it("uses JWT invalid username to get email", async () => {
+      delete profile._json.email
+
+      jwtClaims = {
+        preferred_username: "invalidUsername",
+      }
+
+      await expect(authenticate()).rejects.toThrow(
+        "Could not determine user email from profile"
+      )
+    })
+  })
+})

packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts

@@ -0,0 +1,196 @@
+import { structures, testEnv, mocks } from "../../../../../tests"
+import { SSOAuthDetails, User } from "@budibase/types"
+
+import { HTTPError } from "../../../../errors"
+import * as sso from "../sso"
+import * as context from "../../../../context"
+
+const mockDone = jest.fn()
+const mockSaveUser = jest.fn()
+
+jest.mock("../../../../users")
+import * as _users from "../../../../users"
+const users = jest.mocked(_users)
+
+const getErrorMessage = () => {
+  return mockDone.mock.calls[0][2].message
+}
+
+describe("sso", () => {
+  describe("authenticate", () => {
+    beforeEach(() => {
+      jest.clearAllMocks()
+      testEnv.singleTenant()
+    })
+
+    describe("validation", () => {
+      const testValidation = async (
+        details: SSOAuthDetails,
+        message: string
+      ) => {
+        await sso.authenticate(details, false, mockDone, mockSaveUser)
+
+        expect(mockDone.mock.calls.length).toBe(1)
+        expect(getErrorMessage()).toContain(message)
+      }
+
+      it("user id fails", async () => {
+        const details = structures.sso.authDetails()
+        details.userId = undefined!
+
+        await testValidation(details, "sso user id required")
+      })
+
+      it("email fails", async () => {
+        const details = structures.sso.authDetails()
+        details.email = undefined!
+
+        await testValidation(details, "sso user email required")
+      })
+    })
+
+    function mockGetProfilePicture() {
+      mocks.fetch.mockReturnValueOnce(
+        Promise.resolve({
+          status: 200,
+          headers: { get: () => "image/" },
+        })
+      )
+    }
+
+    describe("when the user doesn't exist", () => {
+      let user: User
+      let details: SSOAuthDetails
+
+      beforeEach(() => {
+        users.getById.mockImplementationOnce(() => {
+          throw new HTTPError("", 404)
+        })
+        mockGetProfilePicture()
+
+        user = structures.users.user()
+        delete user._rev
+        delete user._id
+
+        details = structures.sso.authDetails(user)
+        details.userId = structures.uuid()
+      })
+
+      describe("when a local account is required", () => {
+        it("returns an error message", async () => {
+          const details = structures.sso.authDetails()
+
+          await sso.authenticate(details, true, mockDone, mockSaveUser)
+
+          expect(mockDone.mock.calls.length).toBe(1)
+          expect(getErrorMessage()).toContain(
+            "Email does not yet exist. You must set up your local budibase account first."
+          )
+        })
+      })
+
+      describe("when a local account isn't required", () => {
+        it("creates and authenticates the user", async () => {
+          const ssoUser = structures.users.ssoUser({ user, details })
+          mockSaveUser.mockReturnValueOnce(ssoUser)
+
+          await sso.authenticate(details, false, mockDone, mockSaveUser)
+
+          // default roles for new user
+          ssoUser.roles = {}
+
+          // modified external id to match user format
+          ssoUser._id = "us_" + details.userId
+
+          // new sso user won't have a password
+          delete ssoUser.password
+
+          // new user isn't saved with rev
+          delete ssoUser._rev
+
+          // tenant id added
+          ssoUser.tenantId = context.getTenantId()
+
+          expect(mockSaveUser).toBeCalledWith(ssoUser, {
+            hashPassword: false,
+            requirePassword: false,
+          })
+          expect(mockDone).toBeCalledWith(null, ssoUser)
+        })
+      })
+    })
+
+    describe("when the user exists", () => {
+      let existingUser: User
+      let details: SSOAuthDetails
+
+      beforeEach(() => {
+        existingUser = structures.users.user()
+        existingUser._id = structures.uuid()
+        details = structures.sso.authDetails(existingUser)
+        mockGetProfilePicture()
+      })
+
+      describe("exists by email", () => {
+        beforeEach(() => {
+          users.getById.mockImplementationOnce(() => {
+            throw new HTTPError("", 404)
+          })
+          users.getGlobalUserByEmail.mockReturnValueOnce(
+            Promise.resolve(existingUser)
+          )
+        })
+
+        it("syncs and authenticates the user", async () => {
+          const ssoUser = structures.users.ssoUser({
+            user: existingUser,
+            details,
+          })
+          mockSaveUser.mockReturnValueOnce(ssoUser)
+
+          await sso.authenticate(details, true, mockDone, mockSaveUser)
+
+          // roles preserved
+          ssoUser.roles = existingUser.roles
+
+          // existing id preserved
+          ssoUser._id = existingUser._id
+
+          expect(mockSaveUser).toBeCalledWith(ssoUser, {
+            hashPassword: false,
+            requirePassword: false,
+          })
+          expect(mockDone).toBeCalledWith(null, ssoUser)
+        })
+      })
+
+      describe("exists by id", () => {
+        beforeEach(() => {
+          users.getById.mockReturnValueOnce(Promise.resolve(existingUser))
+        })
+
+        it("syncs and authenticates the user", async () => {
+          const ssoUser = structures.users.ssoUser({
+            user: existingUser,
+            details,
+          })
+          mockSaveUser.mockReturnValueOnce(ssoUser)
+
+          await sso.authenticate(details, true, mockDone, mockSaveUser)
+
+          // roles preserved
+          ssoUser.roles = existingUser.roles
+
+          // existing id preserved
+          ssoUser._id = existingUser._id
+
+          expect(mockSaveUser).toBeCalledWith(ssoUser, {
+            hashPassword: false,
+            requirePassword: false,
+          })
+          expect(mockDone).toBeCalledWith(null, ssoUser)
+        })
+      })
+    })
+  })
+})

packages/backend-core/src/middleware/passport/tests/google.spec.js

@@ -1,79 +0,0 @@
-// Mock data
-
-const { data } = require("./utilities/mock-data")
-
-const TENANT_ID = "default"
-
-const googleConfig = {
-  clientID: data.clientID,
-  clientSecret: data.clientSecret,
-}
-
-const profile = {
-  id: "mockId",
-  _json: {
-    email : data.email
-  },
-  provider: "google"
-}
-
-const user = data.buildThirdPartyUser("google", "google", profile)
-
-describe("google", () => {
-  describe("strategyFactory", () => {  
-    // mock passport strategy factory
-    jest.mock("passport-google-oauth")
-    const mockStrategy = require("passport-google-oauth").OAuth2Strategy
-  
-    it("should create successfully create a google strategy", async () => {
-      const google = require("../google")
-
-      const callbackUrl = `/api/global/auth/${TENANT_ID}/google/callback`
-      await google.strategyFactory(googleConfig, callbackUrl)
-  
-      const expectedOptions = {
-        clientID: googleConfig.clientID,
-        clientSecret: googleConfig.clientSecret,
-        callbackURL: callbackUrl,
-      }
-
-      expect(mockStrategy).toHaveBeenCalledWith(
-        expectedOptions,
-        expect.anything()
-      )
-    })
-  })
-  
-  describe("authenticate", () => {    
-    afterEach(() => {
-      jest.clearAllMocks();
-    });
-
-    // mock third party common authentication
-    jest.mock("../third-party-common")
-    const authenticateThirdParty = require("../third-party-common").authenticateThirdParty
-
-    // mock the passport callback
-    const mockDone = jest.fn()
-
-    it("delegates authentication to third party common", async () => {
-      const google = require("../google")
-      const mockSaveUserFn = jest.fn()
-      const authenticate = await google.buildVerifyFn(mockSaveUserFn)
-
-      await authenticate(
-        data.accessToken,
-        data.refreshToken,
-        profile,
-        mockDone
-      )
-
-      expect(authenticateThirdParty).toHaveBeenCalledWith(
-        user,
-        true, 
-        mockDone,
-        mockSaveUserFn)
-    })
-  })
-})
-

packages/backend-core/src/middleware/passport/tests/oidc.spec.js

@@ -1,144 +0,0 @@
-// Mock data
-const mockFetch = require("node-fetch")
-const { data } = require("./utilities/mock-data")
-const issuer = "mockIssuer"
-const sub = "mockSub"
-const profile = {
-  id: "mockId",
-  _json: {
-    email : data.email
-  }
-}
-let jwtClaims = {}  
-const idToken = "mockIdToken"
-const params = {}
-
-const callbackUrl = "http://somecallbackurl"
-
-// response from .well-known/openid-configuration
-const oidcConfigUrlResponse = {
-  issuer: issuer,
-  authorization_endpoint: "mockAuthorizationEndpoint",
-  token_endpoint: "mockTokenEndpoint",
-  userinfo_endpoint: "mockUserInfoEndpoint"
-}
-
-const oidcConfig = {
-  configUrl: "http://someconfigurl",
-  clientID: data.clientID,
-  clientSecret: data.clientSecret,
-}
-
-const user = data.buildThirdPartyUser(issuer, "oidc", profile)
-
-describe("oidc", () => {
-  describe("strategyFactory", () => {  
-    // mock passport strategy factory
-    jest.mock("@techpass/passport-openidconnect")
-    const mockStrategy = require("@techpass/passport-openidconnect").Strategy
-    
-    // mock the request to retrieve the oidc configuration
-    mockFetch.mockReturnValue({
-      ok: true,
-      json: () => oidcConfigUrlResponse
-    })
-  
-    it("should create successfully create an oidc strategy", async () => {
-      const oidc = require("../oidc")
-      const enrichedConfig = await oidc.fetchStrategyConfig(oidcConfig, callbackUrl)
-      await oidc.strategyFactory(enrichedConfig, callbackUrl)
-  
-      expect(mockFetch).toHaveBeenCalledWith(oidcConfig.configUrl)
-
-      const expectedOptions = {
-        issuer: oidcConfigUrlResponse.issuer,
-        authorizationURL: oidcConfigUrlResponse.authorization_endpoint,
-        tokenURL: oidcConfigUrlResponse.token_endpoint,
-        userInfoURL: oidcConfigUrlResponse.userinfo_endpoint,
-        clientID: oidcConfig.clientID,
-        clientSecret: oidcConfig.clientSecret,
-        callbackURL: callbackUrl,
-      }
-      expect(mockStrategy).toHaveBeenCalledWith(
-        expectedOptions,
-        expect.anything()
-      )
-    })
-  })
-  
-  describe("authenticate", () => {    
-    afterEach(() => {
-      jest.clearAllMocks()
-    });
-
-    // mock third party common authentication
-    jest.mock("../third-party-common")
-    const authenticateThirdParty = require("../third-party-common").authenticateThirdParty
-    
-    // mock the passport callback
-    const mockDone = jest.fn()
-    const mockSaveUserFn = jest.fn()
-
-    async function doAuthenticate() {
-      const oidc = require("../oidc")
-      const authenticate = await oidc.buildVerifyFn(mockSaveUserFn)
-
-      await authenticate(
-        issuer,
-        sub,
-        profile,
-        jwtClaims,
-        data.accessToken,
-        data.refreshToken,
-        idToken,
-        params,
-        mockDone
-      )
-    }
-
-    async function doTest() {
-      await doAuthenticate()
-
-      expect(authenticateThirdParty).toHaveBeenCalledWith(
-        user,
-        false, 
-        mockDone,
-        mockSaveUserFn,
-      )
-    }
-
-    it("delegates authentication to third party common", async () => {
-      await doTest()
-    })
-
-    it("uses JWT email to get email", async () => {
-      delete profile._json.email
-      jwtClaims = {
-        email : "mock@budibase.com"
-      }
-
-      await doTest()
-    })
-  
-    it("uses JWT username to get email", async () => {
-      delete profile._json.email
-      jwtClaims = {
-        preferred_username : "mock@budibase.com"
-      }
-
-      await doTest()
-    })
-
-    it("uses JWT invalid username to get email", async () => {
-      delete profile._json.email
-
-      jwtClaims = {
-        preferred_username : "invalidUsername"
-      }
-
-      await expect(doAuthenticate()).rejects.toThrow("Could not determine user email from profile");
-    })
-  
-  })
-})
-

packages/backend-core/src/middleware/passport/tests/third-party-common.seq.spec.js

@@ -1,178 +0,0 @@
-require("../../../../tests")
-const { authenticateThirdParty } = require("../third-party-common")
-const { data } = require("./utilities/mock-data")
-const { DEFAULT_TENANT_ID } = require("../../../constants")
-
-const { generateGlobalUserID } = require("../../../db/utils")
-const { newid } = require("../../../utils")
-const { doWithGlobalDB, doInTenant } = require("../../../tenancy")
-
-const done = jest.fn()
-
-const getErrorMessage = () => {
-  return done.mock.calls[0][2].message
-}
-
-const saveUser = async (user) => {
-  return doWithGlobalDB(DEFAULT_TENANT_ID, async db => {
-    return await db.put(user)
-  })
-}
-
-function authenticate(user, requireLocal, saveFn) {
-  return doInTenant(DEFAULT_TENANT_ID, () => {
-    return authenticateThirdParty(user, requireLocal, done, saveFn)
-  })
-}
-
-describe("third party common", () => {
-  describe("authenticateThirdParty", () => {
-    let thirdPartyUser
-
-    beforeEach(() => {
-      thirdPartyUser = data.buildThirdPartyUser()
-    })
-
-    afterEach(async () => {
-      return doWithGlobalDB(DEFAULT_TENANT_ID, async db => {
-        jest.clearAllMocks()
-        await db.destroy()
-      })
-    })
-
-    describe("validation", () => {
-      const testValidation = async (message) => {
-        await authenticate(thirdPartyUser, false, saveUser)
-        expect(done.mock.calls.length).toBe(1)
-        expect(getErrorMessage()).toContain(message)
-      }
-
-      it("provider fails", async () => {
-        delete thirdPartyUser.provider
-        await testValidation("third party user provider required")
-      })
-
-      it("user id fails", async () => {
-        delete thirdPartyUser.userId
-        await testValidation("third party user id required")
-      })
-
-      it("email fails", async () => {
-        delete thirdPartyUser.email
-        await testValidation("third party user email required")
-      })
-    })
-
-    const expectUserIsAuthenticated = () => {
-      const user = done.mock.calls[0][1]
-      expect(user).toBeDefined()
-      expect(user._id).toBeDefined()
-      expect(user._rev).toBeDefined()
-      expect(user.token).toBeDefined()
-      return user
-    }
-
-    const expectUserIsSynced = (user, thirdPartyUser) => {
-      expect(user.provider).toBe(thirdPartyUser.provider)
-      expect(user.firstName).toBe(thirdPartyUser.profile.name.givenName)
-      expect(user.lastName).toBe(thirdPartyUser.profile.name.familyName)
-      expect(user.thirdPartyProfile).toStrictEqual(thirdPartyUser.profile._json)
-      expect(user.oauth2).toStrictEqual(thirdPartyUser.oauth2)
-    }
-
-    describe("when the user doesn't exist", () => {
-      describe("when a local account is required", () => {
-        it("returns an error message", async () => {
-          await authenticate(thirdPartyUser, true, saveUser)
-          expect(done.mock.calls.length).toBe(1)
-          expect(getErrorMessage()).toContain("Email does not yet exist. You must set up your local budibase account first.")
-        })
-      })
-
-      describe("when a local account isn't required", () => {
-        it("creates and authenticates the user", async () => {
-          await authenticate(thirdPartyUser, false, saveUser)
-          const user = expectUserIsAuthenticated()
-          expectUserIsSynced(user, thirdPartyUser)
-          expect(user.roles).toStrictEqual({})
-        })
-      })
-    })
-
-    describe("when the user exists", () => {
-      let dbUser
-      let id
-      let email
-
-      const createUser = async () => {
-        return doWithGlobalDB(DEFAULT_TENANT_ID, async db => {
-          dbUser = {
-            _id: id,
-            email: email,
-          }
-          const response = await db.put(dbUser)
-          dbUser._rev = response.rev
-          return dbUser
-        })
-      }
-
-      const expectUserIsUpdated = (user) => {
-        // id is unchanged
-        expect(user._id).toBe(id)
-        // user is updated
-        expect(user._rev).not.toBe(dbUser._rev)
-      }
-
-      describe("exists by email", () => {
-        beforeEach(async () => {
-          id = generateGlobalUserID(newid()) // random id
-          email = thirdPartyUser.email //  matching email
-          await createUser()
-        })
-
-        it("syncs and authenticates the user", async () => {
-          await authenticate(thirdPartyUser, true, saveUser)
-
-          const user = expectUserIsAuthenticated()
-          expectUserIsSynced(user, thirdPartyUser)
-          expectUserIsUpdated(user)
-        })
-      })
-
-      describe("exists by email with different casing", () => {
-        beforeEach(async () => {
-          id = generateGlobalUserID(newid()) // random id
-          email = thirdPartyUser.email.toUpperCase() //  matching email except for casing
-          await createUser()
-        })
-
-        it("syncs and authenticates the user", async () => {
-          await authenticate(thirdPartyUser, true, saveUser)
-
-          const user = expectUserIsAuthenticated()
-          expectUserIsSynced(user, thirdPartyUser)
-          expectUserIsUpdated(user)
-          expect(user.email).toBe(thirdPartyUser.email.toUpperCase())
-        })
-      })
-
-
-      describe("exists by id", () => {
-        beforeEach(async () => {
-          id = generateGlobalUserID(thirdPartyUser.userId) // matching id
-          email = "test@test.com" // random email
-          await createUser()
-        })
-
-        it("syncs and authenticates the user", async () => {
-          await authenticate(thirdPartyUser, true, saveUser)
-
-          const user = expectUserIsAuthenticated()
-          expectUserIsSynced(user, thirdPartyUser)
-          expectUserIsUpdated(user)
-        })
-      })
-    })
-  })
-})
-

packages/backend-core/src/middleware/passport/tests/utilities/mock-data.js

@@ -1,54 +0,0 @@
-// Mock Data
-
-const mockClientID = "mockClientID"
-const mockClientSecret = "mockClientSecret"
-
-const mockEmail = "mock@budibase.com"
-const mockAccessToken = "mockAccessToken"
-const mockRefreshToken = "mockRefreshToken"
-
-const mockProvider = "mockProvider"
-const mockProviderType = "mockProviderType"
-
-const mockProfile = {
-  id: "mockId",
-  name: {
-    givenName: "mockGivenName",
-    familyName: "mockFamilyName",
-  },
-  _json: {
-    email: mockEmail,
-  },
-}
-
-const buildOauth2 = (
-  accessToken = mockAccessToken,
-  refreshToken = mockRefreshToken
-) => ({
-  accessToken: accessToken,
-  refreshToken: refreshToken,
-})
-
-const buildThirdPartyUser = (
-  provider = mockProvider,
-  providerType = mockProviderType,
-  profile = mockProfile,
-  email = mockEmail,
-  oauth2 = buildOauth2()
-) => ({
-  provider: provider,
-  providerType: providerType,
-  userId: profile.id,
-  profile: profile,
-  email: email,
-  oauth2: oauth2,
-})
-
-exports.data = {
-  clientID: mockClientID,
-  clientSecret: mockClientSecret,
-  email: mockEmail,
-  accessToken: mockAccessToken,
-  refreshToken: mockRefreshToken,
-  buildThirdPartyUser,
-}

packages/backend-core/src/middleware/passport/third-party-common.ts

@@ -1,177 +0,0 @@
-import env from "../../environment"
-import { generateGlobalUserID } from "../../db"
-import { authError } from "./utils"
-import { newid } from "../../utils"
-import { createASession } from "../../security/sessions"
-import * as users from "../../users"
-import { getGlobalDB, getTenantId } from "../../tenancy"
-import fetch from "node-fetch"
-import { ThirdPartyUser } from "@budibase/types"
-const jwt = require("jsonwebtoken")
-
-type SaveUserOpts = {
-  requirePassword?: boolean
-  hashPassword?: boolean
-  currentUserId?: string
-}
-
-export type SaveUserFunction = (
-  user: ThirdPartyUser,
-  opts: SaveUserOpts
-) => Promise<any>
-
-/**
- * Common authentication logic for third parties. e.g. OAuth, OIDC.
- */
-export async function authenticateThirdParty(
-  thirdPartyUser: ThirdPartyUser,
-  requireLocalAccount: boolean = true,
-  done: Function,
-  saveUserFn?: SaveUserFunction
-) {
-  if (!saveUserFn) {
-    throw new Error("Save user function must be provided")
-  }
-  if (!thirdPartyUser.provider) {
-    return authError(done, "third party user provider required")
-  }
-  if (!thirdPartyUser.userId) {
-    return authError(done, "third party user id required")
-  }
-  if (!thirdPartyUser.email) {
-    return authError(done, "third party user email required")
-  }
-
-  // use the third party id
-  const userId = generateGlobalUserID(thirdPartyUser.userId)
-  const db = getGlobalDB()
-
-  let dbUser
-
-  // try to load by id
-  try {
-    dbUser = await db.get(userId)
-  } catch (err: any) {
-    // abort when not 404 error
-    if (!err.status || err.status !== 404) {
-      return authError(
-        done,
-        "Unexpected error when retrieving existing user",
-        err
-      )
-    }
-  }
-
-  // fallback to loading by email
-  if (!dbUser) {
-    dbUser = await users.getGlobalUserByEmail(thirdPartyUser.email)
-  }
-
-  // exit early if there is still no user and auto creation is disabled
-  if (!dbUser && requireLocalAccount) {
-    return authError(
-      done,
-      "Email does not yet exist. You must set up your local budibase account first."
-    )
-  }
-
-  // first time creation
-  if (!dbUser) {
-    // setup a blank user using the third party id
-    dbUser = {
-      _id: userId,
-      email: thirdPartyUser.email,
-      roles: {},
-    }
-  }
-
-  dbUser = await syncUser(dbUser, thirdPartyUser)
-
-  // never prompt for password reset
-  dbUser.forceResetPassword = false
-
-  // create or sync the user
-  try {
-    await saveUserFn(dbUser, { hashPassword: false, requirePassword: false })
-  } catch (err: any) {
-    return authError(done, "Error saving user", err)
-  }
-
-  // now that we're sure user exists, load them from the db
-  dbUser = await db.get(dbUser._id)
-
-  // authenticate
-  const sessionId = newid()
-  const tenantId = getTenantId()
-  await createASession(dbUser._id, { sessionId, tenantId })
-
-  dbUser.token = jwt.sign(
-    {
-      userId: dbUser._id,
-      sessionId,
-    },
-    env.JWT_SECRET
-  )
-
-  return done(null, dbUser)
-}
-
-async function syncProfilePicture(
-  user: ThirdPartyUser,
-  thirdPartyUser: ThirdPartyUser
-) {
-  const pictureUrl = thirdPartyUser.profile?._json.picture
-  if (pictureUrl) {
-    const response = await fetch(pictureUrl)
-
-    if (response.status === 200) {
-      const type = response.headers.get("content-type") as string
-      if (type.startsWith("image/")) {
-        user.pictureUrl = pictureUrl
-      }
-    }
-  }
-
-  return user
-}
-
-/**
- * @returns a user that has been sync'd with third party information
- */
-async function syncUser(user: ThirdPartyUser, thirdPartyUser: ThirdPartyUser) {
-  // provider
-  user.provider = thirdPartyUser.provider
-  user.providerType = thirdPartyUser.providerType
-
-  if (thirdPartyUser.profile) {
-    const profile = thirdPartyUser.profile
-
-    if (profile.name) {
-      const name = profile.name
-      // first name
-      if (name.givenName) {
-        user.firstName = name.givenName
-      }
-      // last name
-      if (name.familyName) {
-        user.lastName = name.familyName
-      }
-    }
-
-    user = await syncProfilePicture(user, thirdPartyUser)
-
-    // profile
-    user.thirdPartyProfile = {
-      ...profile._json,
-    }
-  }
-
-  // oauth tokens for future use
-  if (thirdPartyUser.oauth2) {
-    user.oauth2 = {
-      ...thirdPartyUser.oauth2,
-    }
-  }
-
-  return user
-}

packages/backend-core/src/middleware/passport/utils.ts

@@ -1,6 +1,6 @@
-import { isMultiTenant, getTenantId } from "../../tenancy"
+import { isMultiTenant, getTenantId } from "../../context"
 import { getScopedConfig } from "../../db"
-import { ConfigType, Database, Config } from "@budibase/types"
+import { ConfigType, Database } from "@budibase/types"
 
 /**
  * Utility to handle authentication errors.

packages/backend-core/src/middleware/tenancy.ts

@@ -1,4 +1,5 @@
-import { doInTenant, getTenantIDFromCtx } from "../tenancy"
+import { doInTenant } from "../context"
+import { getTenantIDFromCtx } from "../tenancy"
 import { buildMatcherRegex, matches } from "./matchers"
 import { Header } from "../constants"
 import {

packages/backend-core/src/migrations/migrations.ts

@@ -4,10 +4,10 @@ import {
   StaticDatabases,
   getAllApps,
   getGlobalDBName,
-  doWithDB,
+  getDB,
 } from "../db"
 import environment from "../environment"
-import { doInTenant, getTenantIds, getTenantId } from "../tenancy"
+import * as platform from "../platform"
 import * as context from "../context"
 import { DEFINITIONS } from "."
 import {
@@ -47,7 +47,7 @@ export const runMigration = async (
   const migrationType = migration.type
   let tenantId: string | undefined
   if (migrationType !== MigrationType.INSTALLATION) {
-    tenantId = getTenantId()
+    tenantId = context.getTenantId()
   }
   const migrationName = migration.name
   const silent = migration.silent
@@ -86,66 +86,65 @@ export const runMigration = async (
     count++
     const lengthStatement = length > 1 ? `[${count}/${length}]` : ""
 
-    await doWithDB(dbName, async (db: any) => {
-      try {
-        const doc = await getMigrationsDoc(db)
+    const db = getDB(dbName)
+    try {
+      const doc = await getMigrationsDoc(db)
 
-        // the migration has already been run
-        if (doc[migrationName]) {
-          // check for force
-          if (
-            options.force &&
-            options.force[migrationType] &&
-            options.force[migrationType].includes(migrationName)
-          ) {
-            log(
-              `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
-            )
-          } else {
-            // no force, exit
-            return
-          }
-        }
-
-        // check if the migration is not a no-op
-        if (!options.noOp) {
+      // the migration has already been run
+      if (doc[migrationName]) {
+        // check for force
+        if (
+          options.force &&
+          options.force[migrationType] &&
+          options.force[migrationType].includes(migrationName)
+        ) {
           log(
-            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
-          )
-
-          if (migration.preventRetry) {
-            // eagerly set the completion date
-            // so that we never run this migration twice even upon failure
-            doc[migrationName] = Date.now()
-            const response = await db.put(doc)
-            doc._rev = response.rev
-          }
-
-          // run the migration
-          if (migrationType === MigrationType.APP) {
-            await context.doInAppContext(db.name, async () => {
-              await migration.fn(db)
-            })
-          } else {
-            await migration.fn(db)
-          }
-
-          log(
-            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
+            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
           )
+        } else {
+          // no force, exit
+          return
         }
-
-        // mark as complete
-        doc[migrationName] = Date.now()
-        await db.put(doc)
-      } catch (err) {
-        console.error(
-          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
-          err
-        )
-        throw err
       }
-    })
+
+      // check if the migration is not a no-op
+      if (!options.noOp) {
+        log(
+          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
+        )
+
+        if (migration.preventRetry) {
+          // eagerly set the completion date
+          // so that we never run this migration twice even upon failure
+          doc[migrationName] = Date.now()
+          const response = await db.put(doc)
+          doc._rev = response.rev
+        }
+
+        // run the migration
+        if (migrationType === MigrationType.APP) {
+          await context.doInAppContext(db.name, async () => {
+            await migration.fn(db)
+          })
+        } else {
+          await migration.fn(db)
+        }
+
+        log(
+          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
+        )
+      }
+
+      // mark as complete
+      doc[migrationName] = Date.now()
+      await db.put(doc)
+    } catch (err) {
+      console.error(
+        `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
+        err
+      )
+      throw err
+    }
   }
 }
 
@@ -160,7 +159,7 @@ export const runMigrations = async (
       tenantIds = [options.noOp.tenantId]
     } else if (!options.tenantIds || !options.tenantIds.length) {
       // run for all tenants
-      tenantIds = await getTenantIds()
+      tenantIds = await platform.tenants.getTenantIds()
     } else {
       tenantIds = options.tenantIds
     }
@@ -185,7 +184,10 @@ export const runMigrations = async (
     // for all migrations
     for (const migration of migrations) {
       // run the migration
-      await doInTenant(tenantId, () => runMigration(migration, options))
+      await context.doInTenant(
+        tenantId,
+        async () => await runMigration(migration, options)
+      )
     }
   }
   console.log("Migrations complete")

packages/backend-core/src/migrations/tests/index.spec.js

@@ -1,57 +0,0 @@
-require("../../../tests")
-const { runMigrations, getMigrationsDoc } = require("../index")
-const { getGlobalDBName, getDB } = require("../../db")
-
-const { structures, testEnv } = require("../../../tests")
-testEnv.multiTenant()
-
-let db
-
-describe("migrations", () => {
-
-  const migrationFunction = jest.fn()
-
-  const MIGRATIONS = [{
-    type: "global",
-    name: "test",
-    fn: migrationFunction
-  }]
-
-  let tenantId
-
-  beforeEach(() => {
-    tenantId = structures.tenant.id()
-    db = getDB(getGlobalDBName(tenantId))
-  })
-
-  afterEach(async () => {
-    jest.clearAllMocks()
-    await db.destroy()
-  })
-
-  const migrate = () => {
-    return runMigrations(MIGRATIONS, { tenantIds: [tenantId]})
-  }
-
-  it("should run a new migration", async () => {
-    await migrate()
-    expect(migrationFunction).toHaveBeenCalled()
-    const doc = await getMigrationsDoc(db)
-    expect(doc.test).toBeDefined()
-  })
-
-  it("should match snapshot", async () => {
-    await migrate()
-    const doc = await getMigrationsDoc(db)
-    expect(doc).toMatchSnapshot()
-  })
-
-  it("should skip a previously run migration", async () => {
-    await migrate()
-    const previousMigrationTime = await getMigrationsDoc(db).test
-    await migrate()
-    const currentMigrationTime = await getMigrationsDoc(db).test
-    expect(migrationFunction).toHaveBeenCalledTimes(1)
-    expect(currentMigrationTime).toBe(previousMigrationTime)
-  })
-})

packages/backend-core/src/migrations/tests/migrations.spec.ts

@@ -0,0 +1,64 @@
+import { testEnv, DBTestConfiguration } from "../../../tests"
+import * as migrations from "../index"
+import * as context from "../../context"
+import { MigrationType } from "@budibase/types"
+
+testEnv.multiTenant()
+
+describe("migrations", () => {
+  const config = new DBTestConfiguration()
+
+  const migrationFunction = jest.fn()
+
+  const MIGRATIONS = [
+    {
+      type: MigrationType.GLOBAL,
+      name: "test" as any,
+      fn: migrationFunction,
+    },
+  ]
+
+  beforeEach(() => {
+    config.newTenant()
+  })
+
+  afterEach(async () => {
+    jest.clearAllMocks()
+  })
+
+  const migrate = () => {
+    return migrations.runMigrations(MIGRATIONS, {
+      tenantIds: [config.tenantId],
+    })
+  }
+
+  it("should run a new migration", async () => {
+    await config.doInTenant(async () => {
+      await migrate()
+      expect(migrationFunction).toHaveBeenCalled()
+      const db = context.getGlobalDB()
+      const doc = await migrations.getMigrationsDoc(db)
+      expect(doc.test).toBeDefined()
+    })
+  })
+
+  it("should match snapshot", async () => {
+    await config.doInTenant(async () => {
+      await migrate()
+      const doc = await migrations.getMigrationsDoc(context.getGlobalDB())
+      expect(doc).toMatchSnapshot()
+    })
+  })
+
+  it("should skip a previously run migration", async () => {
+    await config.doInTenant(async () => {
+      const db = context.getGlobalDB()
+      await migrate()
+      const previousDoc = await migrations.getMigrationsDoc(db)
+      await migrate()
+      const currentDoc = await migrations.getMigrationsDoc(db)
+      expect(migrationFunction).toHaveBeenCalledTimes(1)
+      expect(currentDoc.test).toBe(previousDoc.test)
+    })
+  })
+})

packages/backend-core/src/objectStore/buckets/global.ts

@@ -1,5 +1,5 @@
 import env from "../../environment"
-import * as tenancy from "../../tenancy"
+import * as context from "../../context"
 import * as objectStore from "../objectStore"
 import * as cloudfront from "../cloudfront"
 
@@ -22,7 +22,7 @@ export const getGlobalFileUrl = (type: string, name: string, etag?: string) => {
 export const getGlobalFileS3Key = (type: string, name: string) => {
   let file = `${type}/${name}`
   if (env.MULTI_TENANCY) {
-    const tenantId = tenancy.getTenantId()
+    const tenantId = context.getTenantId()
     file = `${tenantId}/${file}`
   }
   return file

packages/backend-core/src/objectStore/buckets/plugins.ts

@@ -1,6 +1,6 @@
 import env from "../../environment"
 import * as objectStore from "../objectStore"
-import * as tenancy from "../../tenancy"
+import * as context from "../../context"
 import * as cloudfront from "../cloudfront"
 import { Plugin } from "@budibase/types"
 
@@ -61,7 +61,7 @@ const getPluginS3Key = (plugin: Plugin, fileName: string) => {
 export const getPluginS3Dir = (pluginName: string) => {
   let s3Key = `${pluginName}`
   if (env.MULTI_TENANCY) {
-    const tenantId = tenancy.getTenantId()
+    const tenantId = context.getTenantId()
     s3Key = `${tenantId}/${s3Key}`
   }
   if (env.CLOUDFRONT_CDN) {

packages/backend-core/src/platform/index.ts

@@ -0,0 +1,3 @@
+export * as users from "./users"
+export * as tenants from "./tenants"
+export * from "./platformDb"

packages/backend-core/src/platform/platformDb.ts

@@ -0,0 +1,6 @@
+import { StaticDatabases } from "../constants"
+import { getDB } from "../db/db"
+
+export function getPlatformDB() {
+  return getDB(StaticDatabases.PLATFORM_INFO.name)
+}

packages/backend-core/src/platform/tenants.ts

@@ -0,0 +1,101 @@
+import { StaticDatabases } from "../constants"
+import { getPlatformDB } from "./platformDb"
+import { LockName, LockOptions, LockType, Tenants } from "@budibase/types"
+import * as locks from "../redis/redlock"
+
+const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
+
+export const tenacyLockOptions: LockOptions = {
+  type: LockType.DEFAULT,
+  name: LockName.UPDATE_TENANTS_DOC,
+  ttl: 10 * 1000, // auto expire after 10 seconds
+  systemLock: true,
+}
+
+// READ
+
+export async function getTenantIds(): Promise<string[]> {
+  const tenants = await getTenants()
+  return tenants.tenantIds
+}
+
+async function getTenants(): Promise<Tenants> {
+  const db = getPlatformDB()
+  let tenants: Tenants
+
+  try {
+    tenants = await db.get(TENANT_DOC)
+  } catch (e: any) {
+    // doesn't exist yet - create
+    if (e.status === 404) {
+      tenants = await createTenantsDoc()
+    } else {
+      throw e
+    }
+  }
+
+  return tenants
+}
+
+export async function exists(tenantId: string) {
+  const tenants = await getTenants()
+  return tenants.tenantIds.indexOf(tenantId) !== -1
+}
+
+// CREATE / UPDATE
+
+function newTenantsDoc(): Tenants {
+  return {
+    _id: TENANT_DOC,
+    tenantIds: [],
+  }
+}
+
+async function createTenantsDoc(): Promise<Tenants> {
+  const db = getPlatformDB()
+  let tenants = newTenantsDoc()
+
+  try {
+    const response = await db.put(tenants)
+    tenants._rev = response.rev
+  } catch (e: any) {
+    // don't throw 409 is doc has already been created
+    if (e.status === 409) {
+      return db.get(TENANT_DOC)
+    }
+    throw e
+  }
+
+  return tenants
+}
+
+export async function addTenant(tenantId: string) {
+  const db = getPlatformDB()
+
+  // use a lock as tenant creation is conflict prone
+  await locks.doWithLock(tenacyLockOptions, async () => {
+    const tenants = await getTenants()
+
+    // write the new tenant if it doesn't already exist
+    if (tenants.tenantIds.indexOf(tenantId) === -1) {
+      tenants.tenantIds.push(tenantId)
+      await db.put(tenants)
+    }
+  })
+}
+
+// DELETE
+
+export async function removeTenant(tenantId: string) {
+  try {
+    await locks.doWithLock(tenacyLockOptions, async () => {
+      const db = getPlatformDB()
+      const tenants = await getTenants()
+      tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)
+      await db.put(tenants)
+    })
+  } catch (err) {
+    console.error(`Error removing tenant ${tenantId} from info db`, err)
+    throw err
+  }
+}

packages/backend-core/src/platform/tests/tenants.spec.ts

@@ -0,0 +1,25 @@
+import { DBTestConfiguration, structures } from "../../../tests"
+import * as tenants from "../tenants"
+
+describe("tenants", () => {
+  const config = new DBTestConfiguration()
+
+  describe("addTenant", () => {
+    it("concurrently adds multiple tenants safely", async () => {
+      const tenant1 = structures.tenant.id()
+      const tenant2 = structures.tenant.id()
+      const tenant3 = structures.tenant.id()
+
+      await Promise.all([
+        tenants.addTenant(tenant1),
+        tenants.addTenant(tenant2),
+        tenants.addTenant(tenant3),
+      ])
+
+      const tenantIds = await tenants.getTenantIds()
+      expect(tenantIds.includes(tenant1)).toBe(true)
+      expect(tenantIds.includes(tenant2)).toBe(true)
+      expect(tenantIds.includes(tenant3)).toBe(true)
+    })
+  })
+})

packages/backend-core/src/platform/users.ts

@@ -0,0 +1,90 @@
+import { getPlatformDB } from "./platformDb"
+import { DEFAULT_TENANT_ID } from "../constants"
+import env from "../environment"
+import {
+  PlatformUser,
+  PlatformUserByEmail,
+  PlatformUserById,
+  User,
+} from "@budibase/types"
+
+// READ
+
+export async function lookupTenantId(userId: string) {
+  if (!env.MULTI_TENANCY) {
+    return DEFAULT_TENANT_ID
+  }
+
+  const user = await getUserDoc(userId)
+  return user.tenantId
+}
+
+async function getUserDoc(emailOrId: string): Promise<PlatformUser> {
+  const db = getPlatformDB()
+  return db.get(emailOrId)
+}
+
+// CREATE
+
+function newUserIdDoc(id: string, tenantId: string): PlatformUserById {
+  return {
+    _id: id,
+    tenantId,
+  }
+}
+
+function newUserEmailDoc(
+  userId: string,
+  email: string,
+  tenantId: string
+): PlatformUserByEmail {
+  return {
+    _id: email,
+    userId,
+    tenantId,
+  }
+}
+
+/**
+ * Add a new user id or email doc if it doesn't exist.
+ */
+async function addUserDoc(emailOrId: string, newDocFn: () => PlatformUser) {
+  const db = getPlatformDB()
+  let user: PlatformUser
+
+  try {
+    await db.get(emailOrId)
+  } catch (e: any) {
+    if (e.status === 404) {
+      user = newDocFn()
+      await db.put(user)
+    } else {
+      throw e
+    }
+  }
+}
+
+export async function addUser(tenantId: string, userId: string, email: string) {
+  await Promise.all([
+    addUserDoc(userId, () => newUserIdDoc(userId, tenantId)),
+    addUserDoc(email, () => newUserEmailDoc(userId, email, tenantId)),
+  ])
+}
+
+// DELETE
+
+export async function removeUser(user: User) {
+  const db = getPlatformDB()
+  const keys = [user._id!, user.email]
+  const userDocs = await db.allDocs({
+    keys,
+    include_docs: true,
+  })
+  const toDelete = userDocs.rows.map((row: any) => {
+    return {
+      ...row.doc,
+      _deleted: true,
+    }
+  })
+  await db.bulkDocs(toDelete)
+}

packages/backend-core/src/queue/queue.ts

@@ -4,7 +4,6 @@ import { JobQueue } from "./constants"
 import InMemoryQueue from "./inMemoryQueue"
 import BullQueue from "bull"
 import { addListeners, StalledFn } from "./listeners"
-const { opts: redisOpts, redisProtocolUrl } = getRedisOptions()
 
 const CLEANUP_PERIOD_MS = 60 * 1000
 let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
@@ -20,6 +19,7 @@ export function createQueue<T>(
   jobQueue: JobQueue,
   opts: { removeStalledCb?: StalledFn } = {}
 ): BullQueue.Queue<T> {
+  const { opts: redisOpts, redisProtocolUrl } = getRedisOptions()
   const queueConfig: any = redisProtocolUrl || { redis: redisOpts }
   let queue: any
   if (!env.isTest()) {

packages/backend-core/src/redis/index.ts

@@ -3,4 +3,4 @@
 export { default as Client } from "./redis"
 export * as utils from "./utils"
 export * as clients from "./init"
-export * as redlock from "./redlock"
+export * as locks from "./redlock"

packages/backend-core/src/redis/init.ts

@@ -20,13 +20,17 @@ async function init() {
   ).init()
 }
 
-process.on("exit", async () => {
+export async function shutdown() {
   if (userClient) await userClient.finish()
   if (sessionClient) await sessionClient.finish()
   if (appClient) await appClient.finish()
   if (cacheClient) await cacheClient.finish()
   if (writethroughClient) await writethroughClient.finish()
   if (lockClient) await lockClient.finish()
+}
+
+process.on("exit", async () => {
+  await shutdown()
 })
 
 export async function getUserClient() {

packages/backend-core/src/redis/redis.ts

@@ -1,6 +1,6 @@
 import env from "../environment"
 // ioredis mock is all in memory
-const Redis = env.isTest() ? require("ioredis-mock") : require("ioredis")
+const Redis = env.MOCK_REDIS ? require("ioredis-mock") : require("ioredis")
 import {
   addDbPrefix,
   removeDbPrefix,
@@ -17,8 +17,13 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT
 // for testing just generate the client once
 let CLOSED = false
 let CLIENTS: { [key: number]: any } = {}
-// if in test always connected
-let CONNECTED = env.isTest()
+
+let CONNECTED = false
+
+// mock redis always connected
+if (env.MOCK_REDIS) {
+  CONNECTED = true
+}
 
 function pickClient(selectDb: number): any {
   return CLIENTS[selectDb]
@@ -57,7 +62,7 @@ function init(selectDb = DEFAULT_SELECT_DB) {
     return
   }
   // testing uses a single in memory client
-  if (env.isTest()) {
+  if (env.MOCK_REDIS) {
     CLIENTS[selectDb] = new Redis(getRedisOptions())
   }
   // start the timer - only allowed 5 seconds to connect
@@ -86,6 +91,11 @@ function init(selectDb = DEFAULT_SELECT_DB) {
   }
   // attach handlers
   client.on("end", (err: Error) => {
+    if (env.isTest()) {
+      // don't try to re-connect in test env
+      // allow the process to exit
+      return
+    }
     connectionError(selectDb, timeout, err)
   })
   client.on("error", (err: Error) => {

packages/backend-core/src/redis/redlock.ts

@@ -1,29 +1,22 @@
 import Redlock, { Options } from "redlock"
 import { getLockClient } from "./init"
 import { LockOptions, LockType } from "@budibase/types"
-import * as tenancy from "../tenancy"
-
-let noRetryRedlock: Redlock | undefined
+import * as context from "../context"
+import env from "../environment"
 
 const getClient = async (type: LockType): Promise<Redlock> => {
+  if (env.isTest() && type !== LockType.TRY_ONCE) {
+    return newRedlock(OPTIONS.TEST)
+  }
   switch (type) {
     case LockType.TRY_ONCE: {
-      if (!noRetryRedlock) {
-        noRetryRedlock = await newRedlock(OPTIONS.TRY_ONCE)
-      }
-      return noRetryRedlock
+      return newRedlock(OPTIONS.TRY_ONCE)
     }
     case LockType.DEFAULT: {
-      if (!noRetryRedlock) {
-        noRetryRedlock = await newRedlock(OPTIONS.DEFAULT)
-      }
-      return noRetryRedlock
+      return newRedlock(OPTIONS.DEFAULT)
     }
     case LockType.DELAY_500: {
-      if (!noRetryRedlock) {
-        noRetryRedlock = await newRedlock(OPTIONS.DELAY_500)
-      }
-      return noRetryRedlock
+      return newRedlock(OPTIONS.DELAY_500)
     }
     default: {
       throw new Error(`Could not get redlock client: ${type}`)
@@ -36,6 +29,11 @@ export const OPTIONS = {
     // immediately throws an error if the lock is already held
     retryCount: 0,
   },
+  TEST: {
+    // higher retry count in unit tests
+    // due to high contention.
+    retryCount: 100,
+  },
   DEFAULT: {
     // the expected clock drift; for more details
     // see http://redis.io/topics/distlock
@@ -69,28 +67,38 @@ export const doWithLock = async (opts: LockOptions, task: any) => {
   const redlock = await getClient(opts.type)
   let lock
   try {
-    // aquire lock
-    let name: string = `lock:${tenancy.getTenantId()}_${opts.name}`
+    // determine lock name
+    // by default use the tenantId for uniqueness, unless using a system lock
+    const prefix = opts.systemLock ? "system" : context.getTenantId()
+    let name: string = `lock:${prefix}_${opts.name}`
+
+    // add additional unique name if required
     if (opts.nameSuffix) {
       name = name + `_${opts.nameSuffix}`
     }
+
+    // create the lock
     lock = await redlock.lock(name, opts.ttl)
+
     // perform locked task
     // need to await to ensure completion before unlocking
     const result = await task()
     return result
   } catch (e: any) {
-    console.log("lock error")
+    console.warn("lock error")
     // lock limit exceeded
     if (e.name === "LockError") {
       if (opts.type === LockType.TRY_ONCE) {
         // don't throw for try-once locks, they will always error
         // due to retry count (0) exceeded
+        console.warn(e)
         return
       } else {
+        console.error(e)
         throw e
       }
     } else {
+      console.error(e)
       throw e
     }
   } finally {

packages/backend-core/src/redis/utils.ts

@@ -2,8 +2,6 @@ import env from "../environment"
 
 const SLOT_REFRESH_MS = 2000
 const CONNECT_TIMEOUT_MS = 10000
-const REDIS_URL = !env.REDIS_URL ? "localhost:6379" : env.REDIS_URL
-const REDIS_PASSWORD = !env.REDIS_PASSWORD ? "budibase" : env.REDIS_PASSWORD
 export const SEPARATOR = "-"
 
 /**
@@ -60,8 +58,8 @@ export enum SelectableDatabase {
 }
 
 export function getRedisOptions(clustered = false) {
-  let password = REDIS_PASSWORD
-  let url: string[] | string = REDIS_URL.split("//")
+  let password = env.REDIS_PASSWORD
+  let url: string[] | string = env.REDIS_URL.split("//")
   // get rid of the protocol
   url = url.length > 1 ? url[1] : url[0]
   // check for a password etc
@@ -78,8 +76,8 @@ export function getRedisOptions(clustered = false) {
   let redisProtocolUrl
 
   // fully qualified redis URL
-  if (/rediss?:\/\//.test(REDIS_URL)) {
-    redisProtocolUrl = REDIS_URL
+  if (/rediss?:\/\//.test(env.REDIS_URL)) {
+    redisProtocolUrl = env.REDIS_URL
   }
 
   const opts: any = {

packages/backend-core/src/tenancy/db.ts

@@ -0,0 +1,6 @@
+import { getDB } from "../db/db"
+import { getGlobalDBName } from "../context"
+
+export function getTenantDB(tenantId: string) {
+  return getDB(getGlobalDBName(tenantId))
+}

packages/backend-core/src/tenancy/index.ts

@@ -1,2 +1,2 @@
-export * from "../context"
+export * from "./db"
 export * from "./tenancy"

packages/backend-core/src/tenancy/tenancy.ts

@@ -1,4 +1,3 @@
-import { doWithDB, getGlobalDBName } from "../db"
 import {
   DEFAULT_TENANT_ID,
   getTenantId,
@@ -11,10 +10,7 @@ import {
   TenantResolutionStrategy,
   GetTenantIdOptions,
 } from "@budibase/types"
-import { Header, StaticDatabases } from "../constants"
-
-const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
-const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
+import { Header } from "../constants"
 
 export function addTenantToUrl(url: string) {
   const tenantId = getTenantId()
@@ -27,89 +23,6 @@ export function addTenantToUrl(url: string) {
   return url
 }
 
-export async function doesTenantExist(tenantId: string) {
-  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
-    let tenants
-    try {
-      tenants = await db.get(TENANT_DOC)
-    } catch (err) {
-      // if theres an error the doc doesn't exist, no tenants exist
-      return false
-    }
-    return (
-      tenants &&
-      Array.isArray(tenants.tenantIds) &&
-      tenants.tenantIds.indexOf(tenantId) !== -1
-    )
-  })
-}
-
-export async function tryAddTenant(
-  tenantId: string,
-  userId: string,
-  email: string,
-  afterCreateTenant: () => Promise<void>
-) {
-  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
-    const getDoc = async (id: string) => {
-      if (!id) {
-        return null
-      }
-      try {
-        return await db.get(id)
-      } catch (err) {
-        return { _id: id }
-      }
-    }
-    let [tenants, userIdDoc, emailDoc] = await Promise.all([
-      getDoc(TENANT_DOC),
-      getDoc(userId),
-      getDoc(email),
-    ])
-    if (!Array.isArray(tenants.tenantIds)) {
-      tenants = {
-        _id: TENANT_DOC,
-        tenantIds: [],
-      }
-    }
-    let promises = []
-    if (userIdDoc) {
-      userIdDoc.tenantId = tenantId
-      promises.push(db.put(userIdDoc))
-    }
-    if (emailDoc) {
-      emailDoc.tenantId = tenantId
-      emailDoc.userId = userId
-      promises.push(db.put(emailDoc))
-    }
-    if (tenants.tenantIds.indexOf(tenantId) === -1) {
-      tenants.tenantIds.push(tenantId)
-      promises.push(db.put(tenants))
-      await afterCreateTenant()
-    }
-    await Promise.all(promises)
-  })
-}
-
-export function doWithGlobalDB(tenantId: string, cb: any) {
-  return doWithDB(getGlobalDBName(tenantId), cb)
-}
-
-export async function lookupTenantId(userId: string) {
-  return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: any) => {
-    let tenantId = env.MULTI_TENANCY ? DEFAULT_TENANT_ID : null
-    try {
-      const doc = await db.get(userId)
-      if (doc && doc.tenantId) {
-        tenantId = doc.tenantId
-      }
-    } catch (err) {
-      // just return the default
-    }
-    return tenantId
-  })
-}
-
 export const isUserInAppTenant = (appId: string, user?: any) => {
   let userTenantId
   if (user) {
@@ -121,19 +34,6 @@ export const isUserInAppTenant = (appId: string, user?: any) => {
   return tenantId === userTenantId
 }
 
-export async function getTenantIds() {
-  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
-    let tenants
-    try {
-      tenants = await db.get(TENANT_DOC)
-    } catch (err) {
-      // if theres an error the doc doesn't exist, no tenants exist
-      return []
-    }
-    return (tenants && tenants.tenantIds) || []
-  })
-}
-
 const ALL_STRATEGIES = Object.values(TenantResolutionStrategy)
 
 export const getTenantIDFromCtx = (

packages/backend-core/src/users.ts

@@ -8,6 +8,7 @@ import {
 } from "./db"
 import { BulkDocsResponse, User } from "@budibase/types"
 import { getGlobalDB } from "./context"
+import * as context from "./context"
 
 export const bulkGetGlobalUsersById = async (userIds: string[]) => {
   const db = getGlobalDB()
@@ -24,6 +25,11 @@ export const bulkUpdateGlobalUsers = async (users: User[]) => {
   return (await db.bulkDocs(users)) as BulkDocsResponse
 }
 
+export async function getById(id: string): Promise<User> {
+  const db = context.getGlobalDB()
+  return db.get(id)
+}
+
 /**
  * Given an email address this will use a view to search through
  * all the users to find one with this email address.

packages/backend-core/src/utils/tests/utils.spec.ts

@@ -1,21 +1,12 @@
-import { structures } from "../../../tests"
+import { structures, DBTestConfiguration } from "../../../tests"
 import * as utils from "../../utils"
-import * as events from "../../events"
 import * as db from "../../db"
 import { Header } from "../../constants"
-import { doInTenant } from "../../context"
 import { newid } from "../../utils"
+import env from "../../environment"
 
 describe("utils", () => {
-  describe("platformLogout", () => {
-    it("should call platform logout", async () => {
-      await doInTenant(structures.tenant.id(), async () => {
-        const ctx = structures.koa.newContext()
-        await utils.platformLogout({ ctx, userId: "test" })
-        expect(events.auth.logout).toBeCalledTimes(1)
-      })
-    })
-  })
+  const config = new DBTestConfiguration()
 
   describe("getAppIdFromCtx", () => {
     it("gets appId from header", async () => {
@@ -50,21 +41,28 @@ describe("utils", () => {
     })
 
     it("gets appId from url", async () => {
-      const ctx = structures.koa.newContext()
-      const expected = db.generateAppID()
-      const app = structures.apps.app(expected)
+      await config.doInTenant(async () => {
+        const url = "http://test.com"
+        env._set("PLATFORM_URL", url)
 
-      // set custom url
-      const appUrl = newid()
-      app.url = `/${appUrl}`
-      ctx.path = `/app/${appUrl}`
+        const ctx = structures.koa.newContext()
+        ctx.host = `${config.tenantId}.test.com`
 
-      // save the app
-      const database = db.getDB(expected)
-      await database.put(app)
+        const expected = db.generateAppID(config.tenantId)
+        const app = structures.apps.app(expected)
 
-      const actual = await utils.getAppIdFromCtx(ctx)
-      expect(actual).toBe(expected)
+        // set custom url
+        const appUrl = newid()
+        app.url = `/${appUrl}`
+        ctx.path = `/app/${appUrl}`
+
+        // save the app
+        const database = db.getDB(expected)
+        await database.put(app)
+
+        const actual = await utils.getAppIdFromCtx(ctx)
+        expect(actual).toBe(expected)
+      })
     })
 
     it("doesn't get appId from url when previewing", async () => {

packages/backend-core/src/utils/utils.ts

@@ -2,23 +2,15 @@ import { getAllApps, queryGlobalView } from "../db"
 import { options } from "../middleware/passport/jwt"
 import {
   Header,
-  Cookie,
   MAX_VALID_DATE,
   DocumentType,
   SEPARATOR,
   ViewName,
 } from "../constants"
 import env from "../environment"
-import * as userCache from "../cache/user"
-import { getSessionsForUser, invalidateSessions } from "../security/sessions"
-import * as events from "../events"
 import * as tenancy from "../tenancy"
-import {
-  App,
-  Ctx,
-  PlatformLogoutOpts,
-  TenantResolutionStrategy,
-} from "@budibase/types"
+import * as context from "../context"
+import { App, Ctx, TenantResolutionStrategy } from "@budibase/types"
 import { SetOption } from "cookies"
 const jwt = require("jsonwebtoken")
 
@@ -38,7 +30,7 @@ export async function resolveAppUrl(ctx: Ctx) {
   const appUrl = ctx.path.split("/")[2]
   let possibleAppUrl = `/${appUrl.toLowerCase()}`
 
-  let tenantId: string | null = tenancy.getTenantId()
+  let tenantId: string | null = context.getTenantId()
   if (env.MULTI_TENANCY) {
     // always use the tenant id from the subdomain in multi tenancy
     // this ensures the logged-in user tenant id doesn't overwrite
@@ -49,7 +41,7 @@ export async function resolveAppUrl(ctx: Ctx) {
   }
 
   // search prod apps for a url that matches
-  const apps: App[] = await tenancy.doInTenant(tenantId, () =>
+  const apps: App[] = await context.doInTenant(tenantId, () =>
     getAllApps({ dev: false })
   )
   const app = apps.filter(
@@ -222,35 +214,6 @@ export async function getBuildersCount() {
   return builders.length
 }
 
-/**
- * Logs a user out from budibase. Re-used across account portal and builder.
- */
-export async function platformLogout(opts: PlatformLogoutOpts) {
-  const ctx = opts.ctx
-  const userId = opts.userId
-  const keepActiveSession = opts.keepActiveSession
-
-  if (!ctx) throw new Error("Koa context must be supplied to logout.")
-
-  const currentSession = getCookie(ctx, Cookie.Auth)
-  let sessions = await getSessionsForUser(userId)
-
-  if (keepActiveSession) {
-    sessions = sessions.filter(
-      session => session.sessionId !== currentSession.sessionId
-    )
-  } else {
-    // clear cookies
-    clearCookie(ctx, Cookie.Auth)
-    clearCookie(ctx, Cookie.CurrentApp)
-  }
-
-  const sessionIds = sessions.map(({ sessionId }) => sessionId)
-  await invalidateSessions(userId, { sessionIds, reason: "logout" })
-  await events.auth.logout()
-  await userCache.invalidateUser(userId)
-}
-
 export function timeout(timeMs: number) {
   return new Promise(resolve => setTimeout(resolve, timeMs))
 }

packages/backend-core/tests/jestEnv.ts

@@ -1,23 +1,6 @@
-import env from "../src/environment"
-import { mocks } from "./utilities"
-
-// must explicitly enable fetch mock
-mocks.fetch.enable()
-
-// mock all dates to 2020-01-01T00:00:00.000Z
-// use tk.reset() to use real dates in individual tests
-import tk from "timekeeper"
-tk.freeze(mocks.date.MOCK_DATE)
-
-env._set("SELF_HOSTED", "1")
-env._set("NODE_ENV", "jest")
-
-if (!process.env.DEBUG) {
-  global.console.log = jest.fn() // console.log are ignored in tests
-}
-
-if (!process.env.CI) {
-  // set a longer timeout in dev for debugging
-  // 100 seconds
-  jest.setTimeout(100000)
-}
+process.env.SELF_HOSTED = "1"
+process.env.MULTI_TENANCY = "1"
+process.env.NODE_ENV = "jest"
+process.env.MOCK_REDIS = "1"
+process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
+process.env.ENABLE_4XX_HTTP_LOGGING = "0"

packages/backend-core/tests/jestSetup.ts

@@ -1,4 +1,23 @@
+import "./logging"
 import env from "../src/environment"
-import { testContainerUtils } from "./utilities"
+import { mocks, testContainerUtils } from "./utilities"
+
+// must explicitly enable fetch mock
+mocks.fetch.enable()
+
+// mock all dates to 2020-01-01T00:00:00.000Z
+// use tk.reset() to use real dates in individual tests
+import tk from "timekeeper"
+tk.freeze(mocks.date.MOCK_DATE)
+
+if (!process.env.DEBUG) {
+  console.log = jest.fn() // console.log are ignored in tests
+}
+
+if (!process.env.CI) {
+  // set a longer timeout in dev for debugging
+  // 100 seconds
+  jest.setTimeout(100000)
+}
 
 testContainerUtils.setupEnv(env)

packages/backend-core/tests/logging.ts

@@ -0,0 +1,34 @@
+export enum LogLevel {
+  TRACE = "trace",
+  DEBUG = "debug",
+  INFO = "info",
+  WARN = "warn",
+  ERROR = "error",
+}
+
+const LOG_INDEX: { [key in LogLevel]: number } = {
+  [LogLevel.TRACE]: 1,
+  [LogLevel.DEBUG]: 2,
+  [LogLevel.INFO]: 3,
+  [LogLevel.WARN]: 4,
+  [LogLevel.ERROR]: 5,
+}
+
+const setIndex = LOG_INDEX[process.env.LOG_LEVEL as LogLevel]
+
+if (setIndex > LOG_INDEX.trace) {
+  global.console.trace = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.debug) {
+  global.console.debug = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.info) {
+  global.console.info = jest.fn()
+  global.console.log = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.warn) {
+  global.console.warn = jest.fn()
+}

packages/backend-core/tests/utilities/DBTestConfiguration.ts

@@ -0,0 +1,36 @@
+import "./mocks"
+import * as structures from "./structures"
+import * as testEnv from "./testEnv"
+import * as context from "../../src/context"
+
+class DBTestConfiguration {
+  tenantId: string
+
+  constructor() {
+    // db tests need to be multi tenant to prevent conflicts
+    testEnv.multiTenant()
+    this.tenantId = structures.tenant.id()
+  }
+
+  newTenant() {
+    this.tenantId = structures.tenant.id()
+  }
+
+  // TENANCY
+
+  doInTenant(task: any) {
+    return context.doInTenant(this.tenantId, () => {
+      return task()
+    })
+  }
+
+  getTenantId() {
+    try {
+      return context.getTenantId()
+    } catch (e) {
+      return this.tenantId!
+    }
+  }
+}
+
+export default DBTestConfiguration

packages/backend-core/tests/utilities/db.ts

@@ -1,9 +0,0 @@
-import * as db from "../../src/db"
-
-const dbConfig = {
-  inMemory: true,
-}
-
-export const init = () => {
-  db.init(dbConfig)
-}

packages/backend-core/tests/utilities/index.ts

@@ -4,5 +4,4 @@ export { generator } from "./structures"
 export * as testEnv from "./testEnv"
 export * as testContainerUtils from "./testContainerUtils"
 
-import * as dbConfig from "./db"
-dbConfig.init()
+export { default as DBTestConfiguration } from "./DBTestConfiguration"

packages/backend-core/tests/utilities/mocks/accounts.ts

@@ -1,13 +0,0 @@
-const mockGetAccount = jest.fn()
-const mockGetAccountByTenantId = jest.fn()
-const mockGetStatus = jest.fn()
-
-jest.mock("../../../src/cloud/accounts", () => ({
-  getAccount: mockGetAccount,
-  getAccountByTenantId: mockGetAccountByTenantId,
-  getStatus: mockGetStatus,
-}))
-
-export const getAccount = mockGetAccount
-export const getAccountByTenantId = mockGetAccountByTenantId
-export const getStatus = mockGetStatus

packages/backend-core/tests/utilities/mocks/index.ts

@@ -1,4 +1,7 @@
-export * as accounts from "./accounts"
+jest.mock("../../../src/accounts")
+import * as _accounts from "../../../src/accounts"
+export const accounts = jest.mocked(_accounts)
+
 export * as date from "./date"
 export * as licenses from "./licenses"
 export { default as fetch } from "./fetch"

packages/backend-core/tests/utilities/structures/accounts.ts

@@ -1,6 +1,15 @@
 import { generator, uuid } from "."
 import * as db from "../../../src/db/utils"
-import { Account, AuthType, CloudAccount, Hosting } from "@budibase/types"
+import {
+  Account,
+  AccountSSOProvider,
+  AccountSSOProviderType,
+  AuthType,
+  CloudAccount,
+  Hosting,
+  SSOAccount,
+} from "@budibase/types"
+import _ from "lodash"
 
 export const account = (): Account => {
   return {
@@ -27,3 +36,28 @@ export const cloudAccount = (): CloudAccount => {
     budibaseUserId: db.generateGlobalUserID(),
   }
 }
+
+function providerType(): AccountSSOProviderType {
+  return _.sample(
+    Object.values(AccountSSOProviderType)
+  ) as AccountSSOProviderType
+}
+
+function provider(): AccountSSOProvider {
+  return _.sample(Object.values(AccountSSOProvider)) as AccountSSOProvider
+}
+
+export function ssoAccount(): SSOAccount {
+  return {
+    ...cloudAccount(),
+    authType: AuthType.SSO,
+    oauth2: {
+      accessToken: generator.string(),
+      refreshToken: generator.string(),
+    },
+    pictureUrl: generator.url(),
+    provider: provider(),
+    providerType: providerType(),
+    thirdPartyProfile: {},
+  }
+}

packages/backend-core/tests/utilities/structures/index.ts

@@ -5,8 +5,10 @@ export const generator = new Chance()
 
 export * as accounts from "./accounts"
 export * as apps from "./apps"
+export * as db from "./db"
 export * as koa from "./koa"
 export * as licenses from "./licenses"
 export * as plugins from "./plugins"
+export * as sso from "./sso"
 export * as tenant from "./tenants"
-export * as db from "./db"
+export * as users from "./users"

packages/backend-core/tests/utilities/structures/sso.ts

@@ -0,0 +1,100 @@
+import {
+  GoogleInnerConfig,
+  JwtClaims,
+  OIDCInnerConfig,
+  OIDCWellKnownConfig,
+  SSOAuthDetails,
+  SSOProfile,
+  SSOProviderType,
+  User,
+} from "@budibase/types"
+import { uuid, generator, users, email } from "./index"
+import _ from "lodash"
+
+export function providerType(): SSOProviderType {
+  return _.sample(Object.values(SSOProviderType)) as SSOProviderType
+}
+
+export function ssoProfile(user?: User): SSOProfile {
+  if (!user) {
+    user = users.user()
+  }
+  return {
+    id: user._id!,
+    name: {
+      givenName: user.firstName,
+      familyName: user.lastName,
+    },
+    _json: {
+      email: user.email,
+      picture: "http://test.com",
+    },
+    provider: generator.string(),
+  }
+}
+
+export function authDetails(user?: User): SSOAuthDetails {
+  if (!user) {
+    user = users.user()
+  }
+
+  const userId = user._id || uuid()
+  const provider = generator.string()
+
+  const profile = ssoProfile(user)
+  profile.provider = provider
+  profile.id = userId
+
+  return {
+    email: user.email,
+    oauth2: {
+      refreshToken: generator.string(),
+      accessToken: generator.string(),
+    },
+    profile,
+    provider,
+    providerType: providerType(),
+    userId,
+  }
+}
+
+// OIDC
+
+export function oidcConfig(): OIDCInnerConfig {
+  return {
+    uuid: uuid(),
+    activated: true,
+    logo: "",
+    name: generator.string(),
+    configUrl: "http://someconfigurl",
+    clientID: generator.string(),
+    clientSecret: generator.string(),
+  }
+}
+
+// response from .well-known/openid-configuration
+export function oidcWellKnownConfig(): OIDCWellKnownConfig {
+  return {
+    issuer: generator.string(),
+    authorization_endpoint: generator.url(),
+    token_endpoint: generator.url(),
+    userinfo_endpoint: generator.url(),
+  }
+}
+
+export function jwtClaims(): JwtClaims {
+  return {
+    email: email(),
+    preferred_username: email(),
+  }
+}
+
+// GOOGLE
+
+export function googleConfig(): GoogleInnerConfig {
+  return {
+    activated: true,
+    clientID: generator.string(),
+    clientSecret: generator.string(),
+  }
+}

packages/backend-core/tests/utilities/structures/users.ts

@@ -0,0 +1,70 @@
+import { generator } from "../"
+import {
+  AdminUser,
+  BuilderUser,
+  SSOAuthDetails,
+  SSOUser,
+  User,
+} from "@budibase/types"
+import { v4 as uuid } from "uuid"
+import * as sso from "./sso"
+
+export const newEmail = () => {
+  return `${uuid()}@test.com`
+}
+
+export const user = (userProps?: any): User => {
+  return {
+    email: newEmail(),
+    password: "test",
+    roles: { app_test: "admin" },
+    firstName: generator.first(),
+    lastName: generator.last(),
+    pictureUrl: "http://test.com",
+    ...userProps,
+  }
+}
+
+export const adminUser = (userProps?: any): AdminUser => {
+  return {
+    ...user(userProps),
+    admin: {
+      global: true,
+    },
+    builder: {
+      global: true,
+    },
+  }
+}
+
+export const builderUser = (userProps?: any): BuilderUser => {
+  return {
+    ...user(userProps),
+    builder: {
+      global: true,
+    },
+  }
+}
+
+export function ssoUser(
+  opts: { user?: any; details?: SSOAuthDetails } = {}
+): SSOUser {
+  const base = user(opts.user)
+  delete base.password
+
+  if (!opts.details) {
+    opts.details = sso.authDetails(base)
+  }
+
+  return {
+    ...base,
+    forceResetPassword: false,
+    oauth2: opts.details?.oauth2,
+    provider: opts.details?.provider!,
+    providerType: opts.details?.providerType!,
+    thirdPartyProfile: {
+      email: base.email,
+      picture: base.pictureUrl,
+    },
+  }
+}

packages/backend-core/tests/utilities/testContainerUtils.ts

@@ -34,12 +34,17 @@ function getMinioConfig() {
   return getContainerInfo("minio-service", 9000)
 }
 
+function getRedisConfig() {
+  return getContainerInfo("redis-service", 6379)
+}
+
 export function setupEnv(...envs: any[]) {
   const configs = [
     { key: "COUCH_DB_PORT", value: getCouchConfig().port },
     { key: "COUCH_DB_URL", value: getCouchConfig().url },
     { key: "MINIO_PORT", value: getMinioConfig().port },
     { key: "MINIO_URL", value: getMinioConfig().url },
+    { key: "REDIS_URL", value: getRedisConfig().url },
   ]
 
   for (const config of configs.filter(x => !!x.value)) {

packages/backend-core/tests/utilities/testEnv.ts

@@ -1,12 +1,12 @@
 import env from "../../src/environment"
-import * as tenancy from "../../src/tenancy"
-import { newid } from "../../src/utils"
+import * as context from "../../src/context"
+import * as structures from "./structures"
 
 // TENANCY
 
 export async function withTenant(task: (tenantId: string) => any) {
-  const tenantId = newid()
-  return tenancy.doInTenant(tenantId, async () => {
+  const tenantId = structures.tenant.id()
+  return context.doInTenant(tenantId, async () => {
     await task(tenantId)
   })
 }
@@ -19,6 +19,14 @@ export function multiTenant() {
   env._set("MULTI_TENANCY", 1)
 }
 
+export function selfHosted() {
+  env._set("SELF_HOSTED", 1)
+}
+
+export function cloudHosted() {
+  env._set("SELF_HOSTED", 0)
+}
+
 // NODE
 
 export function nodeDev() {

packages/backend-core/yarn.lock

@@ -1197,10 +1197,10 @@
   dependencies:
     "@types/istanbul-lib-report" "*"
 
-"@types/jest@27.5.1":
-  version "27.5.1"
-  resolved "https://registry.yarnpkg.com/@types/jest/-/jest-27.5.1.tgz#2c8b6dc6ff85c33bcd07d0b62cb3d19ddfdb3ab9"
-  integrity sha512-fUy7YRpT+rHXto1YlL+J9rs0uLGyiqVt3ZOTQR+4ROc47yNl8WLdVLgUloBRhOxP1PZvguHl44T3H0wAWxahYQ==
+"@types/jest@28.1.1":
+  version "28.1.1"
+  resolved "https://registry.yarnpkg.com/@types/jest/-/jest-28.1.1.tgz#8c9ba63702a11f8c386ee211280e8b68cb093cd1"
+  integrity sha512-C2p7yqleUKtCkVjlOur9BWVA4HgUQmEj/HWCt5WzZ5mLXrWnyIfl0wGuArc+kBXsy0ZZfLp+7dywB4HtSVYGVA==
   dependencies:
     jest-matcher-utils "^27.0.0"
     pretty-format "^27.0.0"

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.18-alpha.8",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/string-templates": "2.3.17-alpha.4",
+    "@budibase/string-templates": "2.3.18-alpha.8",
     "@spectrum-css/accordion": "3.0.24",
     "@spectrum-css/actionbutton": "1.0.1",
     "@spectrum-css/actiongroup": "1.0.1",

packages/bbui/src/Icon/Icon.svelte

@@ -67,6 +67,9 @@
     color: var(--spectrum-alias-icon-color-selected-hover) !important;
     cursor: pointer;
   }
+  svg.hoverable:active {
+    color: var(--spectrum-global-color-blue-400) !important;
+  }
 
   svg.disabled {
     color: var(--spectrum-global-color-gray-500) !important;

packages/bbui/src/InlineAlert/InlineAlert.svelte

@@ -57,5 +57,7 @@
     --spectrum-semantic-negative-icon-color: #e34850;
     min-width: 100px;
     margin: 0;
+    border-color: var(--spectrum-global-color-gray-400);
+    border-width: 1px;
   }
 </style>

packages/bbui/src/Label/Label.svelte

@@ -21,7 +21,7 @@
   label {
     padding: 0;
     white-space: nowrap;
-    color: var(--spectrum-global-color-gray-600);
+    color: var(--spectrum-global-color-gray-700);
   }
 
   .muted {

packages/bbui/src/Modal/Modal.svelte

@@ -1,7 +1,7 @@
 <script>
   import "@spectrum-css/modal/dist/index-vars.css"
   import "@spectrum-css/underlay/dist/index-vars.css"
-  import { createEventDispatcher, setContext, tick } from "svelte"
+  import { createEventDispatcher, setContext, tick, onMount } from "svelte"
   import { fade, fly } from "svelte/transition"
   import Portal from "svelte-portal"
   import Context from "../context"
@@ -62,9 +62,14 @@
   }
 
   setContext(Context.Modal, { show, hide, cancel })
-</script>
 
-<svelte:window on:keydown={handleKey} />
+  onMount(() => {
+    document.addEventListener("keydown", handleKey)
+    return () => {
+      document.removeEventListener("keydown", handleKey)
+    }
+  })
+</script>
 
 {#if inline}
   {#if visible}

packages/bbui/src/helpers.js

@@ -104,6 +104,9 @@ export const deepSet = (obj, key, value) => {
  * @param obj the object to clone
  */
 export const cloneDeep = obj => {
+  if (!obj) {
+    return obj
+  }
   return JSON.parse(JSON.stringify(obj))
 }
 

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.18-alpha.8",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -58,10 +58,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.3.17-alpha.4",
-    "@budibase/client": "2.3.17-alpha.4",
-    "@budibase/frontend-core": "2.3.17-alpha.4",
-    "@budibase/string-templates": "2.3.17-alpha.4",
+    "@budibase/bbui": "2.3.18-alpha.8",
+    "@budibase/client": "2.3.18-alpha.8",
+    "@budibase/frontend-core": "2.3.18-alpha.8",
+    "@budibase/string-templates": "2.3.18-alpha.8",
     "@fortawesome/fontawesome-svg-core": "^6.2.1",
     "@fortawesome/free-brands-svg-icons": "^6.2.1",
     "@fortawesome/free-solid-svg-icons": "^6.2.1",
@@ -72,6 +72,7 @@
     "codemirror": "^5.59.0",
     "dayjs": "^1.11.2",
     "downloadjs": "1.4.7",
+    "fast-json-patch": "^3.1.1",
     "lodash": "4.17.21",
     "posthog-js": "^1.36.0",
     "remixicon": "2.5.0",

packages/builder/setup.js

@@ -19,6 +19,7 @@ process.env.COUCH_DB_USER = "budibase"
 process.env.COUCH_DB_PASSWORD = "budibase"
 process.env.INTERNAL_API_KEY = "budibase"
 process.env.ALLOW_DEV_AUTOMATIONS = 1
+process.env.MOCK_REDIS = 1
 
 // Stop info logs polluting test outputs
 process.env.LOG_LEVEL = "error"

packages/builder/src/builderStore/index.js

@@ -5,12 +5,47 @@ import { getThemeStore } from "./store/theme"
 import { derived } from "svelte/store"
 import { findComponent, findComponentPath } from "./componentUtils"
 import { RoleUtils } from "@budibase/frontend-core"
+import { createHistoryStore } from "builderStore/store/history"
+import { get } from "svelte/store"
 
 export const store = getFrontendStore()
 export const automationStore = getAutomationStore()
 export const themeStore = getThemeStore()
 export const temporalStore = getTemporalStore()
 
+// Setup history for screens
+export const screenHistoryStore = createHistoryStore({
+  getDoc: id => get(store).screens?.find(screen => screen._id === id),
+  selectDoc: store.actions.screens.select,
+  afterAction: () => {
+    // Ensure a valid component is selected
+    if (!get(selectedComponent)) {
+      store.update(state => ({
+        ...state,
+        selectedComponentId: get(selectedScreen)?.props._id,
+      }))
+    }
+  },
+})
+store.actions.screens.save = screenHistoryStore.wrapSaveDoc(
+  store.actions.screens.save
+)
+store.actions.screens.delete = screenHistoryStore.wrapDeleteDoc(
+  store.actions.screens.delete
+)
+
+// Setup history for automations
+export const automationHistoryStore = createHistoryStore({
+  getDoc: automationStore.actions.getDefinition,
+  selectDoc: automationStore.actions.select,
+})
+automationStore.actions.save = automationHistoryStore.wrapSaveDoc(
+  automationStore.actions.save
+)
+automationStore.actions.delete = automationHistoryStore.wrapDeleteDoc(
+  automationStore.actions.delete
+)
+
 export const selectedScreen = derived(store, $store => {
   return $store.screens.find(screen => screen._id === $store.selectedScreenId)
 })
@@ -71,3 +106,13 @@ export const selectedComponentPath = derived(
     ).map(component => component._id)
   }
 )
+
+// Derived automation state
+export const selectedAutomation = derived(automationStore, $automationStore => {
+  if (!$automationStore.selectedAutomationId) {
+    return null
+  }
+  return $automationStore.automations?.find(
+    x => x._id === $automationStore.selectedAutomationId
+  )
+})

packages/builder/src/builderStore/store/automation/Automation.js

@@ -1,69 +0,0 @@
-import { generate } from "shortid"
-
-/**
- * Class responsible for the traversing of the automation definition.
- * Automation definitions are stored in linked lists.
- */
-export default class Automation {
-  constructor(automation) {
-    this.automation = automation
-  }
-
-  hasTrigger() {
-    return this.automation.definition.trigger
-  }
-
-  addTestData(data) {
-    this.automation.testData = { ...this.automation.testData, ...data }
-  }
-
-  addBlock(block, idx) {
-    // Make sure to add trigger if doesn't exist
-    if (!this.hasTrigger() && block.type === "TRIGGER") {
-      const trigger = { id: generate(), ...block }
-      this.automation.definition.trigger = trigger
-      return trigger
-    }
-
-    const newBlock = { id: generate(), ...block }
-    this.automation.definition.steps.splice(idx, 0, newBlock)
-    return newBlock
-  }
-
-  updateBlock(updatedBlock, id) {
-    const { steps, trigger } = this.automation.definition
-
-    if (trigger && trigger.id === id) {
-      this.automation.definition.trigger = updatedBlock
-      return
-    }
-
-    const stepIdx = steps.findIndex(step => step.id === id)
-    if (stepIdx < 0) throw new Error("Block not found.")
-    steps.splice(stepIdx, 1, updatedBlock)
-    this.automation.definition.steps = steps
-  }
-
-  deleteBlock(id) {
-    const { steps, trigger } = this.automation.definition
-
-    if (trigger && trigger.id === id) {
-      this.automation.definition.trigger = null
-      return
-    }
-
-    const stepIdx = steps.findIndex(step => step.id === id)
-    if (stepIdx < 0) throw new Error("Block not found.")
-    steps.splice(stepIdx, 1)
-    this.automation.definition.steps = steps
-  }
-
-  constructBlock(type, stepId, blockDefinition) {
-    return {
-      ...blockDefinition,
-      inputs: blockDefinition.inputs || {},
-      stepId,
-      type,
-    }
-  }
-}

packages/builder/src/builderStore/store/automation/index.js

@@ -1,16 +1,18 @@
-import { writable } from "svelte/store"
+import { writable, get } from "svelte/store"
 import { API } from "api"
-import Automation from "./Automation"
 import { cloneDeep } from "lodash/fp"
+import { generate } from "shortid"
+import { selectedAutomation } from "builderStore"
 
 const initialAutomationState = {
   automations: [],
+  testResults: null,
   showTestPanel: false,
   blockDefinitions: {
     TRIGGER: [],
     ACTION: [],
   },
-  selectedAutomation: null,
+  selectedAutomationId: null,
 }
 
 export const getAutomationStore = () => {
@@ -37,49 +39,41 @@ const automationActions = store => ({
       API.getAutomationDefinitions(),
     ])
     store.update(state => {
-      let selected = state.selectedAutomation?.automation
       state.automations = responses[0]
+      state.automations.sort((a, b) => {
+        return a.name < b.name ? -1 : 1
+      })
       state.blockDefinitions = {
         TRIGGER: responses[1].trigger,
         ACTION: responses[1].action,
       }
-      // If previously selected find the new obj and select it
-      if (selected) {
-        selected = responses[0].filter(
-          automation => automation._id === selected._id
-        )
-        state.selectedAutomation = new Automation(selected[0])
-      }
       return state
     })
   },
-  create: async ({ name }) => {
+  create: async (name, trigger) => {
     const automation = {
       name,
       type: "automation",
       definition: {
         steps: [],
+        trigger,
       },
     }
-    const response = await API.createAutomation(automation)
-    store.update(state => {
-      state.automations = [...state.automations, response.automation]
-      store.actions.select(response.automation)
-      return state
-    })
+    const response = await store.actions.save(automation)
+    await store.actions.fetch()
+    store.actions.select(response._id)
+    return response
   },
   duplicate: async automation => {
-    const response = await API.createAutomation({
+    const response = await store.actions.save({
       ...automation,
       name: `${automation.name} - copy`,
       _id: undefined,
       _ref: undefined,
     })
-    store.update(state => {
-      state.automations = [...state.automations, response.automation]
-      store.actions.select(response.automation)
-      return state
-    })
+    await store.actions.fetch()
+    store.actions.select(response._id)
+    return response
   },
   save: async automation => {
     const response = await API.updateAutomation(automation)
@@ -90,11 +84,13 @@ const automationActions = store => ({
       )
       if (existingIdx !== -1) {
         state.automations.splice(existingIdx, 1, updatedAutomation)
-        state.automations = [...state.automations]
-        store.actions.select(updatedAutomation)
         return state
+      } else {
+        state.automations = [...state.automations, updatedAutomation]
       }
+      return state
     })
+    return response.automation
   },
   delete: async automation => {
     await API.deleteAutomation({
@@ -102,34 +98,83 @@ const automationActions = store => ({
       automationRev: automation?._rev,
     })
     store.update(state => {
-      const existingIdx = state.automations.findIndex(
-        existing => existing._id === automation?._id
+      // Remove the automation
+      state.automations = state.automations.filter(
+        x => x._id !== automation._id
       )
-      state.automations.splice(existingIdx, 1)
-      state.automations = [...state.automations]
-      state.selectedAutomation = null
-      state.selectedBlock = null
+      // Select a new automation if required
+      if (automation._id === state.selectedAutomationId) {
+        store.actions.select(state.automations[0]?._id)
+      }
       return state
     })
+    await store.actions.fetch()
+  },
+  updateBlockInputs: async (block, data) => {
+    // Create new modified block
+    let newBlock = {
+      ...block,
+      inputs: {
+        ...block.inputs,
+        ...data,
+      },
+    }
+
+    // Remove any nullish or empty string values
+    Object.keys(newBlock.inputs).forEach(key => {
+      const val = newBlock.inputs[key]
+      if (val == null || val === "") {
+        delete newBlock.inputs[key]
+      }
+    })
+
+    // Create new modified automation
+    const automation = get(selectedAutomation)
+    const newAutomation = store.actions.getUpdatedDefinition(
+      automation,
+      newBlock
+    )
+
+    // Don't save if no changes were made
+    if (JSON.stringify(newAutomation) === JSON.stringify(automation)) {
+      return
+    }
+    await store.actions.save(newAutomation)
   },
   test: async (automation, testData) => {
-    store.update(state => {
-      state.selectedAutomation.testResults = null
-      return state
-    })
     const result = await API.testAutomation({
       automationId: automation?._id,
       testData,
     })
+    if (!result?.trigger && !result?.steps?.length) {
+      throw "Something went wrong testing your automation"
+    }
     store.update(state => {
-      state.selectedAutomation.testResults = result
+      state.testResults = result
       return state
     })
   },
-  select: automation => {
+  getDefinition: id => {
+    return get(store).automations?.find(x => x._id === id)
+  },
+  getUpdatedDefinition: (automation, block) => {
+    let newAutomation = cloneDeep(automation)
+    if (automation.definition.trigger?.id === block.id) {
+      newAutomation.definition.trigger = block
+    } else {
+      const idx = automation.definition.steps.findIndex(x => x.id === block.id)
+      newAutomation.definition.steps.splice(idx, 1, block)
+    }
+    return newAutomation
+  },
+  select: id => {
+    if (!id || id === get(store).selectedAutomationId) {
+      return
+    }
     store.update(state => {
-      state.selectedAutomation = new Automation(cloneDeep(automation))
-      state.selectedBlock = null
+      state.selectedAutomationId = id
+      state.testResults = null
+      state.showTestPanel = false
       return state
     })
   },
@@ -147,48 +192,57 @@ const automationActions = store => ({
       appId,
     })
   },
-  addTestDataToAutomation: data => {
-    store.update(state => {
-      state.selectedAutomation.addTestData(data)
-      return state
-    })
+  addTestDataToAutomation: async data => {
+    let newAutomation = cloneDeep(get(selectedAutomation))
+    newAutomation.testData = {
+      ...newAutomation.testData,
+      ...data,
+    }
+    await store.actions.save(newAutomation)
   },
-  addBlockToAutomation: (block, blockIdx) => {
-    store.update(state => {
-      state.selectedBlock = state.selectedAutomation.addBlock(
-        cloneDeep(block),
-        blockIdx
-      )
-      return state
-    })
+  constructBlock(type, stepId, blockDefinition) {
+    return {
+      ...blockDefinition,
+      inputs: blockDefinition.inputs || {},
+      stepId,
+      type,
+      id: generate(),
+    }
   },
-  toggleFieldControl: value => {
-    store.update(state => {
-      state.selectedBlock.rowControl = value
-      return state
-    })
+  addBlockToAutomation: async (block, blockIdx) => {
+    const automation = get(selectedAutomation)
+    let newAutomation = cloneDeep(automation)
+    if (!automation) {
+      return
+    }
+    newAutomation.definition.steps.splice(blockIdx, 0, block)
+    await store.actions.save(newAutomation)
   },
-  deleteAutomationBlock: block => {
-    store.update(state => {
-      const idx =
-        state.selectedAutomation.automation.definition.steps.findIndex(
-          x => x.id === block.id
-        )
-      state.selectedAutomation.deleteBlock(block.id)
+  /**
+   * "rowControl" appears to be the name of the flag used to determine whether
+   * a certain automation block uses values or bindings as inputs
+   */
+  toggleRowControl: async (block, rowControl) => {
+    const newBlock = { ...block, rowControl }
+    const newAutomation = store.actions.getUpdatedDefinition(
+      get(selectedAutomation),
+      newBlock
+    )
+    await store.actions.save(newAutomation)
+  },
+  deleteAutomationBlock: async block => {
+    const automation = get(selectedAutomation)
+    let newAutomation = cloneDeep(automation)
 
-      // Select next closest step
-      const steps = state.selectedAutomation.automation.definition.steps
-      let nextSelectedBlock
-      if (steps[idx] != null) {
-        nextSelectedBlock = steps[idx]
-      } else if (steps[idx - 1] != null) {
-        nextSelectedBlock = steps[idx - 1]
-      } else {
-        nextSelectedBlock =
-          state.selectedAutomation.automation.definition.trigger || null
-      }
-      state.selectedBlock = nextSelectedBlock
-      return state
-    })
+    // Delete trigger if required
+    if (newAutomation.definition.trigger?.id === block.id) {
+      delete newAutomation.definition.trigger
+    } else {
+      // Otherwise remove step
+      newAutomation.definition.steps = newAutomation.definition.steps.filter(
+        step => step.id !== block.id
+      )
+    }
+    await store.actions.save(newAutomation)
   },
 })

packages/builder/src/builderStore/store/automation/tests/Automation.spec.js

@@ -1,48 +0,0 @@
-import Automation from "../Automation"
-import TEST_AUTOMATION from "./testAutomation"
-
-const TEST_BLOCK = {
-  id: "AUXJQGZY7",
-  name: "Delay",
-  icon: "ri-time-fill",
-  tagline: "Delay for <b>{{time}}</b> milliseconds",
-  description: "Delay the automation until an amount of time has passed.",
-  params: { time: "number" },
-  type: "LOGIC",
-  args: { time: "5000" },
-  stepId: "DELAY",
-}
-
-describe("Automation Data Object", () => {
-  let automation
-
-  beforeEach(() => {
-    automation = new Automation({ ...TEST_AUTOMATION })
-  })
-
-  it("adds a automation block to the automation", () => {
-    automation.addBlock(TEST_BLOCK)
-    expect(automation.automation.definition)
-  })
-
-  it("updates a automation block with new attributes", () => {
-    const firstBlock = automation.automation.definition.steps[0]
-    const updatedBlock = {
-      ...firstBlock,
-      name: "UPDATED",
-    }
-    automation.updateBlock(updatedBlock, firstBlock.id)
-    expect(automation.automation.definition.steps[0]).toEqual(updatedBlock)
-  })
-
-  it("deletes a automation block successfully", () => {
-    const { steps } = automation.automation.definition
-    const originalLength = steps.length
-
-    const lastBlock = steps[steps.length - 1]
-    automation.deleteBlock(lastBlock.id)
-    expect(automation.automation.definition.steps.length).toBeLessThan(
-      originalLength
-    )
-  })
-})

packages/builder/src/builderStore/store/automation/tests/testAutomation.js

@@ -1,78 +0,0 @@
-export default {
-  name: "Test automation",
-  definition: {
-    steps: [
-      {
-        id: "ANBDINAPS",
-        description: "Send an email.",
-        tagline: "Send email to <b>{{to}}</b>",
-        icon: "ri-mail-open-fill",
-        name: "Send Email",
-        params: {
-          to: "string",
-          from: "string",
-          subject: "longText",
-          text: "longText",
-        },
-        type: "ACTION",
-        args: {
-          text: "A user was created!",
-          subject: "New Budibase User",
-          from: "budimaster@budibase.com",
-          to: "test@test.com",
-        },
-        stepId: "SEND_EMAIL",
-      },
-    ],
-    trigger: {
-      id: "iRzYMOqND",
-      name: "Row Saved",
-      event: "row:save",
-      icon: "ri-save-line",
-      tagline: "Row is added to <b>{{table.name}}</b>",
-      description: "Fired when a row is saved to your database.",
-      params: { table: "table" },
-      type: "TRIGGER",
-      args: {
-        table: {
-          type: "table",
-          views: {},
-          name: "users",
-          schema: {
-            name: {
-              type: "string",
-              constraints: {
-                type: "string",
-                length: { maximum: 123 },
-                presence: { allowEmpty: false },
-              },
-              name: "name",
-            },
-            age: {
-              type: "number",
-              constraints: {
-                type: "number",
-                presence: { allowEmpty: false },
-                numericality: {
-                  greaterThanOrEqualTo: "",
-                  lessThanOrEqualTo: "",
-                },
-              },
-              name: "age",
-            },
-          },
-          _id: "c6b4e610cd984b588837bca27188a451",
-          _rev: "7-b8aa1ce0b53e88928bb88fc11bdc0aff",
-        },
-      },
-      stepId: "ROW_SAVED",
-    },
-  },
-  type: "automation",
-  ok: true,
-  id: "b384f861f4754e1693835324a7fcca62",
-  rev: "1-aa1c2cbd868ef02e26f8fad531dd7e37",
-  live: false,
-  _id: "b384f861f4754e1693835324a7fcca62",
-  _rev: "108-4116829ec375e0481d0ecab9e83a2caf",
-}

packages/builder/src/builderStore/store/frontend.js

@@ -1,6 +1,11 @@
 import { get, writable } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
-import { selectedScreen, selectedComponent } from "builderStore"
+import {
+  selectedScreen,
+  selectedComponent,
+  screenHistoryStore,
+  automationHistoryStore,
+} from "builderStore"
 import {
   datasources,
   integrations,
@@ -122,6 +127,8 @@ export const getFrontendStore = () => {
         navigation: application.navigation || {},
         usedPlugins: application.usedPlugins || [],
       }))
+      screenHistoryStore.reset()
+      automationHistoryStore.reset()
 
       // Initialise backend stores
       database.set(application.instance)
@@ -179,10 +186,7 @@ export const getFrontendStore = () => {
         }
 
         // Check screen isn't already selected
-        if (
-          state.selectedScreenId === screen._id &&
-          state.selectedComponentId === screen.props?._id
-        ) {
+        if (state.selectedScreenId === screen._id) {
           return
         }
 
@@ -256,7 +260,7 @@ export const getFrontendStore = () => {
         }
       },
       save: async screen => {
-        /* 
+        /*
           Temporarily disabled to accomodate migration issues.
           store.actions.screens.validate(screen)
         */
@@ -347,6 +351,7 @@ export const getFrontendStore = () => {
 
           return state
         })
+        return null
       },
       updateSetting: async (screen, name, value) => {
         if (!screen || !name) {

packages/builder/src/builderStore/store/history.js

@@ -0,0 +1,319 @@
+import * as jsonpatch from "fast-json-patch/index.mjs"
+import { writable, derived, get } from "svelte/store"
+
+const Operations = {
+  Add: "Add",
+  Delete: "Delete",
+  Change: "Change",
+}
+
+const initialState = {
+  history: [],
+  position: 0,
+  loading: false,
+}
+
+export const createHistoryStore = ({
+  getDoc,
+  selectDoc,
+  beforeAction,
+  afterAction,
+}) => {
+  // Use a derived store to check if we are able to undo or redo any operations
+  const store = writable(initialState)
+  const derivedStore = derived(store, $store => {
+    return {
+      ...$store,
+      canUndo: $store.position > 0,
+      canRedo: $store.position < $store.history.length,
+    }
+  })
+
+  // Wrapped versions of essential functions which we call ourselves when using
+  // undo and redo
+  let saveFn
+  let deleteFn
+
+  /**
+   * Internal util to set the loading flag
+   */
+  const startLoading = () => {
+    store.update(state => {
+      state.loading = true
+      return state
+    })
+  }
+
+  /**
+   * Internal util to unset the loading flag
+   */
+  const stopLoading = () => {
+    store.update(state => {
+      state.loading = false
+      return state
+    })
+  }
+
+  /**
+   * Resets history state
+   */
+  const reset = () => {
+    store.set(initialState)
+  }
+
+  /**
+   * Adds or updates an operation in history.
+   * For internal use only.
+   * @param operation the operation to save
+   */
+  const saveOperation = operation => {
+    store.update(state => {
+      // Update history
+      let history = state.history
+      let position = state.position
+      if (!operation.id) {
+        // Every time a new operation occurs we discard any redo potential
+        operation.id = Math.random()
+        history = [...history.slice(0, state.position), operation]
+        position += 1
+      } else {
+        // If this is a redo/undo of an existing operation, just update history
+        // to replace the doc object as revisions may have changed
+        const idx = history.findIndex(op => op.id === operation.id)
+        history[idx].doc = operation.doc
+      }
+      return { history, position }
+    })
+  }
+
+  /**
+   * Wraps the save function, which asynchronously updates a doc.
+   * The returned function is an enriched version of the real save function so
+   * that we can control history.
+   * @param fn the save function
+   * @returns {function} a wrapped version of the save function
+   */
+  const wrapSaveDoc = fn => {
+    saveFn = async (doc, operationId) => {
+      // Only works on a single doc at a time
+      if (!doc || Array.isArray(doc)) {
+        return
+      }
+      startLoading()
+      try {
+        const oldDoc = getDoc(doc._id)
+        const newDoc = jsonpatch.deepClone(await fn(doc))
+
+        // Store the change
+        if (!oldDoc) {
+          // If no old doc, this is an add operation
+          saveOperation({
+            type: Operations.Add,
+            doc: newDoc,
+            id: operationId,
+          })
+        } else {
+          // Otherwise this is a change operation
+          saveOperation({
+            type: Operations.Change,
+            forwardPatch: jsonpatch.compare(oldDoc, doc),
+            backwardsPatch: jsonpatch.compare(doc, oldDoc),
+            doc: newDoc,
+            id: operationId,
+          })
+        }
+        stopLoading()
+        return newDoc
+      } catch (error) {
+        // We want to allow errors to propagate up to normal handlers, but we
+        // want to stop loading first
+        stopLoading()
+        throw error
+      }
+    }
+    return saveFn
+  }
+
+  /**
+   * Wraps the delete function, which asynchronously deletes a doc.
+   * The returned function is an enriched version of the real delete function so
+   * that we can control history.
+   * @param fn the delete function
+   * @returns {function} a wrapped version of the delete function
+   */
+  const wrapDeleteDoc = fn => {
+    deleteFn = async (doc, operationId) => {
+      // Only works on a single doc at a time
+      if (!doc || Array.isArray(doc)) {
+        return
+      }
+      startLoading()
+      try {
+        const oldDoc = jsonpatch.deepClone(doc)
+        await fn(doc)
+        saveOperation({
+          type: Operations.Delete,
+          doc: oldDoc,
+          id: operationId,
+        })
+        stopLoading()
+      } catch (error) {
+        // We want to allow errors to propagate up to normal handlers, but we
+        // want to stop loading first
+        stopLoading()
+        throw error
+      }
+    }
+    return deleteFn
+  }
+
+  /**
+   * Asynchronously undoes the previous operation.
+   * Optionally selects the changed document so that changes are visible.
+   * @returns {Promise<void>}
+   */
+  const undo = async () => {
+    // Sanity checks
+    const { canUndo, history, position, loading } = get(derivedStore)
+    if (!canUndo || loading) {
+      return
+    }
+    const operation = history[position - 1]
+    if (!operation) {
+      return
+    }
+    startLoading()
+
+    // Before hook
+    await beforeAction?.(operation)
+
+    // Update state immediately to prevent further clicks and to prevent bad
+    // history in the event of an update failing
+    store.update(state => {
+      return {
+        ...state,
+        position: state.position - 1,
+      }
+    })
+
+    // Undo the operation
+    try {
+      // Undo ADD
+      if (operation.type === Operations.Add) {
+        // Try to get the latest doc version to delete
+        const latestDoc = getDoc(operation.doc._id)
+        const doc = latestDoc || operation.doc
+        await deleteFn(doc, operation.id)
+      }
+
+      // Undo DELETE
+      else if (operation.type === Operations.Delete) {
+        // Delete the _rev from the deleted doc so that we can save it as a new
+        // doc again without conflicts
+        let doc = jsonpatch.deepClone(operation.doc)
+        delete doc._rev
+        const created = await saveFn(doc, operation.id)
+        selectDoc?.(created?._id || doc._id)
+      }
+
+      // Undo CHANGE
+      else {
+        // Get the current doc and apply the backwards patch on top of it
+        let doc = jsonpatch.deepClone(getDoc(operation.doc._id))
+        if (doc) {
+          jsonpatch.applyPatch(
+            doc,
+            jsonpatch.deepClone(operation.backwardsPatch)
+          )
+          await saveFn(doc, operation.id)
+          selectDoc?.(doc._id)
+        }
+      }
+      stopLoading()
+    } catch (error) {
+      stopLoading()
+      throw error
+    }
+
+    // After hook
+    await afterAction?.(operation)
+  }
+
+  /**
+   * Asynchronously redoes the previous undo.
+   * Optionally selects the changed document so that changes are visible.
+   * @returns {Promise<void>}
+   */
+  const redo = async () => {
+    // Sanity checks
+    const { canRedo, history, position, loading } = get(derivedStore)
+    if (!canRedo || loading) {
+      return
+    }
+    const operation = history[position]
+    if (!operation) {
+      return
+    }
+    startLoading()
+
+    // Before hook
+    await beforeAction?.(operation)
+
+    // Update state immediately to prevent further clicks and to prevent bad
+    // history in the event of an update failing
+    store.update(state => {
+      return {
+        ...state,
+        position: state.position + 1,
+      }
+    })
+
+    // Redo the operation
+    try {
+      // Redo ADD
+      if (operation.type === Operations.Add) {
+        // Delete the _rev from the deleted doc so that we can save it as a new
+        // doc again without conflicts
+        let doc = jsonpatch.deepClone(operation.doc)
+        delete doc._rev
+        const created = await saveFn(doc, operation.id)
+        selectDoc?.(created?._id || doc._id)
+      }
+
+      // Redo DELETE
+      else if (operation.type === Operations.Delete) {
+        // Try to get the latest doc version to delete
+        const latestDoc = getDoc(operation.doc._id)
+        const doc = latestDoc || operation.doc
+        await deleteFn(doc, operation.id)
+      }
+
+      // Redo CHANGE
+      else {
+        // Get the current doc and apply the forwards patch on top of it
+        let doc = jsonpatch.deepClone(getDoc(operation.doc._id))
+        if (doc) {
+          jsonpatch.applyPatch(doc, jsonpatch.deepClone(operation.forwardPatch))
+          await saveFn(doc, operation.id)
+          selectDoc?.(doc._id)
+        }
+      }
+      stopLoading()
+    } catch (error) {
+      stopLoading()
+      throw error
+    }
+
+    // After hook
+    await afterAction?.(operation)
+  }
+
+  return {
+    subscribe: derivedStore.subscribe,
+    wrapSaveDoc,
+    wrapDeleteDoc,
+    reset,
+    undo,
+    redo,
+  }
+}

packages/builder/src/components/automation/AutomationBuilder/AutomationBuilder.svelte

@@ -1,10 +1,10 @@
 <script>
-  import { automationStore } from "builderStore"
+  import { selectedAutomation } from "builderStore"
   import Flowchart from "./FlowChart/FlowChart.svelte"
-
-  $: automation = $automationStore.selectedAutomation?.automation
 </script>
 
-{#if automation}
-  <Flowchart {automation} />
+{#if $selectedAutomation}
+  {#key $selectedAutomation._id}
+    <Flowchart automation={$selectedAutomation} />
+  {/key}
 {/if}

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte

@@ -5,7 +5,6 @@
     Detail,
     Body,
     Icon,
-    Tooltip,
     notifications,
   } from "@budibase/bbui"
   import { automationStore } from "builderStore"
@@ -13,7 +12,6 @@
   import { externalActions } from "./ExternalActions"
 
   export let blockIdx
-  export let blockComplete
 
   const disabled = {
     SEND_EMAIL_SMTP: {
@@ -50,15 +48,12 @@
 
   async function addBlockToAutomation() {
     try {
-      const newBlock = $automationStore.selectedAutomation.constructBlock(
+      const newBlock = automationStore.actions.constructBlock(
         "ACTION",
         actionVal.stepId,
         actionVal
       )
-      automationStore.actions.addBlockToAutomation(newBlock, blockIdx + 1)
-      await automationStore.actions.save(
-        $automationStore.selectedAutomation?.automation
-      )
+      await automationStore.actions.addBlockToAutomation(newBlock, blockIdx + 1)
     } catch (error) {
       notifications.error("Error saving automation")
     }
@@ -66,20 +61,14 @@
 </script>
 
 <ModalContent
-  title="Create Automation"
+  title="Add automation step"
   confirmText="Save"
   size="M"
   disabled={!selectedAction}
-  onConfirm={() => {
-    blockComplete = true
-    addBlockToAutomation()
-  }}
+  onConfirm={addBlockToAutomation}
 >
-  <Body size="XS">Select an app or event.</Body>
-
-  <Layout noPadding>
-    <Body size="S">Apps</Body>
-
+  <Layout noPadding gap="XS">
+    <Detail size="S">Apps</Detail>
     <div class="item-list">
       {#each Object.entries(external) as [idx, action]}
         <div
@@ -95,64 +84,45 @@
               alt="zapier"
             />
             <span class="icon-spacing">
-              <Body size="XS">{idx.charAt(0).toUpperCase() + idx.slice(1)}</Body
-              ></span
-            >
+              <Body size="XS">
+                {idx.charAt(0).toUpperCase() + idx.slice(1)}
+              </Body>
+            </span>
           </div>
         </div>
       {/each}
     </div>
+  </Layout>
 
+  <Layout noPadding gap="XS">
     <Detail size="S">Actions</Detail>
-
     <div class="item-list">
       {#each Object.entries(internal) as [idx, action]}
-        {#if disabled[idx] && disabled[idx].disabled}
-          <Tooltip text={disabled[idx].message} direction="bottom">
-            <div
-              class="item"
-              class:selected={selectedAction === action.name}
-              class:disabled={true}
-              on:click={() => selectAction(action)}
-            >
-              <div class="item-body">
-                <Icon name={action.icon} />
-                <span class="icon-spacing">
-                  <Body size="XS">{action.name}</Body></span
-                >
-              </div>
-            </div>
-          </Tooltip>
-        {:else}
-          <div
-            class="item"
-            class:selected={selectedAction === action.name}
-            on:click={() => selectAction(action)}
-          >
-            <div class="item-body">
-              <Icon name={action.icon} />
-              <span class="icon-spacing">
-                <Body size="XS">{action.name}</Body></span
-              >
-            </div>
+        {@const isDisabled = disabled[idx] && disabled[idx].disabled}
+        <div
+          class="item"
+          class:disabled={isDisabled}
+          class:selected={selectedAction === action.name}
+          on:click={isDisabled ? null : () => selectAction(action)}
+        >
+          <div class="item-body">
+            <Icon name={action.icon} />
+            <Body size="XS">{action.name}</Body>
+            {#if isDisabled}
+              <Icon name="Help" tooltip={disabled[idx].message} />
+            {/if}
           </div>
-        {/if}
+        </div>
       {/each}
     </div>
   </Layout>
 </ModalContent>
 
 <style>
-  .disabled {
-    opacity: 0.3;
-    pointer-events: none;
-  }
-  .icon-spacing {
-    margin-left: var(--spacing-m);
-  }
   .item-body {
     display: flex;
     margin-left: var(--spacing-m);
+    gap: var(--spacing-m);
   }
   .item-list {
     display: grid;
@@ -171,8 +141,15 @@
     box-sizing: border-box;
     border-width: 2px;
   }
-  .item:hover,
+  .item:not(.disabled):hover,
   .selected {
     background: var(--spectrum-alias-background-color-tertiary);
   }
+  .disabled {
+    background: var(--spectrum-global-color-gray-200);
+    color: var(--spectrum-global-color-gray-500);
+  }
+  .disabled :global(.spectrum-Body) {
+    color: var(--spectrum-global-color-gray-600);
+  }
 </style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { automationStore } from "builderStore"
+  import { automationStore, selectedAutomation } from "builderStore"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import FlowItem from "./FlowItem.svelte"
   import TestDataModal from "./TestDataModal.svelte"
@@ -13,27 +13,28 @@
     Modal,
   } from "@budibase/bbui"
   import { ActionStepID } from "constants/backend/automations"
+  import UndoRedoControl from "components/common/UndoRedoControl.svelte"
+  import { automationHistoryStore } from "builderStore"
 
   export let automation
+
   let testDataModal
-  let blocks
   let confirmDeleteDialog
 
-  $: {
-    blocks = []
-    if (automation) {
-      if (automation.definition.trigger) {
-        blocks.push(automation.definition.trigger)
-      }
-      blocks = blocks.concat(automation.definition.steps || [])
+  $: blocks = getBlocks(automation)
+
+  const getBlocks = automation => {
+    let blocks = []
+    if (automation.definition.trigger) {
+      blocks.push(automation.definition.trigger)
     }
+    blocks = blocks.concat(automation.definition.steps || [])
+    return blocks
   }
 
   async function deleteAutomation() {
     try {
-      await automationStore.actions.delete(
-        $automationStore.selectedAutomation?.automation
-      )
+      await automationStore.actions.delete($selectedAutomation)
     } catch (error) {
       notifications.error("Error deleting automation")
     }
@@ -41,20 +42,17 @@
 </script>
 
 <div class="canvas">
-  <div style="float: left; padding-left: var(--spacing-xl);">
+  <div class="header">
     <Heading size="S">{automation.name}</Heading>
-  </div>
-  <div style="float: right; padding-right: var(--spacing-xl);" class="title">
-    <div class="subtitle">
-      <div style="display:flex; align-items: center;">
-        <div class="icon">
-          <Icon
-            on:click={confirmDeleteDialog.show}
-            hoverable
-            size="M"
-            name="DeleteOutline"
-          />
-        </div>
+    <div class="controls">
+      <UndoRedoControl store={automationHistoryStore} />
+      <Icon
+        on:click={confirmDeleteDialog.show}
+        hoverable
+        size="M"
+        name="DeleteOutline"
+      />
+      <div class="buttons">
         <ActionButton
           on:click={() => {
             testDataModal.show()
@@ -62,15 +60,13 @@
           icon="MultipleCheck"
           size="M">Run test</ActionButton
         >
-        <div style="padding-left: var(--spacing-m);">
-          <ActionButton
-            disabled={!$automationStore.selectedAutomation?.testResults}
-            on:click={() => {
-              $automationStore.showTestPanel = true
-            }}
-            size="M">Test Details</ActionButton
-          >
-        </div>
+        <ActionButton
+          disabled={!$automationStore.testResults}
+          on:click={() => {
+            $automationStore.showTestPanel = true
+          }}
+          size="M">Test Details</ActionButton
+        >
       </div>
     </div>
   </div>
@@ -80,7 +76,7 @@
     <div
       class="block"
       animate:flip={{ duration: 500 }}
-      in:fly|local={{ x: 500, duration: 500 }}
+      in:fly={{ x: 500, duration: 500 }}
       out:fly|local={{ x: 500, duration: 500 }}
     >
       {#if block.stepId !== ActionStepID.LOOP}
@@ -105,6 +101,9 @@
 </Modal>
 
 <style>
+  .canvas {
+    padding: var(--spacing-l) var(--spacing-xl);
+  }
   /* Fix for firefox not respecting bottom padding in scrolling containers */
   .canvas > *:last-child {
     padding-bottom: 40px;
@@ -122,18 +121,19 @@
     text-align: left;
   }
 
-  .title {
-    padding-bottom: var(--spacing-xl);
-  }
-
-  .subtitle {
-    padding-bottom: var(--spacing-xl);
+  .header {
     display: flex;
     justify-content: space-between;
     align-items: center;
   }
-  .icon {
-    cursor: pointer;
-    padding-right: var(--spacing-m);
+  .controls,
+  .buttons {
+    display: flex;
+    justify-content: flex-end;
+    align-items: center;
+    gap: var(--spacing-xl);
+  }
+  .buttons {
+    gap: var(--spacing-s);
   }
 </style>