Merge remote-tracking branch 'origin/develop' into feature/app-user-onboarding-ux

.github/workflows/budibase_ci.yml

@@ -11,7 +11,6 @@ on:
     branches:
       - master
       - develop 
-      - release 
  workflow_dispatch:
 
 env:
@@ -20,9 +19,53 @@ env:
   PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
 
 jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Use Node.js 14.x
+      uses: actions/setup-node@v1
+      with:
+        node-version: 14.x
+    - run: yarn
+    - run: yarn lint
+
   build:
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - name: Install Pro
+        run: yarn install:pro $BRANCH $BASE_BRANCH
+      - run: yarn
+      - run: yarn bootstrap
+      - run: yarn build
 
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - name: Install Pro
+        run: yarn install:pro $BRANCH $BASE_BRANCH
+      - run: yarn
+      - run: yarn bootstrap
+      - run: yarn test
+      - uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
+          files: ./packages/server/coverage/clover.xml,./packages/worker/coverage/clover.xml,./packages/backend-core/coverage/clover.xml
+          name: codecov-umbrella
+          verbose: true
+
+  integration-test:
+    runs-on: ubuntu-latest
     services:
       couchdb:
         image: ibmcom/couchdb3
@@ -31,39 +74,18 @@ jobs:
           COUCHDB_USER: budibase
         ports:
           - 4567:5984
-
-    strategy:
-      matrix:
-        node-version: [14.x]
-
     steps:
-    - uses: actions/checkout@v2
+      - uses: actions/checkout@v2
-
+      - name: Use Node.js 14.x
-    - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
-      uses: actions/setup-node@v1
+        with:
-      with:
+          node-version: 14.x
-        node-version: ${{ matrix.node-version }}
+      - name: Install Pro
-
+        run: yarn install:pro $BRANCH $BASE_BRANCH
-    - name: Install Pro
+      - run: yarn
-      run: yarn install:pro $BRANCH $BASE_BRANCH
+      - run: yarn bootstrap
-
+      - run: yarn build
-    - run: yarn
+      - run: |
-    - run: yarn bootstrap
+          cd qa-core
-    - run: yarn lint
+          yarn
-    - run: yarn build
+          yarn api:test:ci
-    - run: yarn test
-      env:
-        CI: true
-        name: Budibase CI
-    - uses: codecov/codecov-action@v1
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
-        files: ./packages/server/coverage/clover.xml,./packages/worker/coverage/clover.xml,./packages/backend-core/coverage/clover.xml
-        name: codecov-umbrella
-        verbose: true 
-
-    - name: QA Core Integration Tests
-      run: | 
-        cd qa-core
-        yarn
-        yarn api:test:ci

.husky/pre-commit

@@ -1,4 +1,2 @@
 #!/bin/sh
 . "$(dirname "$0")/_/husky.sh"
-
-yarn run lint

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.18-alpha.6",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/jest.config.ts

@@ -9,15 +9,9 @@ const baseConfig: Config.InitialProjectOptions = {
   transform: {
     "^.+\\.ts?$": "@swc/jest",
   },
-}
+  moduleNameMapper: {
-
-if (!process.env.CI) {
-  // use sources when not in CI
-  baseConfig.moduleNameMapper = {
     "@budibase/types": "<rootDir>/../types/src",
-  }
+  },
-} else {
-  console.log("Running tests with compiled dependency sources")
 }
 
 const config: Config.InitialOptions = {

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.18-alpha.6",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -18,13 +18,13 @@
     "build:pro": "../../scripts/pro/build.sh",
     "postbuild": "yarn run build:pro",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
-    "test": "jest --coverage --maxWorkers=2",
+    "test": "bash scripts/test.sh",
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
     "@budibase/nano": "10.1.1",
     "@budibase/pouchdb-replication-stream": "1.2.10",
-    "@budibase/types": "2.3.17-alpha.4",
+    "@budibase/types": "2.3.18-alpha.6",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",
@@ -62,7 +62,7 @@
     "@trendyol/jest-testcontainers": "^2.1.1",
     "@types/chance": "1.1.3",
     "@types/ioredis": "4.28.0",
-    "@types/jest": "27.5.1",
+    "@types/jest": "28.1.1",
     "@types/koa": "2.13.4",
     "@types/koa-pino-logger": "3.0.0",
     "@types/lodash": "4.14.180",

packages/backend-core/scripts/test.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [[ -n $CI ]]
+then
+  # --runInBand performs better in ci where resources are limited
+  echo "jest --coverage --runInBand"
+  jest --coverage --runInBand
+else
+  # --maxWorkers performs better in development
+  echo "jest --coverage"
+  jest --coverage
+fi

packages/backend-core/src/accounts/accounts.ts

@@ -1,13 +1,24 @@
 import API from "./api"
 import env from "../environment"
 import { Header } from "../constants"
-import { CloudAccount } from "@budibase/types"
+import { CloudAccount, HealthStatusResponse } from "@budibase/types"
 
 const api = new API(env.ACCOUNT_PORTAL_URL)
 
+/**
+ * This client is intended to be used in a cloud hosted deploy only.
+ * Rather than relying on each consumer to perform the necessary environmental checks
+ * we use the following check to exit early with a undefined response which should be
+ * handled by the caller.
+ */
+const EXIT_EARLY = env.SELF_HOSTED || env.DISABLE_ACCOUNT_PORTAL
+
 export const getAccount = async (
   email: string
 ): Promise<CloudAccount | undefined> => {
+  if (EXIT_EARLY) {
+    return
+  }
   const payload = {
     email,
   }
@@ -29,6 +40,9 @@ export const getAccount = async (
 export const getAccountByTenantId = async (
   tenantId: string
 ): Promise<CloudAccount | undefined> => {
+  if (EXIT_EARLY) {
+    return
+  }
   const payload = {
     tenantId,
   }
@@ -47,7 +61,12 @@ export const getAccountByTenantId = async (
   return json[0]
 }
 
-export const getStatus = async () => {
+export const getStatus = async (): Promise<
+  HealthStatusResponse | undefined
+> => {
+  if (EXIT_EARLY) {
+    return
+  }
   const response = await api.get(`/api/status`, {
     headers: {
       [Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,

packages/backend-core/src/accounts/index.ts

@@ -0,0 +1 @@
+export * from "./accounts"

packages/backend-core/src/auth/auth.ts

@@ -1,10 +1,11 @@
 const _passport = require("koa-passport")
 const LocalStrategy = require("passport-local").Strategy
 const JwtStrategy = require("passport-jwt").Strategy
-import { getGlobalDB } from "../tenancy"
+import { getGlobalDB } from "../context"
 const refresh = require("passport-oauth2-refresh")
-import { Config } from "../constants"
+import { Config, Cookie } from "../constants"
 import { getScopedConfig } from "../db"
+import { getSessionsForUser, invalidateSessions } from "../security/sessions"
 import {
   jwt as jwtPassport,
   local,
@@ -15,8 +16,11 @@ import {
   google,
 } from "../middleware"
 import { invalidateUser } from "../cache/user"
-import { User } from "@budibase/types"
+import { PlatformLogoutOpts, User } from "@budibase/types"
 import { logAlert } from "../logging"
+import * as events from "../events"
+import * as userCache from "../cache/user"
+import { clearCookie, getCookie } from "../utils"
 export {
   auditLog,
   authError,
@@ -29,6 +33,7 @@ export {
   google,
   oidc,
 } from "../middleware"
+import { ssoSaveUserNoOp } from "../middleware/passport/sso/sso"
 export const buildAuthMiddleware = authenticated
 export const buildTenancyMiddleware = tenancy
 export const buildCsrfMiddleware = csrf
@@ -71,7 +76,7 @@ async function refreshOIDCAccessToken(
     if (!enrichedConfig) {
       throw new Error("OIDC Config contents invalid")
     }
-    strategy = await oidc.strategyFactory(enrichedConfig)
+    strategy = await oidc.strategyFactory(enrichedConfig, ssoSaveUserNoOp)
   } catch (err) {
     console.error(err)
     throw new Error("Could not refresh OAuth Token")
@@ -103,7 +108,11 @@ async function refreshGoogleAccessToken(
 
   let strategy
   try {
-    strategy = await google.strategyFactory(config, callbackUrl)
+    strategy = await google.strategyFactory(
+      config,
+      callbackUrl,
+      ssoSaveUserNoOp
+    )
   } catch (err: any) {
     console.error(err)
     throw new Error(
@@ -161,6 +170,8 @@ export async function refreshOAuthToken(
   return refreshResponse
 }
 
+// TODO: Refactor to use user save function instead to prevent the need for
+// manually saving and invalidating on callback
 export async function updateUserOAuth(userId: string, oAuthConfig: any) {
   const details = {
     accessToken: oAuthConfig.accessToken,
@@ -188,3 +199,32 @@ export async function updateUserOAuth(userId: string, oAuthConfig: any) {
     console.error("Could not update OAuth details for current user", e)
   }
 }
+
+/**
+ * Logs a user out from budibase. Re-used across account portal and builder.
+ */
+export async function platformLogout(opts: PlatformLogoutOpts) {
+  const ctx = opts.ctx
+  const userId = opts.userId
+  const keepActiveSession = opts.keepActiveSession
+
+  if (!ctx) throw new Error("Koa context must be supplied to logout.")
+
+  const currentSession = getCookie(ctx, Cookie.Auth)
+  let sessions = await getSessionsForUser(userId)
+
+  if (keepActiveSession) {
+    sessions = sessions.filter(
+      session => session.sessionId !== currentSession.sessionId
+    )
+  } else {
+    // clear cookies
+    clearCookie(ctx, Cookie.Auth)
+    clearCookie(ctx, Cookie.CurrentApp)
+  }
+
+  const sessionIds = sessions.map(({ sessionId }) => sessionId)
+  await invalidateSessions(userId, { sessionIds, reason: "logout" })
+  await events.auth.logout()
+  await userCache.invalidateUser(userId)
+}

packages/backend-core/src/auth/tests/auth.spec.ts

@@ -0,0 +1,13 @@
+import { structures, testEnv } from "../../../tests"
+import * as auth from "../auth"
+import * as events from "../../events"
+
+describe("platformLogout", () => {
+  it("should call platform logout", async () => {
+    await testEnv.withTenant(async () => {
+      const ctx = structures.koa.newContext()
+      await auth.platformLogout({ ctx, userId: "test" })
+      expect(events.auth.logout).toBeCalledTimes(1)
+    })
+  })
+})

packages/backend-core/src/cache/tests/writethrough.spec.js

@@ -1,61 +0,0 @@
-require("../../../tests")
-const { Writethrough } = require("../writethrough")
-const { getDB } = require("../../db")
-const tk = require("timekeeper")
-const { structures } = require("../../../tests")
-
-const START_DATE = Date.now()
-tk.freeze(START_DATE)
-
-
-const DELAY = 5000
-
-const db = getDB(structures.db.id())
-const db2 = getDB(structures.db.id())
-const writethrough = new Writethrough(db, DELAY), writethrough2 = new Writethrough(db2, DELAY)
-
-describe("writethrough", () => {
-  describe("put", () => {
-    let first
-    it("should be able to store, will go to DB", async () => {
-      const response = await writethrough.put({ _id: "test", value: 1 })
-      const output = await db.get(response.id)
-      first = output
-      expect(output.value).toBe(1)
-    })
-
-    it("second put shouldn't update DB", async () => {
-      const response = await writethrough.put({ ...first, value: 2 })
-      const output = await db.get(response.id)
-      expect(first._rev).toBe(output._rev)
-      expect(output.value).toBe(1)
-    })
-
-    it("should put it again after delay period", async () => {
-      tk.freeze(START_DATE + DELAY + 1)
-      const response = await writethrough.put({ ...first, value: 3 })
-      const output = await db.get(response.id)
-      expect(response.rev).not.toBe(first._rev)
-      expect(output.value).toBe(3)
-    })
-  })
-
-  describe("get", () => {
-    it("should be able to retrieve", async () => {
-      const response = await writethrough.get("test")
-      expect(response.value).toBe(3)
-    })
-  })
-
-  describe("same doc, different databases (tenancy)", () => {
-    it("should be able to two different databases", async () => {
-      const resp1 = await writethrough.put({ _id: "db1", value: "first" })
-      const resp2 = await writethrough2.put({ _id: "db1", value: "second" })
-      expect(resp1.rev).toBeDefined()
-      expect(resp2.rev).toBeDefined()
-      expect((await db.get("db1")).value).toBe("first")
-      expect((await db2.get("db1")).value).toBe("second")
-    })
-  })
-})
-

packages/backend-core/src/cache/tests/writethrough.spec.ts

@@ -0,0 +1,73 @@
+import { structures, DBTestConfiguration } from "../../../tests"
+import { Writethrough } from "../writethrough"
+import { getDB } from "../../db"
+import tk from "timekeeper"
+
+const START_DATE = Date.now()
+tk.freeze(START_DATE)
+
+const DELAY = 5000
+
+describe("writethrough", () => {
+  const config = new DBTestConfiguration()
+
+  const db = getDB(structures.db.id())
+  const db2 = getDB(structures.db.id())
+
+  const writethrough = new Writethrough(db, DELAY)
+  const writethrough2 = new Writethrough(db2, DELAY)
+
+  describe("put", () => {
+    let first: any
+
+    it("should be able to store, will go to DB", async () => {
+      await config.doInTenant(async () => {
+        const response = await writethrough.put({ _id: "test", value: 1 })
+        const output = await db.get(response.id)
+        first = output
+        expect(output.value).toBe(1)
+      })
+    })
+
+    it("second put shouldn't update DB", async () => {
+      await config.doInTenant(async () => {
+        const response = await writethrough.put({ ...first, value: 2 })
+        const output = await db.get(response.id)
+        expect(first._rev).toBe(output._rev)
+        expect(output.value).toBe(1)
+      })
+    })
+
+    it("should put it again after delay period", async () => {
+      await config.doInTenant(async () => {
+        tk.freeze(START_DATE + DELAY + 1)
+        const response = await writethrough.put({ ...first, value: 3 })
+        const output = await db.get(response.id)
+        expect(response.rev).not.toBe(first._rev)
+        expect(output.value).toBe(3)
+      })
+    })
+  })
+
+  describe("get", () => {
+    it("should be able to retrieve", async () => {
+      await config.doInTenant(async () => {
+        const response = await writethrough.get("test")
+        expect(response.value).toBe(3)
+      })
+    })
+  })
+
+  describe("same doc, different databases (tenancy)", () => {
+    it("should be able to two different databases", async () => {
+      await config.doInTenant(async () => {
+        const resp1 = await writethrough.put({ _id: "db1", value: "first" })
+        const resp2 = await writethrough2.put({ _id: "db1", value: "second" })
+        expect(resp1.rev).toBeDefined()
+        expect(resp2.rev).toBeDefined()
+        expect((await db.get("db1")).value).toBe("first")
+        expect((await db2.get("db1")).value).toBe("second")
+      })
+    })
+  })
+})

packages/backend-core/src/cache/user.ts

@@ -1,8 +1,9 @@
 import * as redis from "../redis/init"
-import { getTenantId, lookupTenantId, doWithGlobalDB } from "../tenancy"
+import * as tenancy from "../tenancy"
+import * as context from "../context"
+import * as platform from "../platform"
 import env from "../environment"
-import * as accounts from "../cloud/accounts"
+import * as accounts from "../accounts"
-import { Database } from "@budibase/types"
 
 const EXPIRY_SECONDS = 3600
 
@@ -10,7 +11,8 @@ const EXPIRY_SECONDS = 3600
  * The default populate user function
  */
 async function populateFromDB(userId: string, tenantId: string) {
-  const user = await doWithGlobalDB(tenantId, (db: Database) => db.get(userId))
+  const db = tenancy.getTenantDB(tenantId)
+  const user = await db.get(userId)
   user.budibaseAccess = true
   if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     const account = await accounts.getAccount(user.email)
@@ -42,9 +44,9 @@ export async function getUser(
   }
   if (!tenantId) {
     try {
-      tenantId = getTenantId()
+      tenantId = context.getTenantId()
     } catch (err) {
-      tenantId = await lookupTenantId(userId)
+      tenantId = await platform.users.lookupTenantId(userId)
     }
   }
   const client = await redis.getUserClient()

packages/backend-core/src/context/deprovision.ts

@@ -1,108 +0,0 @@
-import {
-  getGlobalUserParams,
-  getAllApps,
-  doWithDB,
-  StaticDatabases,
-} from "../db"
-import { doWithGlobalDB } from "../tenancy"
-import { App, Tenants, User, Database } from "@budibase/types"
-
-const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
-const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
-
-async function removeTenantFromInfoDB(tenantId: string) {
-  try {
-    await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => {
-      const tenants = (await infoDb.get(TENANT_DOC)) as Tenants
-      tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)
-
-      await infoDb.put(tenants)
-    })
-  } catch (err) {
-    console.error(`Error removing tenant ${tenantId} from info db`, err)
-    throw err
-  }
-}
-
-export async function removeUserFromInfoDB(dbUser: User) {
-  await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => {
-    const keys = [dbUser._id!, dbUser.email]
-    const userDocs = await infoDb.allDocs({
-      keys,
-      include_docs: true,
-    })
-    const toDelete = userDocs.rows.map((row: any) => {
-      return {
-        ...row.doc,
-        _deleted: true,
-      }
-    })
-    await infoDb.bulkDocs(toDelete)
-  })
-}
-
-async function removeUsersFromInfoDB(tenantId: string) {
-  return doWithGlobalDB(tenantId, async (db: any) => {
-    try {
-      const allUsers = await db.allDocs(
-        getGlobalUserParams(null, {
-          include_docs: true,
-        })
-      )
-      await doWithDB(PLATFORM_INFO_DB, async (infoDb: any) => {
-        const allEmails = allUsers.rows.map((row: any) => row.doc.email)
-        // get the id docs
-        let keys = allUsers.rows.map((row: any) => row.id)
-        // and the email docs
-        keys = keys.concat(allEmails)
-        // retrieve the docs and delete them
-        const userDocs = await infoDb.allDocs({
-          keys,
-          include_docs: true,
-        })
-        const toDelete = userDocs.rows.map((row: any) => {
-          return {
-            ...row.doc,
-            _deleted: true,
-          }
-        })
-        await infoDb.bulkDocs(toDelete)
-      })
-    } catch (err) {
-      console.error(`Error removing tenant ${tenantId} users from info db`, err)
-      throw err
-    }
-  })
-}
-
-async function removeGlobalDB(tenantId: string) {
-  return doWithGlobalDB(tenantId, async (db: Database) => {
-    try {
-      await db.destroy()
-    } catch (err) {
-      console.error(`Error removing tenant ${tenantId} users from info db`, err)
-      throw err
-    }
-  })
-}
-
-async function removeTenantApps(tenantId: string) {
-  try {
-    const apps = (await getAllApps({ all: true })) as App[]
-    const destroyPromises = apps.map(app =>
-      doWithDB(app.appId, (db: Database) => db.destroy())
-    )
-    await Promise.allSettled(destroyPromises)
-  } catch (err) {
-    console.error(`Error removing tenant ${tenantId} apps`, err)
-    throw err
-  }
-}
-
-// can't live in tenancy package due to circular dependency on db/utils
-export async function deleteTenant(tenantId: string) {
-  await removeTenantFromInfoDB(tenantId)
-  await removeUsersFromInfoDB(tenantId)
-  await removeGlobalDB(tenantId)
-  await removeTenantApps(tenantId)
-}

packages/backend-core/src/context/tests/index.spec.ts

@@ -1,11 +1,14 @@
-require("../../../tests")
+import { testEnv } from "../../../tests"
 const context = require("../")
 const { DEFAULT_TENANT_ID } = require("../../constants")
-import env from "../../environment"
 
 describe("context", () => {
   describe("doInTenant", () => {
     describe("single-tenancy", () => {
+      beforeAll(() => {
+        testEnv.singleTenant()
+      })
+
       it("defaults to the default tenant", () => {
         const tenantId = context.getTenantId()
         expect(tenantId).toBe(DEFAULT_TENANT_ID)
@@ -20,8 +23,8 @@ describe("context", () => {
     })
 
     describe("multi-tenancy", () => {
-      beforeEach(() => {
+      beforeAll(() => {
-        env._set("MULTI_TENANCY", 1)
+        testEnv.multiTenant()
       })
 
       it("fails when no tenant id is set", () => {

packages/backend-core/src/db/db.ts

@@ -1,7 +1,6 @@
 import env from "../environment"
-import { directCouchQuery, getPouchDB } from "./couch"
+import { directCouchQuery, DatabaseImpl } from "./couch"
 import { CouchFindOptions, Database } from "@budibase/types"
-import { DatabaseImpl } from "../db"
 
 const dbList = new Set()
 

packages/backend-core/src/db/tests/utils.seq.spec.ts

@@ -1,190 +0,0 @@
-require("../../../tests")
-const {
-  getDevelopmentAppID,
-  getProdAppID,
-  isDevAppID,
-  isProdAppID,
-} = require("../conversions")
-const { generateAppID, getPlatformUrl, getScopedConfig } = require("../utils")
-const tenancy = require("../../tenancy")
-const { Config, DEFAULT_TENANT_ID } = require("../../constants")
-import { generator } from "../../../tests"
-import env from "../../environment"
-
-describe("utils", () => {
-  describe("app ID manipulation", () => {
-    function getID() {
-      const appId = generateAppID()
-      const split = appId.split("_")
-      const uuid = split[split.length - 1]
-      const devAppId = `app_dev_${uuid}`
-      return { appId, devAppId, split, uuid }
-    }
-
-    it("should be able to generate a new app ID", () => {
-      expect(generateAppID().startsWith("app_")).toEqual(true)
-    })
-
-    it("should be able to convert a production app ID to development", () => {
-      const { appId, uuid } = getID()
-      expect(getDevelopmentAppID(appId)).toEqual(`app_dev_${uuid}`)
-    })
-
-    it("should be able to convert a development app ID to development", () => {
-      const { devAppId, uuid } = getID()
-      expect(getDevelopmentAppID(devAppId)).toEqual(`app_dev_${uuid}`)
-    })
-
-    it("should be able to convert a development ID to a production", () => {
-      const { devAppId, uuid } = getID()
-      expect(getProdAppID(devAppId)).toEqual(`app_${uuid}`)
-    })
-
-    it("should be able to convert a production ID to production", () => {
-      const { appId, uuid } = getID()
-      expect(getProdAppID(appId)).toEqual(`app_${uuid}`)
-    })
-
-    it("should be able to confirm dev app ID is development", () => {
-      const { devAppId } = getID()
-      expect(isDevAppID(devAppId)).toEqual(true)
-    })
-
-    it("should be able to confirm prod app ID is not development", () => {
-      const { appId } = getID()
-      expect(isDevAppID(appId)).toEqual(false)
-    })
-
-    it("should be able to confirm prod app ID is prod", () => {
-      const { appId } = getID()
-      expect(isProdAppID(appId)).toEqual(true)
-    })
-
-    it("should be able to confirm dev app ID is not prod", () => {
-      const { devAppId } = getID()
-      expect(isProdAppID(devAppId)).toEqual(false)
-    })
-  })
-})
-
-const DEFAULT_URL = "http://localhost:10000"
-const ENV_URL = "http://env.com"
-
-const setDbPlatformUrl = async (dbUrl: string) => {
-  const db = tenancy.getGlobalDB()
-  await db.put({
-    _id: "config_settings",
-    type: Config.SETTINGS,
-    config: {
-      platformUrl: dbUrl,
-    },
-  })
-}
-
-const clearSettingsConfig = async () => {
-  await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-    const db = tenancy.getGlobalDB()
-    try {
-      const config = await db.get("config_settings")
-      await db.remove("config_settings", config._rev)
-    } catch (e: any) {
-      if (e.status !== 404) {
-        throw e
-      }
-    }
-  })
-}
-
-describe("getPlatformUrl", () => {
-  describe("self host", () => {
-    beforeEach(async () => {
-      env._set("SELF_HOST", 1)
-      await clearSettingsConfig()
-    })
-
-    it("gets the default url", async () => {
-      await tenancy.doInTenant(null, async () => {
-        const url = await getPlatformUrl()
-        expect(url).toBe(DEFAULT_URL)
-      })
-    })
-
-    it("gets the platform url from the environment", async () => {
-      await tenancy.doInTenant(null, async () => {
-        env._set("PLATFORM_URL", ENV_URL)
-        const url = await getPlatformUrl()
-        expect(url).toBe(ENV_URL)
-      })
-    })
-
-    it("gets the platform url from the database", async () => {
-      await tenancy.doInTenant(null, async () => {
-        const dbUrl = generator.url()
-        await setDbPlatformUrl(dbUrl)
-        const url = await getPlatformUrl()
-        expect(url).toBe(dbUrl)
-      })
-    })
-  })
-
-  describe("cloud", () => {
-    const TENANT_AWARE_URL = "http://default.env.com"
-
-    beforeEach(async () => {
-      env._set("SELF_HOSTED", 0)
-      env._set("MULTI_TENANCY", 1)
-      env._set("PLATFORM_URL", ENV_URL)
-      await clearSettingsConfig()
-    })
-
-    it("gets the platform url from the environment without tenancy", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        const url = await getPlatformUrl({ tenantAware: false })
-        expect(url).toBe(ENV_URL)
-      })
-    })
-
-    it("gets the platform url from the environment with tenancy", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        const url = await getPlatformUrl()
-        expect(url).toBe(TENANT_AWARE_URL)
-      })
-    })
-
-    it("never gets the platform url from the database", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        await setDbPlatformUrl(generator.url())
-        const url = await getPlatformUrl()
-        expect(url).toBe(TENANT_AWARE_URL)
-      })
-    })
-  })
-})
-
-describe("getScopedConfig", () => {
-  describe("settings config", () => {
-    beforeEach(async () => {
-      env._set("SELF_HOSTED", 1)
-      env._set("PLATFORM_URL", "")
-      await clearSettingsConfig()
-    })
-
-    it("returns the platform url with an existing config", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        const dbUrl = generator.url()
-        await setDbPlatformUrl(dbUrl)
-        const db = tenancy.getGlobalDB()
-        const config = await getScopedConfig(db, { type: Config.SETTINGS })
-        expect(config.platformUrl).toBe(dbUrl)
-      })
-    })
-
-    it("returns the platform url without an existing config", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        const db = tenancy.getGlobalDB()
-        const config = await getScopedConfig(db, { type: Config.SETTINGS })
-        expect(config.platformUrl).toBe(DEFAULT_URL)
-      })
-    })
-  })
-})

packages/backend-core/src/db/tests/utils.spec.ts

@@ -0,0 +1,192 @@
+import { generator, DBTestConfiguration, testEnv } from "../../../tests"
+import {
+  getDevelopmentAppID,
+  getProdAppID,
+  isDevAppID,
+  isProdAppID,
+} from "../conversions"
+import { generateAppID, getPlatformUrl, getScopedConfig } from "../utils"
+import * as context from "../../context"
+import { Config } from "../../constants"
+import env from "../../environment"
+
+describe("utils", () => {
+  const config = new DBTestConfiguration()
+
+  describe("app ID manipulation", () => {
+    function getID() {
+      const appId = generateAppID()
+      const split = appId.split("_")
+      const uuid = split[split.length - 1]
+      const devAppId = `app_dev_${uuid}`
+      return { appId, devAppId, split, uuid }
+    }
+
+    it("should be able to generate a new app ID", () => {
+      expect(generateAppID().startsWith("app_")).toEqual(true)
+    })
+
+    it("should be able to convert a production app ID to development", () => {
+      const { appId, uuid } = getID()
+      expect(getDevelopmentAppID(appId)).toEqual(`app_dev_${uuid}`)
+    })
+
+    it("should be able to convert a development app ID to development", () => {
+      const { devAppId, uuid } = getID()
+      expect(getDevelopmentAppID(devAppId)).toEqual(`app_dev_${uuid}`)
+    })
+
+    it("should be able to convert a development ID to a production", () => {
+      const { devAppId, uuid } = getID()
+      expect(getProdAppID(devAppId)).toEqual(`app_${uuid}`)
+    })
+
+    it("should be able to convert a production ID to production", () => {
+      const { appId, uuid } = getID()
+      expect(getProdAppID(appId)).toEqual(`app_${uuid}`)
+    })
+
+    it("should be able to confirm dev app ID is development", () => {
+      const { devAppId } = getID()
+      expect(isDevAppID(devAppId)).toEqual(true)
+    })
+
+    it("should be able to confirm prod app ID is not development", () => {
+      const { appId } = getID()
+      expect(isDevAppID(appId)).toEqual(false)
+    })
+
+    it("should be able to confirm prod app ID is prod", () => {
+      const { appId } = getID()
+      expect(isProdAppID(appId)).toEqual(true)
+    })
+
+    it("should be able to confirm dev app ID is not prod", () => {
+      const { devAppId } = getID()
+      expect(isProdAppID(devAppId)).toEqual(false)
+    })
+  })
+
+  const DEFAULT_URL = "http://localhost:10000"
+  const ENV_URL = "http://env.com"
+
+  const setDbPlatformUrl = async (dbUrl: string) => {
+    const db = context.getGlobalDB()
+    await db.put({
+      _id: "config_settings",
+      type: Config.SETTINGS,
+      config: {
+        platformUrl: dbUrl,
+      },
+    })
+  }
+
+  const clearSettingsConfig = async () => {
+    await config.doInTenant(async () => {
+      const db = context.getGlobalDB()
+      try {
+        const config = await db.get("config_settings")
+        await db.remove("config_settings", config._rev)
+      } catch (e: any) {
+        if (e.status !== 404) {
+          throw e
+        }
+      }
+    })
+  }
+
+  describe("getPlatformUrl", () => {
+    describe("self host", () => {
+      beforeEach(async () => {
+        testEnv.selfHosted()
+        await clearSettingsConfig()
+      })
+
+      it("gets the default url", async () => {
+        await config.doInTenant(async () => {
+          const url = await getPlatformUrl()
+          expect(url).toBe(DEFAULT_URL)
+        })
+      })
+
+      it("gets the platform url from the environment", async () => {
+        await config.doInTenant(async () => {
+          env._set("PLATFORM_URL", ENV_URL)
+          const url = await getPlatformUrl()
+          expect(url).toBe(ENV_URL)
+        })
+      })
+
+      it("gets the platform url from the database", async () => {
+        await config.doInTenant(async () => {
+          const dbUrl = generator.url()
+          await setDbPlatformUrl(dbUrl)
+          const url = await getPlatformUrl()
+          expect(url).toBe(dbUrl)
+        })
+      })
+    })
+
+    describe("cloud", () => {
+      const TENANT_AWARE_URL = `http://${config.tenantId}.env.com`
+
+      beforeEach(async () => {
+        testEnv.cloudHosted()
+        testEnv.multiTenant()
+
+        env._set("PLATFORM_URL", ENV_URL)
+        await clearSettingsConfig()
+      })
+
+      it("gets the platform url from the environment without tenancy", async () => {
+        await config.doInTenant(async () => {
+          const url = await getPlatformUrl({ tenantAware: false })
+          expect(url).toBe(ENV_URL)
+        })
+      })
+
+      it("gets the platform url from the environment with tenancy", async () => {
+        await config.doInTenant(async () => {
+          const url = await getPlatformUrl()
+          expect(url).toBe(TENANT_AWARE_URL)
+        })
+      })
+
+      it("never gets the platform url from the database", async () => {
+        await config.doInTenant(async () => {
+          await setDbPlatformUrl(generator.url())
+          const url = await getPlatformUrl()
+          expect(url).toBe(TENANT_AWARE_URL)
+        })
+      })
+    })
+  })
+
+  describe("getScopedConfig", () => {
+    describe("settings config", () => {
+      beforeEach(async () => {
+        env._set("SELF_HOSTED", 1)
+        env._set("PLATFORM_URL", "")
+        await clearSettingsConfig()
+      })
+
+      it("returns the platform url with an existing config", async () => {
+        await config.doInTenant(async () => {
+          const dbUrl = generator.url()
+          await setDbPlatformUrl(dbUrl)
+          const db = context.getGlobalDB()
+          const config = await getScopedConfig(db, { type: Config.SETTINGS })
+          expect(config.platformUrl).toBe(dbUrl)
+        })
+      })
+
+      it("returns the platform url without an existing config", async () => {
+        await config.doInTenant(async () => {
+          const db = context.getGlobalDB()
+          const config = await getScopedConfig(db, { type: Config.SETTINGS })
+          expect(config.platformUrl).toBe(DEFAULT_URL)
+        })
+      })
+    })
+  })
+})

packages/backend-core/src/db/views.ts

@@ -1,13 +1,14 @@
 import {
-  DocumentType,
-  ViewName,
   DeprecatedViews,
+  DocumentType,
   SEPARATOR,
   StaticDatabases,
+  ViewName,
 } from "../constants"
 import { getGlobalDB } from "../context"
 import { doWithDB } from "./"
 import { Database, DatabaseQueryOpts } from "@budibase/types"
+import env from "../environment"
 
 const DESIGN_DB = "_design/database"
 
@@ -69,17 +70,6 @@ export const createNewUserEmailView = async () => {
   await createView(db, viewJs, ViewName.USER_BY_EMAIL)
 }
 
-export const createAccountEmailView = async () => {
-  const viewJs = `function(doc) {
-    if (doc._id.startsWith("${DocumentType.ACCOUNT_METADATA}${SEPARATOR}")) {
-      emit(doc.email.toLowerCase(), doc._id)
-    }
-  }`
-  await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
-    await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL)
-  })
-}
-
 export const createUserAppView = async () => {
   const db = getGlobalDB()
   const viewJs = `function(doc) {
@@ -113,17 +103,6 @@ export const createUserBuildersView = async () => {
   await createView(db, viewJs, ViewName.USER_BY_BUILDERS)
 }
 
-export const createPlatformUserView = async () => {
-  const viewJs = `function(doc) {
-    if (doc.tenantId) {
-      emit(doc._id.toLowerCase(), doc._id)
-    }
-  }`
-  await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
-    await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
-  })
-}
-
 export interface QueryViewOptions {
   arrayResponse?: boolean
 }
@@ -162,13 +141,48 @@ export const queryView = async <T>(
   }
 }
 
+// PLATFORM
+
+async function createPlatformView(viewJs: string, viewName: ViewName) {
+  try {
+    await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
+      await createView(db, viewJs, viewName)
+    })
+  } catch (e: any) {
+    if (e.status === 409 && env.isTest()) {
+      // multiple tests can try to initialise platforms views
+      // at once - safe to exit on conflict
+      return
+    }
+    throw e
+  }
+}
+
+export const createPlatformAccountEmailView = async () => {
+  const viewJs = `function(doc) {
+    if (doc._id.startsWith("${DocumentType.ACCOUNT_METADATA}${SEPARATOR}")) {
+      emit(doc.email.toLowerCase(), doc._id)
+    }
+  }`
+  await createPlatformView(viewJs, ViewName.ACCOUNT_BY_EMAIL)
+}
+
+export const createPlatformUserView = async () => {
+  const viewJs = `function(doc) {
+    if (doc.tenantId) {
+      emit(doc._id.toLowerCase(), doc._id)
+    }
+  }`
+  await createPlatformView(viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
+}
+
 export const queryPlatformView = async <T>(
   viewName: ViewName,
   params: DatabaseQueryOpts,
   opts?: QueryViewOptions
 ): Promise<T[] | T | undefined> => {
   const CreateFuncByName: any = {
-    [ViewName.ACCOUNT_BY_EMAIL]: createAccountEmailView,
+    [ViewName.ACCOUNT_BY_EMAIL]: createPlatformAccountEmailView,
     [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView,
   }
 

packages/backend-core/src/environment.ts

@@ -44,8 +44,9 @@ const environment = {
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
   GOOGLE_CLIENT_SECRET: process.env.GOOGLE_CLIENT_SECRET,
   SALT_ROUNDS: process.env.SALT_ROUNDS,
-  REDIS_URL: process.env.REDIS_URL,
+  REDIS_URL: process.env.REDIS_URL || "localhost:6379",
-  REDIS_PASSWORD: process.env.REDIS_PASSWORD,
+  REDIS_PASSWORD: process.env.REDIS_PASSWORD || "budibase",
+  MOCK_REDIS: process.env.MOCK_REDIS,
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   AWS_REGION: process.env.AWS_REGION,
@@ -82,6 +83,7 @@ const environment = {
   SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD,
   DEPLOYMENT_ENVIRONMENT:
     process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose",
+  ENABLE_4XX_HTTP_LOGGING: process.env.ENABLE_4XX_HTTP_LOGGING || true,
   _set(key: any, value: any) {
     process.env[key] = value
     // @ts-ignore

packages/backend-core/src/events/analytics.ts

@@ -1,5 +1,5 @@
 import env from "../environment"
-import * as tenancy from "../tenancy"
+import * as context from "../context"
 import * as dbUtils from "../db/utils"
 import { Config } from "../constants"
 import { withCache, TTL, CacheKey } from "../cache"
@@ -42,7 +42,7 @@ export const enabled = async () => {
 }
 
 const getSettingsDoc = async () => {
-  const db = tenancy.getGlobalDB()
+  const db = context.getGlobalDB()
   let settings
   try {
     settings = await db.get(dbUtils.generateConfigID({ type: Config.SETTINGS }))

packages/backend-core/src/events/identification.ts

@@ -16,6 +16,7 @@ import {
   InstallationGroup,
   UserContext,
   Group,
+  isSSOUser,
 } from "@budibase/types"
 import { processors } from "./processors"
 import * as dbUtils from "../db/utils"
@@ -166,7 +167,10 @@ const identifyUser = async (
   const type = IdentityType.USER
   let builder = user.builder?.global || false
   let admin = user.admin?.global || false
-  let providerType = user.providerType
+  let providerType
+  if (isSSOUser(user)) {
+    providerType = user.providerType
+  }
   const accountHolder = account?.budibaseUserId === user._id || false
   const verified =
     account && account?.budibaseUserId === user._id ? account.verified : false

packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts

@@ -1,4 +1,4 @@
-import "../../../../../tests"
+import { testEnv } from "../../../../../tests"
 import PosthogProcessor from "../PosthogProcessor"
 import { Event, IdentityType, Hosting } from "@budibase/types"
 const tk = require("timekeeper")
@@ -16,6 +16,10 @@ const newIdentity = () => {
 }
 
 describe("PosthogProcessor", () => {
+  beforeAll(() => {
+    testEnv.singleTenant()
+  })
+
   beforeEach(async () => {
     jest.clearAllMocks()
     await cache.bustCache(

packages/backend-core/src/featureFlags/index.ts

@@ -1,5 +1,5 @@
 import env from "../environment"
-import * as tenancy from "../tenancy"
+import * as context from "../context"
 
 /**
  * Read the TENANT_FEATURE_FLAGS env var and return an array of features flags for each tenant.
@@ -28,7 +28,7 @@ export function buildFeatureFlags() {
 }
 
 export function isEnabled(featureFlag: string) {
-  const tenantId = tenancy.getTenantId()
+  const tenantId = context.getTenantId()
   const flags = getTenantFeatureFlags(tenantId)
   return flags.includes(featureFlag)
 }

packages/backend-core/src/index.ts

@@ -3,12 +3,11 @@ export * as migrations from "./migrations"
 export * as users from "./users"
 export * as roles from "./security/roles"
 export * as permissions from "./security/permissions"
-export * as accounts from "./cloud/accounts"
+export * as accounts from "./accounts"
 export * as installation from "./installation"
-export * as tenancy from "./tenancy"
 export * as featureFlags from "./featureFlags"
 export * as sessions from "./security/sessions"
-export * as deprovisioning from "./context/deprovision"
+export * as platform from "./platform"
 export * as auth from "./auth"
 export * as constants from "./constants"
 export * as logging from "./logging"
@@ -21,20 +20,27 @@ export * as context from "./context"
 export * as cache from "./cache"
 export * as objectStore from "./objectStore"
 export * as redis from "./redis"
+export * as locks from "./redis/redlock"
 export * as utils from "./utils"
 export * as errors from "./errors"
 export { default as env } from "./environment"
 
+// Add context to tenancy for backwards compatibility
+// only do this for external usages to prevent internal
+// circular dependencies
+import * as context from "./context"
+import * as _tenancy from "./tenancy"
+export const tenancy = {
+  ..._tenancy,
+  ...context,
+}
+
 // expose error classes directly
 export * from "./errors"
 
 // expose constants directly
 export * from "./constants"
 
-// expose inner locks from redis directly
-import * as redis from "./redis"
-export const locks = redis.redlock
-
 // expose package init function
 import * as db from "./db"
 export const init = (opts: any = {}) => {

packages/backend-core/src/middleware/authenticated.ts

@@ -4,7 +4,7 @@ import { getUser } from "../cache/user"
 import { getSession, updateSessionTTL } from "../security/sessions"
 import { buildMatcherRegex, matches } from "./matchers"
 import { SEPARATOR, queryGlobalView, ViewName } from "../db"
-import { getGlobalDB, doInTenant } from "../tenancy"
+import { getGlobalDB, doInTenant } from "../context"
 import { decrypt } from "../security/encryption"
 import * as identity from "../context/identity"
 import env from "../environment"

packages/backend-core/src/middleware/errorHandling.ts

@@ -0,0 +1,28 @@
+import { APIError } from "@budibase/types"
+import * as errors from "../errors"
+import env from "../environment"
+
+export async function errorHandling(ctx: any, next: any) {
+  try {
+    await next()
+  } catch (err: any) {
+    const status = err.status || err.statusCode || 500
+    ctx.status = status
+
+    if (status > 499 || env.ENABLE_4XX_HTTP_LOGGING) {
+      ctx.log.error(err)
+    }
+
+    const error = errors.getPublicError(err)
+    const body: APIError = {
+      message: err.message,
+      status: status,
+      validationErrors: err.validation,
+      error,
+    }
+
+    ctx.body = body
+  }
+}
+
+export default errorHandling

packages/backend-core/src/middleware/index.ts

@@ -1,7 +1,7 @@
 export * as jwt from "./passport/jwt"
 export * as local from "./passport/local"
-export * as google from "./passport/google"
+export * as google from "./passport/sso/google"
-export * as oidc from "./passport/oidc"
+export * as oidc from "./passport/sso/oidc"
 import * as datasourceGoogle from "./passport/datasource/google"
 export const datasource = {
   google: datasourceGoogle,
@@ -16,4 +16,5 @@ export { default as adminOnly } from "./adminOnly"
 export { default as builderOrAdmin } from "./builderOrAdmin"
 export { default as builderOnly } from "./builderOnly"
 export { default as logging } from "./logging"
+export { default as errorHandling } from "./errorHandling"
 export * as joiValidator from "./joi-validator"

packages/backend-core/src/middleware/passport/datasource/google.ts

@@ -1,10 +1,11 @@
-import * as google from "../google"
+import * as google from "../sso/google"
 import { Cookie, Config } from "../../../constants"
 import { clearCookie, getCookie } from "../../../utils"
 import { getScopedConfig, getPlatformUrl, doWithDB } from "../../../db"
 import environment from "../../../environment"
-import { getGlobalDB } from "../../../tenancy"
+import { getGlobalDB } from "../../../context"
 import { BBContext, Database, SSOProfile } from "@budibase/types"
+import { ssoSaveUserNoOp } from "../sso/sso"
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
 
 type Passport = {
@@ -36,7 +37,11 @@ export async function preAuth(
   const platformUrl = await getPlatformUrl({ tenantAware: false })
 
   let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback`
-  const strategy = await google.strategyFactory(googleConfig, callbackUrl)
+  const strategy = await google.strategyFactory(
+    googleConfig,
+    callbackUrl,
+    ssoSaveUserNoOp
+  )
 
   if (!ctx.query.appId || !ctx.query.datasourceId) {
     ctx.throw(400, "appId and datasourceId query params not present.")

packages/backend-core/src/middleware/passport/local.ts

@@ -1,15 +1,10 @@
 import { UserStatus } from "../../constants"
-import { compare, newid } from "../../utils"
+import { compare } from "../../utils"
-import env from "../../environment"
 import * as users from "../../users"
 import { authError } from "./utils"
-import { createASession } from "../../security/sessions"
-import { getTenantId } from "../../tenancy"
 import { BBContext } from "@budibase/types"
-const jwt = require("jsonwebtoken")
 
 const INVALID_ERR = "Invalid credentials"
-const SSO_NO_PASSWORD = "SSO user does not have a password set"
 const EXPIRED = "This account has expired. Please reset your password"
 
 export const options = {
@@ -35,50 +30,25 @@ export async function authenticate(
 
   const dbUser = await users.getGlobalUserByEmail(email)
   if (dbUser == null) {
-    return authError(done, `User not found: [${email}]`)
+    console.info(`user=${email} could not be found`)
-  }
-
-  // check that the user is currently inactive, if this is the case throw invalid
-  if (dbUser.status === UserStatus.INACTIVE) {
     return authError(done, INVALID_ERR)
   }
 
-  // check that the user has a stored password before proceeding
+  if (dbUser.status === UserStatus.INACTIVE) {
-  if (!dbUser.password) {
+    console.info(`user=${email} is inactive`, dbUser)
-    if (
+    return authError(done, INVALID_ERR)
-      (dbUser.account && dbUser.account.authType === "sso") || // root account sso
+  }
-      dbUser.thirdPartyProfile // internal sso
-    ) {
-      return authError(done, SSO_NO_PASSWORD)
-    }
 
-    console.error("Non SSO usser has no password set", dbUser)
+  if (!dbUser.password) {
+    console.info(`user=${email} has no password set`, dbUser)
     return authError(done, EXPIRED)
   }
 
-  // authenticate
+  if (!(await compare(password, dbUser.password))) {
-  if (await compare(password, dbUser.password)) {
-    const sessionId = newid()
-    const tenantId = getTenantId()
-
-    await createASession(dbUser._id!, { sessionId, tenantId })
-
-    const token = jwt.sign(
-      {
-        userId: dbUser._id,
-        sessionId,
-        tenantId,
-      },
-      env.JWT_SECRET
-    )
-    // Remove users password in payload
-    delete dbUser.password
-
-    return done(null, {
-      ...dbUser,
-      token,
-    })
-  } else {
     return authError(done, INVALID_ERR)
   }
+
+  // intentionally remove the users password in payload
+  delete dbUser.password
+  return done(null, dbUser)
 }

packages/backend-core/src/middleware/passport/sso/google.ts

@@ -1,18 +1,26 @@
-import { ssoCallbackUrl } from "./utils"
+import { ssoCallbackUrl } from "../utils"
-import { authenticateThirdParty, SaveUserFunction } from "./third-party-common"
+import * as sso from "./sso"
-import { ConfigType, GoogleConfig, Database, SSOProfile } from "@budibase/types"
+import {
+  ConfigType,
+  GoogleConfig,
+  Database,
+  SSOProfile,
+  SSOAuthDetails,
+  SSOProviderType,
+  SaveSSOUserFunction,
+} from "@budibase/types"
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
 
-export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
+export function buildVerifyFn(saveUserFn: SaveSSOUserFunction) {
   return (
     accessToken: string,
     refreshToken: string,
     profile: SSOProfile,
     done: Function
   ) => {
-    const thirdPartyUser = {
+    const details: SSOAuthDetails = {
-      provider: profile.provider, // should always be 'google'
+      provider: "google",
-      providerType: "google",
+      providerType: SSOProviderType.GOOGLE,
       userId: profile.id,
       profile: profile,
       email: profile._json.email,
@@ -22,8 +30,8 @@ export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
       },
     }
 
-    return authenticateThirdParty(
+    return sso.authenticate(
-      thirdPartyUser,
+      details,
       true, // require local accounts to exist
       done,
       saveUserFn
@@ -39,7 +47,7 @@ export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
 export async function strategyFactory(
   config: GoogleConfig["config"],
   callbackUrl: string,
-  saveUserFn?: SaveUserFunction
+  saveUserFn: SaveSSOUserFunction
 ) {
   try {
     const { clientID, clientSecret } = config

packages/backend-core/src/middleware/passport/sso/oidc.ts

@@ -1,22 +1,20 @@
 import fetch from "node-fetch"
-import { authenticateThirdParty, SaveUserFunction } from "./third-party-common"
+import * as sso from "./sso"
-import { ssoCallbackUrl } from "./utils"
+import { ssoCallbackUrl } from "../utils"
 import {
   ConfigType,
-  OIDCInnerCfg,
+  OIDCInnerConfig,
   Database,
   SSOProfile,
-  ThirdPartyUser,
+  OIDCStrategyConfiguration,
-  OIDCConfiguration,
+  SSOAuthDetails,
+  SSOProviderType,
+  JwtClaims,
+  SaveSSOUserFunction,
 } from "@budibase/types"
 const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy
 
-type JwtClaims = {
+export function buildVerifyFn(saveUserFn: SaveSSOUserFunction) {
-  preferred_username: string
-  email: string
-}
-
-export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
   /**
    * @param {*} issuer The identity provider base URL
    * @param {*} sub The user ID
@@ -39,10 +37,10 @@ export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
     params: any,
     done: Function
   ) => {
-    const thirdPartyUser: ThirdPartyUser = {
+    const details: SSOAuthDetails = {
       // store the issuer info to enable sync in future
       provider: issuer,
-      providerType: "oidc",
+      providerType: SSOProviderType.OIDC,
       userId: profile.id,
       profile: profile,
       email: getEmail(profile, jwtClaims),
@@ -52,8 +50,8 @@ export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
       },
     }
 
-    return authenticateThirdParty(
+    return sso.authenticate(
-      thirdPartyUser,
+      details,
       false, // don't require local accounts to exist
       done,
       saveUserFn
@@ -104,8 +102,8 @@ function validEmail(value: string) {
  * @returns Dynamically configured Passport OIDC Strategy
  */
 export async function strategyFactory(
-  config: OIDCConfiguration,
+  config: OIDCStrategyConfiguration,
-  saveUserFn?: SaveUserFunction
+  saveUserFn: SaveSSOUserFunction
 ) {
   try {
     const verify = buildVerifyFn(saveUserFn)
@@ -119,14 +117,14 @@ export async function strategyFactory(
 }
 
 export async function fetchStrategyConfig(
-  enrichedConfig: OIDCInnerCfg,
+  oidcConfig: OIDCInnerConfig,
   callbackUrl?: string
-): Promise<OIDCConfiguration> {
+): Promise<OIDCStrategyConfiguration> {
   try {
-    const { clientID, clientSecret, configUrl } = enrichedConfig
+    const { clientID, clientSecret, configUrl } = oidcConfig
 
     if (!clientID || !clientSecret || !callbackUrl || !configUrl) {
-      //check for remote config and all required elements
+      // check for remote config and all required elements
       throw new Error(
         "Configuration invalid. Must contain clientID, clientSecret, callbackUrl and configUrl"
       )

packages/backend-core/src/middleware/passport/sso/sso.ts

@@ -0,0 +1,165 @@
+import { generateGlobalUserID } from "../../../db"
+import { authError } from "../utils"
+import * as users from "../../../users"
+import * as context from "../../../context"
+import fetch from "node-fetch"
+import {
+  SaveSSOUserFunction,
+  SaveUserOpts,
+  SSOAuthDetails,
+  SSOUser,
+  User,
+} from "@budibase/types"
+
+// no-op function for user save
+// - this allows datasource auth and access token refresh to work correctly
+// - prefer no-op over an optional argument to ensure function is provided to login flows
+export const ssoSaveUserNoOp: SaveSSOUserFunction = (
+  user: SSOUser,
+  opts: SaveUserOpts
+) => Promise.resolve(user)
+
+/**
+ * Common authentication logic for third parties. e.g. OAuth, OIDC.
+ */
+export async function authenticate(
+  details: SSOAuthDetails,
+  requireLocalAccount: boolean = true,
+  done: any,
+  saveUserFn: SaveSSOUserFunction
+) {
+  if (!saveUserFn) {
+    throw new Error("Save user function must be provided")
+  }
+  if (!details.userId) {
+    return authError(done, "sso user id required")
+  }
+  if (!details.email) {
+    return authError(done, "sso user email required")
+  }
+
+  // use the third party id
+  const userId = generateGlobalUserID(details.userId)
+
+  let dbUser: User | undefined
+
+  // try to load by id
+  try {
+    dbUser = await users.getById(userId)
+  } catch (err: any) {
+    // abort when not 404 error
+    if (!err.status || err.status !== 404) {
+      return authError(
+        done,
+        "Unexpected error when retrieving existing user",
+        err
+      )
+    }
+  }
+
+  // fallback to loading by email
+  if (!dbUser) {
+    dbUser = await users.getGlobalUserByEmail(details.email)
+  }
+
+  // exit early if there is still no user and auto creation is disabled
+  if (!dbUser && requireLocalAccount) {
+    return authError(
+      done,
+      "Email does not yet exist. You must set up your local budibase account first."
+    )
+  }
+
+  // first time creation
+  if (!dbUser) {
+    // setup a blank user using the third party id
+    dbUser = {
+      _id: userId,
+      email: details.email,
+      roles: {},
+      tenantId: context.getTenantId(),
+    }
+  }
+
+  let ssoUser = await syncUser(dbUser, details)
+  // never prompt for password reset
+  ssoUser.forceResetPassword = false
+
+  try {
+    // don't try to re-save any existing password
+    delete ssoUser.password
+    // create or sync the user
+    ssoUser = (await saveUserFn(ssoUser, {
+      hashPassword: false,
+      requirePassword: false,
+    })) as SSOUser
+  } catch (err: any) {
+    return authError(done, "Error saving user", err)
+  }
+
+  return done(null, ssoUser)
+}
+
+async function getProfilePictureUrl(user: User, details: SSOAuthDetails) {
+  const pictureUrl = details.profile?._json.picture
+  if (pictureUrl) {
+    const response = await fetch(pictureUrl)
+    if (response.status === 200) {
+      const type = response.headers.get("content-type") as string
+      if (type.startsWith("image/")) {
+        return pictureUrl
+      }
+    }
+  }
+}
+
+/**
+ * @returns a user that has been sync'd with third party information
+ */
+async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
+  let firstName
+  let lastName
+  let pictureUrl
+  let oauth2
+  let thirdPartyProfile
+
+  if (details.profile) {
+    const profile = details.profile
+
+    if (profile.name) {
+      const name = profile.name
+      // first name
+      if (name.givenName) {
+        firstName = name.givenName
+      }
+      // last name
+      if (name.familyName) {
+        lastName = name.familyName
+      }
+    }
+
+    pictureUrl = await getProfilePictureUrl(user, details)
+
+    thirdPartyProfile = {
+      ...profile._json,
+    }
+  }
+
+  // oauth tokens for future use
+  if (details.oauth2) {
+    oauth2 = {
+      ...details.oauth2,
+    }
+  }
+
+  return {
+    ...user,
+    provider: details.provider,
+    providerType: details.providerType,
+    firstName,
+    lastName,
+    thirdPartyProfile,
+    pictureUrl,
+    oauth2,
+  }
+}

packages/backend-core/src/middleware/passport/sso/tests/google.spec.ts

@@ -0,0 +1,67 @@
+import { generator, structures } from "../../../../../tests"
+import { SSOProviderType } from "@budibase/types"
+
+jest.mock("passport-google-oauth")
+const mockStrategy = require("passport-google-oauth").OAuth2Strategy
+
+jest.mock("../sso")
+import * as _sso from "../sso"
+const sso = jest.mocked(_sso)
+
+const mockSaveUserFn = jest.fn()
+const mockDone = jest.fn()
+
+import * as google from "../google"
+
+describe("google", () => {
+  describe("strategyFactory", () => {
+    const googleConfig = structures.sso.googleConfig()
+    const callbackUrl = generator.url()
+
+    it("should create successfully create a google strategy", async () => {
+      await google.strategyFactory(googleConfig, callbackUrl, mockSaveUserFn)
+
+      const expectedOptions = {
+        clientID: googleConfig.clientID,
+        clientSecret: googleConfig.clientSecret,
+        callbackURL: callbackUrl,
+      }
+
+      expect(mockStrategy).toHaveBeenCalledWith(
+        expectedOptions,
+        expect.anything()
+      )
+    })
+  })
+
+  describe("authenticate", () => {
+    const details = structures.sso.authDetails()
+    details.provider = "google"
+    details.providerType = SSOProviderType.GOOGLE
+
+    const profile = details.profile!
+    profile.provider = "google"
+
+    beforeEach(() => {
+      jest.clearAllMocks()
+    })
+
+    it("delegates authentication to third party common", async () => {
+      const authenticate = await google.buildVerifyFn(mockSaveUserFn)
+
+      await authenticate(
+        details.oauth2.accessToken,
+        details.oauth2.refreshToken!,
+        profile,
+        mockDone
+      )
+
+      expect(sso.authenticate).toHaveBeenCalledWith(
+        details,
+        true,
+        mockDone,
+        mockSaveUserFn
+      )
+    })
+  })
+})

packages/backend-core/src/middleware/passport/sso/tests/oidc.spec.ts

@@ -0,0 +1,152 @@
+import { generator, mocks, structures } from "../../../../../tests"
+import {
+  JwtClaims,
+  OIDCInnerConfig,
+  SSOAuthDetails,
+  SSOProviderType,
+} from "@budibase/types"
+import * as _sso from "../sso"
+import * as oidc from "../oidc"
+
+jest.mock("@techpass/passport-openidconnect")
+const mockStrategy = require("@techpass/passport-openidconnect").Strategy
+
+jest.mock("../sso")
+const sso = jest.mocked(_sso)
+
+const mockSaveUser = jest.fn()
+const mockDone = jest.fn()
+
+describe("oidc", () => {
+  const callbackUrl = generator.url()
+  const oidcConfig: OIDCInnerConfig = structures.sso.oidcConfig()
+  const wellKnownConfig = structures.sso.oidcWellKnownConfig()
+
+  function mockRetrieveWellKnownConfig() {
+    // mock the request to retrieve the oidc configuration
+    mocks.fetch.mockReturnValue({
+      ok: true,
+      json: () => wellKnownConfig,
+    })
+  }
+
+  beforeEach(() => {
+    mockRetrieveWellKnownConfig()
+  })
+
+  describe("strategyFactory", () => {
+    it("should create successfully create an oidc strategy", async () => {
+      const strategyConfiguration = await oidc.fetchStrategyConfig(
+        oidcConfig,
+        callbackUrl
+      )
+      await oidc.strategyFactory(strategyConfiguration, mockSaveUser)
+
+      expect(mocks.fetch).toHaveBeenCalledWith(oidcConfig.configUrl)
+
+      const expectedOptions = {
+        issuer: wellKnownConfig.issuer,
+        authorizationURL: wellKnownConfig.authorization_endpoint,
+        tokenURL: wellKnownConfig.token_endpoint,
+        userInfoURL: wellKnownConfig.userinfo_endpoint,
+        clientID: oidcConfig.clientID,
+        clientSecret: oidcConfig.clientSecret,
+        callbackURL: callbackUrl,
+      }
+      expect(mockStrategy).toHaveBeenCalledWith(
+        expectedOptions,
+        expect.anything()
+      )
+    })
+  })
+
+  describe("authenticate", () => {
+    const details: SSOAuthDetails = structures.sso.authDetails()
+    details.providerType = SSOProviderType.OIDC
+    const profile = details.profile!
+    const issuer = profile.provider
+
+    const sub = generator.string()
+    const idToken = generator.string()
+    const params = {}
+
+    let authenticateFn: any
+    let jwtClaims: JwtClaims
+
+    beforeEach(async () => {
+      jest.clearAllMocks()
+      authenticateFn = await oidc.buildVerifyFn(mockSaveUser)
+    })
+
+    async function authenticate() {
+      await authenticateFn(
+        issuer,
+        sub,
+        profile,
+        jwtClaims,
+        details.oauth2.accessToken,
+        details.oauth2.refreshToken,
+        idToken,
+        params,
+        mockDone
+      )
+    }
+
+    it("passes auth details to sso module", async () => {
+      await authenticate()
+
+      expect(sso.authenticate).toHaveBeenCalledWith(
+        details,
+        false,
+        mockDone,
+        mockSaveUser
+      )
+    })
+
+    it("uses JWT email to get email", async () => {
+      delete profile._json.email
+
+      jwtClaims = {
+        email: details.email,
+      }
+
+      await authenticate()
+
+      expect(sso.authenticate).toHaveBeenCalledWith(
+        details,
+        false,
+        mockDone,
+        mockSaveUser
+      )
+    })
+
+    it("uses JWT username to get email", async () => {
+      delete profile._json.email
+
+      jwtClaims = {
+        email: details.email,
+      }
+
+      await authenticate()
+
+      expect(sso.authenticate).toHaveBeenCalledWith(
+        details,
+        false,
+        mockDone,
+        mockSaveUser
+      )
+    })
+
+    it("uses JWT invalid username to get email", async () => {
+      delete profile._json.email
+
+      jwtClaims = {
+        preferred_username: "invalidUsername",
+      }
+
+      await expect(authenticate()).rejects.toThrow(
+        "Could not determine user email from profile"
+      )
+    })
+  })
+})

packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts

@@ -0,0 +1,196 @@
+import { structures, testEnv, mocks } from "../../../../../tests"
+import { SSOAuthDetails, User } from "@budibase/types"
+
+import { HTTPError } from "../../../../errors"
+import * as sso from "../sso"
+import * as context from "../../../../context"
+
+const mockDone = jest.fn()
+const mockSaveUser = jest.fn()
+
+jest.mock("../../../../users")
+import * as _users from "../../../../users"
+const users = jest.mocked(_users)
+
+const getErrorMessage = () => {
+  return mockDone.mock.calls[0][2].message
+}
+
+describe("sso", () => {
+  describe("authenticate", () => {
+    beforeEach(() => {
+      jest.clearAllMocks()
+      testEnv.singleTenant()
+    })
+
+    describe("validation", () => {
+      const testValidation = async (
+        details: SSOAuthDetails,
+        message: string
+      ) => {
+        await sso.authenticate(details, false, mockDone, mockSaveUser)
+
+        expect(mockDone.mock.calls.length).toBe(1)
+        expect(getErrorMessage()).toContain(message)
+      }
+
+      it("user id fails", async () => {
+        const details = structures.sso.authDetails()
+        details.userId = undefined!
+
+        await testValidation(details, "sso user id required")
+      })
+
+      it("email fails", async () => {
+        const details = structures.sso.authDetails()
+        details.email = undefined!
+
+        await testValidation(details, "sso user email required")
+      })
+    })
+
+    function mockGetProfilePicture() {
+      mocks.fetch.mockReturnValueOnce(
+        Promise.resolve({
+          status: 200,
+          headers: { get: () => "image/" },
+        })
+      )
+    }
+
+    describe("when the user doesn't exist", () => {
+      let user: User
+      let details: SSOAuthDetails
+
+      beforeEach(() => {
+        users.getById.mockImplementationOnce(() => {
+          throw new HTTPError("", 404)
+        })
+        mockGetProfilePicture()
+
+        user = structures.users.user()
+        delete user._rev
+        delete user._id
+
+        details = structures.sso.authDetails(user)
+        details.userId = structures.uuid()
+      })
+
+      describe("when a local account is required", () => {
+        it("returns an error message", async () => {
+          const details = structures.sso.authDetails()
+
+          await sso.authenticate(details, true, mockDone, mockSaveUser)
+
+          expect(mockDone.mock.calls.length).toBe(1)
+          expect(getErrorMessage()).toContain(
+            "Email does not yet exist. You must set up your local budibase account first."
+          )
+        })
+      })
+
+      describe("when a local account isn't required", () => {
+        it("creates and authenticates the user", async () => {
+          const ssoUser = structures.users.ssoUser({ user, details })
+          mockSaveUser.mockReturnValueOnce(ssoUser)
+
+          await sso.authenticate(details, false, mockDone, mockSaveUser)
+
+          // default roles for new user
+          ssoUser.roles = {}
+
+          // modified external id to match user format
+          ssoUser._id = "us_" + details.userId
+
+          // new sso user won't have a password
+          delete ssoUser.password
+
+          // new user isn't saved with rev
+          delete ssoUser._rev
+
+          // tenant id added
+          ssoUser.tenantId = context.getTenantId()
+
+          expect(mockSaveUser).toBeCalledWith(ssoUser, {
+            hashPassword: false,
+            requirePassword: false,
+          })
+          expect(mockDone).toBeCalledWith(null, ssoUser)
+        })
+      })
+    })
+
+    describe("when the user exists", () => {
+      let existingUser: User
+      let details: SSOAuthDetails
+
+      beforeEach(() => {
+        existingUser = structures.users.user()
+        existingUser._id = structures.uuid()
+        details = structures.sso.authDetails(existingUser)
+        mockGetProfilePicture()
+      })
+
+      describe("exists by email", () => {
+        beforeEach(() => {
+          users.getById.mockImplementationOnce(() => {
+            throw new HTTPError("", 404)
+          })
+          users.getGlobalUserByEmail.mockReturnValueOnce(
+            Promise.resolve(existingUser)
+          )
+        })
+
+        it("syncs and authenticates the user", async () => {
+          const ssoUser = structures.users.ssoUser({
+            user: existingUser,
+            details,
+          })
+          mockSaveUser.mockReturnValueOnce(ssoUser)
+
+          await sso.authenticate(details, true, mockDone, mockSaveUser)
+
+          // roles preserved
+          ssoUser.roles = existingUser.roles
+
+          // existing id preserved
+          ssoUser._id = existingUser._id
+
+          expect(mockSaveUser).toBeCalledWith(ssoUser, {
+            hashPassword: false,
+            requirePassword: false,
+          })
+          expect(mockDone).toBeCalledWith(null, ssoUser)
+        })
+      })
+
+      describe("exists by id", () => {
+        beforeEach(() => {
+          users.getById.mockReturnValueOnce(Promise.resolve(existingUser))
+        })
+
+        it("syncs and authenticates the user", async () => {
+          const ssoUser = structures.users.ssoUser({
+            user: existingUser,
+            details,
+          })
+          mockSaveUser.mockReturnValueOnce(ssoUser)
+
+          await sso.authenticate(details, true, mockDone, mockSaveUser)
+
+          // roles preserved
+          ssoUser.roles = existingUser.roles
+
+          // existing id preserved
+          ssoUser._id = existingUser._id
+
+          expect(mockSaveUser).toBeCalledWith(ssoUser, {
+            hashPassword: false,
+            requirePassword: false,
+          })
+          expect(mockDone).toBeCalledWith(null, ssoUser)
+        })
+      })
+    })
+  })
+})

packages/backend-core/src/middleware/passport/tests/google.spec.js

@@ -1,79 +0,0 @@
-// Mock data
-
-const { data } = require("./utilities/mock-data")
-
-const TENANT_ID = "default"
-
-const googleConfig = {
-  clientID: data.clientID,
-  clientSecret: data.clientSecret,
-}
-
-const profile = {
-  id: "mockId",
-  _json: {
-    email : data.email
-  },
-  provider: "google"
-}
-
-const user = data.buildThirdPartyUser("google", "google", profile)
-
-describe("google", () => {
-  describe("strategyFactory", () => {  
-    // mock passport strategy factory
-    jest.mock("passport-google-oauth")
-    const mockStrategy = require("passport-google-oauth").OAuth2Strategy
-  
-    it("should create successfully create a google strategy", async () => {
-      const google = require("../google")
-
-      const callbackUrl = `/api/global/auth/${TENANT_ID}/google/callback`
-      await google.strategyFactory(googleConfig, callbackUrl)
-  
-      const expectedOptions = {
-        clientID: googleConfig.clientID,
-        clientSecret: googleConfig.clientSecret,
-        callbackURL: callbackUrl,
-      }
-
-      expect(mockStrategy).toHaveBeenCalledWith(
-        expectedOptions,
-        expect.anything()
-      )
-    })
-  })
-  
-  describe("authenticate", () => {    
-    afterEach(() => {
-      jest.clearAllMocks();
-    });
-
-    // mock third party common authentication
-    jest.mock("../third-party-common")
-    const authenticateThirdParty = require("../third-party-common").authenticateThirdParty
-
-    // mock the passport callback
-    const mockDone = jest.fn()
-
-    it("delegates authentication to third party common", async () => {
-      const google = require("../google")
-      const mockSaveUserFn = jest.fn()
-      const authenticate = await google.buildVerifyFn(mockSaveUserFn)
-
-      await authenticate(
-        data.accessToken,
-        data.refreshToken,
-        profile,
-        mockDone
-      )
-
-      expect(authenticateThirdParty).toHaveBeenCalledWith(
-        user,
-        true, 
-        mockDone,
-        mockSaveUserFn)
-    })
-  })
-})
-

packages/backend-core/src/middleware/passport/tests/oidc.spec.js

@@ -1,144 +0,0 @@
-// Mock data
-const mockFetch = require("node-fetch")
-const { data } = require("./utilities/mock-data")
-const issuer = "mockIssuer"
-const sub = "mockSub"
-const profile = {
-  id: "mockId",
-  _json: {
-    email : data.email
-  }
-}
-let jwtClaims = {}  
-const idToken = "mockIdToken"
-const params = {}
-
-const callbackUrl = "http://somecallbackurl"
-
-// response from .well-known/openid-configuration
-const oidcConfigUrlResponse = {
-  issuer: issuer,
-  authorization_endpoint: "mockAuthorizationEndpoint",
-  token_endpoint: "mockTokenEndpoint",
-  userinfo_endpoint: "mockUserInfoEndpoint"
-}
-
-const oidcConfig = {
-  configUrl: "http://someconfigurl",
-  clientID: data.clientID,
-  clientSecret: data.clientSecret,
-}
-
-const user = data.buildThirdPartyUser(issuer, "oidc", profile)
-
-describe("oidc", () => {
-  describe("strategyFactory", () => {  
-    // mock passport strategy factory
-    jest.mock("@techpass/passport-openidconnect")
-    const mockStrategy = require("@techpass/passport-openidconnect").Strategy
-    
-    // mock the request to retrieve the oidc configuration
-    mockFetch.mockReturnValue({
-      ok: true,
-      json: () => oidcConfigUrlResponse
-    })
-  
-    it("should create successfully create an oidc strategy", async () => {
-      const oidc = require("../oidc")
-      const enrichedConfig = await oidc.fetchStrategyConfig(oidcConfig, callbackUrl)
-      await oidc.strategyFactory(enrichedConfig, callbackUrl)
-  
-      expect(mockFetch).toHaveBeenCalledWith(oidcConfig.configUrl)
-
-      const expectedOptions = {
-        issuer: oidcConfigUrlResponse.issuer,
-        authorizationURL: oidcConfigUrlResponse.authorization_endpoint,
-        tokenURL: oidcConfigUrlResponse.token_endpoint,
-        userInfoURL: oidcConfigUrlResponse.userinfo_endpoint,
-        clientID: oidcConfig.clientID,
-        clientSecret: oidcConfig.clientSecret,
-        callbackURL: callbackUrl,
-      }
-      expect(mockStrategy).toHaveBeenCalledWith(
-        expectedOptions,
-        expect.anything()
-      )
-    })
-  })
-  
-  describe("authenticate", () => {    
-    afterEach(() => {
-      jest.clearAllMocks()
-    });
-
-    // mock third party common authentication
-    jest.mock("../third-party-common")
-    const authenticateThirdParty = require("../third-party-common").authenticateThirdParty
-    
-    // mock the passport callback
-    const mockDone = jest.fn()
-    const mockSaveUserFn = jest.fn()
-
-    async function doAuthenticate() {
-      const oidc = require("../oidc")
-      const authenticate = await oidc.buildVerifyFn(mockSaveUserFn)
-
-      await authenticate(
-        issuer,
-        sub,
-        profile,
-        jwtClaims,
-        data.accessToken,
-        data.refreshToken,
-        idToken,
-        params,
-        mockDone
-      )
-    }
-
-    async function doTest() {
-      await doAuthenticate()
-
-      expect(authenticateThirdParty).toHaveBeenCalledWith(
-        user,
-        false, 
-        mockDone,
-        mockSaveUserFn,
-      )
-    }
-
-    it("delegates authentication to third party common", async () => {
-      await doTest()
-    })
-
-    it("uses JWT email to get email", async () => {
-      delete profile._json.email
-      jwtClaims = {
-        email : "mock@budibase.com"
-      }
-
-      await doTest()
-    })
-  
-    it("uses JWT username to get email", async () => {
-      delete profile._json.email
-      jwtClaims = {
-        preferred_username : "mock@budibase.com"
-      }
-
-      await doTest()
-    })
-
-    it("uses JWT invalid username to get email", async () => {
-      delete profile._json.email
-
-      jwtClaims = {
-        preferred_username : "invalidUsername"
-      }
-
-      await expect(doAuthenticate()).rejects.toThrow("Could not determine user email from profile");
-    })
-  
-  })
-})
-

packages/backend-core/src/middleware/passport/tests/third-party-common.seq.spec.js

@@ -1,178 +0,0 @@
-require("../../../../tests")
-const { authenticateThirdParty } = require("../third-party-common")
-const { data } = require("./utilities/mock-data")
-const { DEFAULT_TENANT_ID } = require("../../../constants")
-
-const { generateGlobalUserID } = require("../../../db/utils")
-const { newid } = require("../../../utils")
-const { doWithGlobalDB, doInTenant } = require("../../../tenancy")
-
-const done = jest.fn()
-
-const getErrorMessage = () => {
-  return done.mock.calls[0][2].message
-}
-
-const saveUser = async (user) => {
-  return doWithGlobalDB(DEFAULT_TENANT_ID, async db => {
-    return await db.put(user)
-  })
-}
-
-function authenticate(user, requireLocal, saveFn) {
-  return doInTenant(DEFAULT_TENANT_ID, () => {
-    return authenticateThirdParty(user, requireLocal, done, saveFn)
-  })
-}
-
-describe("third party common", () => {
-  describe("authenticateThirdParty", () => {
-    let thirdPartyUser
-
-    beforeEach(() => {
-      thirdPartyUser = data.buildThirdPartyUser()
-    })
-
-    afterEach(async () => {
-      return doWithGlobalDB(DEFAULT_TENANT_ID, async db => {
-        jest.clearAllMocks()
-        await db.destroy()
-      })
-    })
-
-    describe("validation", () => {
-      const testValidation = async (message) => {
-        await authenticate(thirdPartyUser, false, saveUser)
-        expect(done.mock.calls.length).toBe(1)
-        expect(getErrorMessage()).toContain(message)
-      }
-
-      it("provider fails", async () => {
-        delete thirdPartyUser.provider
-        await testValidation("third party user provider required")
-      })
-
-      it("user id fails", async () => {
-        delete thirdPartyUser.userId
-        await testValidation("third party user id required")
-      })
-
-      it("email fails", async () => {
-        delete thirdPartyUser.email
-        await testValidation("third party user email required")
-      })
-    })
-
-    const expectUserIsAuthenticated = () => {
-      const user = done.mock.calls[0][1]
-      expect(user).toBeDefined()
-      expect(user._id).toBeDefined()
-      expect(user._rev).toBeDefined()
-      expect(user.token).toBeDefined()
-      return user
-    }
-
-    const expectUserIsSynced = (user, thirdPartyUser) => {
-      expect(user.provider).toBe(thirdPartyUser.provider)
-      expect(user.firstName).toBe(thirdPartyUser.profile.name.givenName)
-      expect(user.lastName).toBe(thirdPartyUser.profile.name.familyName)
-      expect(user.thirdPartyProfile).toStrictEqual(thirdPartyUser.profile._json)
-      expect(user.oauth2).toStrictEqual(thirdPartyUser.oauth2)
-    }
-
-    describe("when the user doesn't exist", () => {
-      describe("when a local account is required", () => {
-        it("returns an error message", async () => {
-          await authenticate(thirdPartyUser, true, saveUser)
-          expect(done.mock.calls.length).toBe(1)
-          expect(getErrorMessage()).toContain("Email does not yet exist. You must set up your local budibase account first.")
-        })
-      })
-
-      describe("when a local account isn't required", () => {
-        it("creates and authenticates the user", async () => {
-          await authenticate(thirdPartyUser, false, saveUser)
-          const user = expectUserIsAuthenticated()
-          expectUserIsSynced(user, thirdPartyUser)
-          expect(user.roles).toStrictEqual({})
-        })
-      })
-    })
-
-    describe("when the user exists", () => {
-      let dbUser
-      let id
-      let email
-
-      const createUser = async () => {
-        return doWithGlobalDB(DEFAULT_TENANT_ID, async db => {
-          dbUser = {
-            _id: id,
-            email: email,
-          }
-          const response = await db.put(dbUser)
-          dbUser._rev = response.rev
-          return dbUser
-        })
-      }
-
-      const expectUserIsUpdated = (user) => {
-        // id is unchanged
-        expect(user._id).toBe(id)
-        // user is updated
-        expect(user._rev).not.toBe(dbUser._rev)
-      }
-
-      describe("exists by email", () => {
-        beforeEach(async () => {
-          id = generateGlobalUserID(newid()) // random id
-          email = thirdPartyUser.email //  matching email
-          await createUser()
-        })
-
-        it("syncs and authenticates the user", async () => {
-          await authenticate(thirdPartyUser, true, saveUser)
-
-          const user = expectUserIsAuthenticated()
-          expectUserIsSynced(user, thirdPartyUser)
-          expectUserIsUpdated(user)
-        })
-      })
-
-      describe("exists by email with different casing", () => {
-        beforeEach(async () => {
-          id = generateGlobalUserID(newid()) // random id
-          email = thirdPartyUser.email.toUpperCase() //  matching email except for casing
-          await createUser()
-        })
-
-        it("syncs and authenticates the user", async () => {
-          await authenticate(thirdPartyUser, true, saveUser)
-
-          const user = expectUserIsAuthenticated()
-          expectUserIsSynced(user, thirdPartyUser)
-          expectUserIsUpdated(user)
-          expect(user.email).toBe(thirdPartyUser.email.toUpperCase())
-        })
-      })
-
-
-      describe("exists by id", () => {
-        beforeEach(async () => {
-          id = generateGlobalUserID(thirdPartyUser.userId) // matching id
-          email = "test@test.com" // random email
-          await createUser()
-        })
-
-        it("syncs and authenticates the user", async () => {
-          await authenticate(thirdPartyUser, true, saveUser)
-
-          const user = expectUserIsAuthenticated()
-          expectUserIsSynced(user, thirdPartyUser)
-          expectUserIsUpdated(user)
-        })
-      })
-    })
-  })
-})
-

packages/backend-core/src/middleware/passport/tests/utilities/mock-data.js

@@ -1,54 +0,0 @@
-// Mock Data
-
-const mockClientID = "mockClientID"
-const mockClientSecret = "mockClientSecret"
-
-const mockEmail = "mock@budibase.com"
-const mockAccessToken = "mockAccessToken"
-const mockRefreshToken = "mockRefreshToken"
-
-const mockProvider = "mockProvider"
-const mockProviderType = "mockProviderType"
-
-const mockProfile = {
-  id: "mockId",
-  name: {
-    givenName: "mockGivenName",
-    familyName: "mockFamilyName",
-  },
-  _json: {
-    email: mockEmail,
-  },
-}
-
-const buildOauth2 = (
-  accessToken = mockAccessToken,
-  refreshToken = mockRefreshToken
-) => ({
-  accessToken: accessToken,
-  refreshToken: refreshToken,
-})
-
-const buildThirdPartyUser = (
-  provider = mockProvider,
-  providerType = mockProviderType,
-  profile = mockProfile,
-  email = mockEmail,
-  oauth2 = buildOauth2()
-) => ({
-  provider: provider,
-  providerType: providerType,
-  userId: profile.id,
-  profile: profile,
-  email: email,
-  oauth2: oauth2,
-})
-
-exports.data = {
-  clientID: mockClientID,
-  clientSecret: mockClientSecret,
-  email: mockEmail,
-  accessToken: mockAccessToken,
-  refreshToken: mockRefreshToken,
-  buildThirdPartyUser,
-}

packages/backend-core/src/middleware/passport/third-party-common.ts

@@ -1,177 +0,0 @@
-import env from "../../environment"
-import { generateGlobalUserID } from "../../db"
-import { authError } from "./utils"
-import { newid } from "../../utils"
-import { createASession } from "../../security/sessions"
-import * as users from "../../users"
-import { getGlobalDB, getTenantId } from "../../tenancy"
-import fetch from "node-fetch"
-import { ThirdPartyUser } from "@budibase/types"
-const jwt = require("jsonwebtoken")
-
-type SaveUserOpts = {
-  requirePassword?: boolean
-  hashPassword?: boolean
-  currentUserId?: string
-}
-
-export type SaveUserFunction = (
-  user: ThirdPartyUser,
-  opts: SaveUserOpts
-) => Promise<any>
-
-/**
- * Common authentication logic for third parties. e.g. OAuth, OIDC.
- */
-export async function authenticateThirdParty(
-  thirdPartyUser: ThirdPartyUser,
-  requireLocalAccount: boolean = true,
-  done: Function,
-  saveUserFn?: SaveUserFunction
-) {
-  if (!saveUserFn) {
-    throw new Error("Save user function must be provided")
-  }
-  if (!thirdPartyUser.provider) {
-    return authError(done, "third party user provider required")
-  }
-  if (!thirdPartyUser.userId) {
-    return authError(done, "third party user id required")
-  }
-  if (!thirdPartyUser.email) {
-    return authError(done, "third party user email required")
-  }
-
-  // use the third party id
-  const userId = generateGlobalUserID(thirdPartyUser.userId)
-  const db = getGlobalDB()
-
-  let dbUser
-
-  // try to load by id
-  try {
-    dbUser = await db.get(userId)
-  } catch (err: any) {
-    // abort when not 404 error
-    if (!err.status || err.status !== 404) {
-      return authError(
-        done,
-        "Unexpected error when retrieving existing user",
-        err
-      )
-    }
-  }
-
-  // fallback to loading by email
-  if (!dbUser) {
-    dbUser = await users.getGlobalUserByEmail(thirdPartyUser.email)
-  }
-
-  // exit early if there is still no user and auto creation is disabled
-  if (!dbUser && requireLocalAccount) {
-    return authError(
-      done,
-      "Email does not yet exist. You must set up your local budibase account first."
-    )
-  }
-
-  // first time creation
-  if (!dbUser) {
-    // setup a blank user using the third party id
-    dbUser = {
-      _id: userId,
-      email: thirdPartyUser.email,
-      roles: {},
-    }
-  }
-
-  dbUser = await syncUser(dbUser, thirdPartyUser)
-
-  // never prompt for password reset
-  dbUser.forceResetPassword = false
-
-  // create or sync the user
-  try {
-    await saveUserFn(dbUser, { hashPassword: false, requirePassword: false })
-  } catch (err: any) {
-    return authError(done, "Error saving user", err)
-  }
-
-  // now that we're sure user exists, load them from the db
-  dbUser = await db.get(dbUser._id)
-
-  // authenticate
-  const sessionId = newid()
-  const tenantId = getTenantId()
-  await createASession(dbUser._id, { sessionId, tenantId })
-
-  dbUser.token = jwt.sign(
-    {
-      userId: dbUser._id,
-      sessionId,
-    },
-    env.JWT_SECRET
-  )
-
-  return done(null, dbUser)
-}
-
-async function syncProfilePicture(
-  user: ThirdPartyUser,
-  thirdPartyUser: ThirdPartyUser
-) {
-  const pictureUrl = thirdPartyUser.profile?._json.picture
-  if (pictureUrl) {
-    const response = await fetch(pictureUrl)
-
-    if (response.status === 200) {
-      const type = response.headers.get("content-type") as string
-      if (type.startsWith("image/")) {
-        user.pictureUrl = pictureUrl
-      }
-    }
-  }
-
-  return user
-}
-
-/**
- * @returns a user that has been sync'd with third party information
- */
-async function syncUser(user: ThirdPartyUser, thirdPartyUser: ThirdPartyUser) {
-  // provider
-  user.provider = thirdPartyUser.provider
-  user.providerType = thirdPartyUser.providerType
-
-  if (thirdPartyUser.profile) {
-    const profile = thirdPartyUser.profile
-
-    if (profile.name) {
-      const name = profile.name
-      // first name
-      if (name.givenName) {
-        user.firstName = name.givenName
-      }
-      // last name
-      if (name.familyName) {
-        user.lastName = name.familyName
-      }
-    }
-
-    user = await syncProfilePicture(user, thirdPartyUser)
-
-    // profile
-    user.thirdPartyProfile = {
-      ...profile._json,
-    }
-  }
-
-  // oauth tokens for future use
-  if (thirdPartyUser.oauth2) {
-    user.oauth2 = {
-      ...thirdPartyUser.oauth2,
-    }
-  }
-
-  return user
-}

packages/backend-core/src/middleware/passport/utils.ts

@@ -1,6 +1,6 @@
-import { isMultiTenant, getTenantId } from "../../tenancy"
+import { isMultiTenant, getTenantId } from "../../context"
 import { getScopedConfig } from "../../db"
-import { ConfigType, Database, Config } from "@budibase/types"
+import { ConfigType, Database } from "@budibase/types"
 
 /**
  * Utility to handle authentication errors.

packages/backend-core/src/middleware/tenancy.ts

@@ -1,4 +1,5 @@
-import { doInTenant, getTenantIDFromCtx } from "../tenancy"
+import { doInTenant } from "../context"
+import { getTenantIDFromCtx } from "../tenancy"
 import { buildMatcherRegex, matches } from "./matchers"
 import { Header } from "../constants"
 import {

packages/backend-core/src/migrations/migrations.ts

@@ -4,10 +4,10 @@ import {
   StaticDatabases,
   getAllApps,
   getGlobalDBName,
-  doWithDB,
+  getDB,
 } from "../db"
 import environment from "../environment"
-import { doInTenant, getTenantIds, getTenantId } from "../tenancy"
+import * as platform from "../platform"
 import * as context from "../context"
 import { DEFINITIONS } from "."
 import {
@@ -47,7 +47,7 @@ export const runMigration = async (
   const migrationType = migration.type
   let tenantId: string | undefined
   if (migrationType !== MigrationType.INSTALLATION) {
-    tenantId = getTenantId()
+    tenantId = context.getTenantId()
   }
   const migrationName = migration.name
   const silent = migration.silent
@@ -86,66 +86,65 @@ export const runMigration = async (
     count++
     const lengthStatement = length > 1 ? `[${count}/${length}]` : ""
 
-    await doWithDB(dbName, async (db: any) => {
+    const db = getDB(dbName)
-      try {
+    try {
-        const doc = await getMigrationsDoc(db)
+      const doc = await getMigrationsDoc(db)
 
-        // the migration has already been run
+      // the migration has already been run
-        if (doc[migrationName]) {
+      if (doc[migrationName]) {
-          // check for force
+        // check for force
-          if (
+        if (
-            options.force &&
+          options.force &&
-            options.force[migrationType] &&
+          options.force[migrationType] &&
-            options.force[migrationType].includes(migrationName)
+          options.force[migrationType].includes(migrationName)
-          ) {
+        ) {
-            log(
-              `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
-            )
-          } else {
-            // no force, exit
-            return
-          }
-        }
-
-        // check if the migration is not a no-op
-        if (!options.noOp) {
           log(
-            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
+            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
-          )
-
-          if (migration.preventRetry) {
-            // eagerly set the completion date
-            // so that we never run this migration twice even upon failure
-            doc[migrationName] = Date.now()
-            const response = await db.put(doc)
-            doc._rev = response.rev
-          }
-
-          // run the migration
-          if (migrationType === MigrationType.APP) {
-            await context.doInAppContext(db.name, async () => {
-              await migration.fn(db)
-            })
-          } else {
-            await migration.fn(db)
-          }
-
-          log(
-            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
           )
+        } else {
+          // no force, exit
+          return
         }
-
-        // mark as complete
-        doc[migrationName] = Date.now()
-        await db.put(doc)
-      } catch (err) {
-        console.error(
-          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
-          err
-        )
-        throw err
       }
-    })
+
+      // check if the migration is not a no-op
+      if (!options.noOp) {
+        log(
+          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
+        )
+
+        if (migration.preventRetry) {
+          // eagerly set the completion date
+          // so that we never run this migration twice even upon failure
+          doc[migrationName] = Date.now()
+          const response = await db.put(doc)
+          doc._rev = response.rev
+        }
+
+        // run the migration
+        if (migrationType === MigrationType.APP) {
+          await context.doInAppContext(db.name, async () => {
+            await migration.fn(db)
+          })
+        } else {
+          await migration.fn(db)
+        }
+
+        log(
+          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
+        )
+      }
+
+      // mark as complete
+      doc[migrationName] = Date.now()
+      await db.put(doc)
+    } catch (err) {
+      console.error(
+        `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
+        err
+      )
+      throw err
+    }
   }
 }
 
@@ -160,7 +159,7 @@ export const runMigrations = async (
       tenantIds = [options.noOp.tenantId]
     } else if (!options.tenantIds || !options.tenantIds.length) {
       // run for all tenants
-      tenantIds = await getTenantIds()
+      tenantIds = await platform.tenants.getTenantIds()
     } else {
       tenantIds = options.tenantIds
     }
@@ -185,7 +184,10 @@ export const runMigrations = async (
     // for all migrations
     for (const migration of migrations) {
       // run the migration
-      await doInTenant(tenantId, () => runMigration(migration, options))
+      await context.doInTenant(
+        tenantId,
+        async () => await runMigration(migration, options)
+      )
     }
   }
   console.log("Migrations complete")

packages/backend-core/src/migrations/tests/index.spec.js

@@ -1,57 +0,0 @@
-require("../../../tests")
-const { runMigrations, getMigrationsDoc } = require("../index")
-const { getGlobalDBName, getDB } = require("../../db")
-
-const { structures, testEnv } = require("../../../tests")
-testEnv.multiTenant()
-
-let db
-
-describe("migrations", () => {
-
-  const migrationFunction = jest.fn()
-
-  const MIGRATIONS = [{
-    type: "global",
-    name: "test",
-    fn: migrationFunction
-  }]
-
-  let tenantId
-
-  beforeEach(() => {
-    tenantId = structures.tenant.id()
-    db = getDB(getGlobalDBName(tenantId))
-  })
-
-  afterEach(async () => {
-    jest.clearAllMocks()
-    await db.destroy()
-  })
-
-  const migrate = () => {
-    return runMigrations(MIGRATIONS, { tenantIds: [tenantId]})
-  }
-
-  it("should run a new migration", async () => {
-    await migrate()
-    expect(migrationFunction).toHaveBeenCalled()
-    const doc = await getMigrationsDoc(db)
-    expect(doc.test).toBeDefined()
-  })
-
-  it("should match snapshot", async () => {
-    await migrate()
-    const doc = await getMigrationsDoc(db)
-    expect(doc).toMatchSnapshot()
-  })
-
-  it("should skip a previously run migration", async () => {
-    await migrate()
-    const previousMigrationTime = await getMigrationsDoc(db).test
-    await migrate()
-    const currentMigrationTime = await getMigrationsDoc(db).test
-    expect(migrationFunction).toHaveBeenCalledTimes(1)
-    expect(currentMigrationTime).toBe(previousMigrationTime)
-  })
-})

packages/backend-core/src/migrations/tests/migrations.spec.ts

@@ -0,0 +1,64 @@
+import { testEnv, DBTestConfiguration } from "../../../tests"
+import * as migrations from "../index"
+import * as context from "../../context"
+import { MigrationType } from "@budibase/types"
+
+testEnv.multiTenant()
+
+describe("migrations", () => {
+  const config = new DBTestConfiguration()
+
+  const migrationFunction = jest.fn()
+
+  const MIGRATIONS = [
+    {
+      type: MigrationType.GLOBAL,
+      name: "test" as any,
+      fn: migrationFunction,
+    },
+  ]
+
+  beforeEach(() => {
+    config.newTenant()
+  })
+
+  afterEach(async () => {
+    jest.clearAllMocks()
+  })
+
+  const migrate = () => {
+    return migrations.runMigrations(MIGRATIONS, {
+      tenantIds: [config.tenantId],
+    })
+  }
+
+  it("should run a new migration", async () => {
+    await config.doInTenant(async () => {
+      await migrate()
+      expect(migrationFunction).toHaveBeenCalled()
+      const db = context.getGlobalDB()
+      const doc = await migrations.getMigrationsDoc(db)
+      expect(doc.test).toBeDefined()
+    })
+  })
+
+  it("should match snapshot", async () => {
+    await config.doInTenant(async () => {
+      await migrate()
+      const doc = await migrations.getMigrationsDoc(context.getGlobalDB())
+      expect(doc).toMatchSnapshot()
+    })
+  })
+
+  it("should skip a previously run migration", async () => {
+    await config.doInTenant(async () => {
+      const db = context.getGlobalDB()
+      await migrate()
+      const previousDoc = await migrations.getMigrationsDoc(db)
+      await migrate()
+      const currentDoc = await migrations.getMigrationsDoc(db)
+      expect(migrationFunction).toHaveBeenCalledTimes(1)
+      expect(currentDoc.test).toBe(previousDoc.test)
+    })
+  })
+})

packages/backend-core/src/objectStore/buckets/global.ts

@@ -1,5 +1,5 @@
 import env from "../../environment"
-import * as tenancy from "../../tenancy"
+import * as context from "../../context"
 import * as objectStore from "../objectStore"
 import * as cloudfront from "../cloudfront"
 
@@ -22,7 +22,7 @@ export const getGlobalFileUrl = (type: string, name: string, etag?: string) => {
 export const getGlobalFileS3Key = (type: string, name: string) => {
   let file = `${type}/${name}`
   if (env.MULTI_TENANCY) {
-    const tenantId = tenancy.getTenantId()
+    const tenantId = context.getTenantId()
     file = `${tenantId}/${file}`
   }
   return file

packages/backend-core/src/objectStore/buckets/plugins.ts

@@ -1,6 +1,6 @@
 import env from "../../environment"
 import * as objectStore from "../objectStore"
-import * as tenancy from "../../tenancy"
+import * as context from "../../context"
 import * as cloudfront from "../cloudfront"
 import { Plugin } from "@budibase/types"
 
@@ -61,7 +61,7 @@ const getPluginS3Key = (plugin: Plugin, fileName: string) => {
 export const getPluginS3Dir = (pluginName: string) => {
   let s3Key = `${pluginName}`
   if (env.MULTI_TENANCY) {
-    const tenantId = tenancy.getTenantId()
+    const tenantId = context.getTenantId()
     s3Key = `${tenantId}/${s3Key}`
   }
   if (env.CLOUDFRONT_CDN) {

packages/backend-core/src/platform/index.ts

@@ -0,0 +1,3 @@
+export * as users from "./users"
+export * as tenants from "./tenants"
+export * from "./platformDb"

packages/backend-core/src/platform/platformDb.ts

@@ -0,0 +1,6 @@
+import { StaticDatabases } from "../constants"
+import { getDB } from "../db/db"
+
+export function getPlatformDB() {
+  return getDB(StaticDatabases.PLATFORM_INFO.name)
+}

packages/backend-core/src/platform/tenants.ts

@@ -0,0 +1,101 @@
+import { StaticDatabases } from "../constants"
+import { getPlatformDB } from "./platformDb"
+import { LockName, LockOptions, LockType, Tenants } from "@budibase/types"
+import * as locks from "../redis/redlock"
+
+const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
+
+export const tenacyLockOptions: LockOptions = {
+  type: LockType.DEFAULT,
+  name: LockName.UPDATE_TENANTS_DOC,
+  ttl: 10 * 1000, // auto expire after 10 seconds
+  systemLock: true,
+}
+
+// READ
+
+export async function getTenantIds(): Promise<string[]> {
+  const tenants = await getTenants()
+  return tenants.tenantIds
+}
+
+async function getTenants(): Promise<Tenants> {
+  const db = getPlatformDB()
+  let tenants: Tenants
+
+  try {
+    tenants = await db.get(TENANT_DOC)
+  } catch (e: any) {
+    // doesn't exist yet - create
+    if (e.status === 404) {
+      tenants = await createTenantsDoc()
+    } else {
+      throw e
+    }
+  }
+
+  return tenants
+}
+
+export async function exists(tenantId: string) {
+  const tenants = await getTenants()
+  return tenants.tenantIds.indexOf(tenantId) !== -1
+}
+
+// CREATE / UPDATE
+
+function newTenantsDoc(): Tenants {
+  return {
+    _id: TENANT_DOC,
+    tenantIds: [],
+  }
+}
+
+async function createTenantsDoc(): Promise<Tenants> {
+  const db = getPlatformDB()
+  let tenants = newTenantsDoc()
+
+  try {
+    const response = await db.put(tenants)
+    tenants._rev = response.rev
+  } catch (e: any) {
+    // don't throw 409 is doc has already been created
+    if (e.status === 409) {
+      return db.get(TENANT_DOC)
+    }
+    throw e
+  }
+
+  return tenants
+}
+
+export async function addTenant(tenantId: string) {
+  const db = getPlatformDB()
+
+  // use a lock as tenant creation is conflict prone
+  await locks.doWithLock(tenacyLockOptions, async () => {
+    const tenants = await getTenants()
+
+    // write the new tenant if it doesn't already exist
+    if (tenants.tenantIds.indexOf(tenantId) === -1) {
+      tenants.tenantIds.push(tenantId)
+      await db.put(tenants)
+    }
+  })
+}
+
+// DELETE
+
+export async function removeTenant(tenantId: string) {
+  try {
+    await locks.doWithLock(tenacyLockOptions, async () => {
+      const db = getPlatformDB()
+      const tenants = await getTenants()
+      tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)
+      await db.put(tenants)
+    })
+  } catch (err) {
+    console.error(`Error removing tenant ${tenantId} from info db`, err)
+    throw err
+  }
+}

packages/backend-core/src/platform/tests/tenants.spec.ts

@@ -0,0 +1,25 @@
+import { DBTestConfiguration, structures } from "../../../tests"
+import * as tenants from "../tenants"
+
+describe("tenants", () => {
+  const config = new DBTestConfiguration()
+
+  describe("addTenant", () => {
+    it("concurrently adds multiple tenants safely", async () => {
+      const tenant1 = structures.tenant.id()
+      const tenant2 = structures.tenant.id()
+      const tenant3 = structures.tenant.id()
+
+      await Promise.all([
+        tenants.addTenant(tenant1),
+        tenants.addTenant(tenant2),
+        tenants.addTenant(tenant3),
+      ])
+
+      const tenantIds = await tenants.getTenantIds()
+      expect(tenantIds.includes(tenant1)).toBe(true)
+      expect(tenantIds.includes(tenant2)).toBe(true)
+      expect(tenantIds.includes(tenant3)).toBe(true)
+    })
+  })
+})

packages/backend-core/src/platform/users.ts

@@ -0,0 +1,90 @@
+import { getPlatformDB } from "./platformDb"
+import { DEFAULT_TENANT_ID } from "../constants"
+import env from "../environment"
+import {
+  PlatformUser,
+  PlatformUserByEmail,
+  PlatformUserById,
+  User,
+} from "@budibase/types"
+
+// READ
+
+export async function lookupTenantId(userId: string) {
+  if (!env.MULTI_TENANCY) {
+    return DEFAULT_TENANT_ID
+  }
+
+  const user = await getUserDoc(userId)
+  return user.tenantId
+}
+
+async function getUserDoc(emailOrId: string): Promise<PlatformUser> {
+  const db = getPlatformDB()
+  return db.get(emailOrId)
+}
+
+// CREATE
+
+function newUserIdDoc(id: string, tenantId: string): PlatformUserById {
+  return {
+    _id: id,
+    tenantId,
+  }
+}
+
+function newUserEmailDoc(
+  userId: string,
+  email: string,
+  tenantId: string
+): PlatformUserByEmail {
+  return {
+    _id: email,
+    userId,
+    tenantId,
+  }
+}
+
+/**
+ * Add a new user id or email doc if it doesn't exist.
+ */
+async function addUserDoc(emailOrId: string, newDocFn: () => PlatformUser) {
+  const db = getPlatformDB()
+  let user: PlatformUser
+
+  try {
+    await db.get(emailOrId)
+  } catch (e: any) {
+    if (e.status === 404) {
+      user = newDocFn()
+      await db.put(user)
+    } else {
+      throw e
+    }
+  }
+}
+
+export async function addUser(tenantId: string, userId: string, email: string) {
+  await Promise.all([
+    addUserDoc(userId, () => newUserIdDoc(userId, tenantId)),
+    addUserDoc(email, () => newUserEmailDoc(userId, email, tenantId)),
+  ])
+}
+
+// DELETE
+
+export async function removeUser(user: User) {
+  const db = getPlatformDB()
+  const keys = [user._id!, user.email]
+  const userDocs = await db.allDocs({
+    keys,
+    include_docs: true,
+  })
+  const toDelete = userDocs.rows.map((row: any) => {
+    return {
+      ...row.doc,
+      _deleted: true,
+    }
+  })
+  await db.bulkDocs(toDelete)
+}

packages/backend-core/src/queue/queue.ts

@@ -4,7 +4,6 @@ import { JobQueue } from "./constants"
 import InMemoryQueue from "./inMemoryQueue"
 import BullQueue from "bull"
 import { addListeners, StalledFn } from "./listeners"
-const { opts: redisOpts, redisProtocolUrl } = getRedisOptions()
 
 const CLEANUP_PERIOD_MS = 60 * 1000
 let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
@@ -20,6 +19,7 @@ export function createQueue<T>(
   jobQueue: JobQueue,
   opts: { removeStalledCb?: StalledFn } = {}
 ): BullQueue.Queue<T> {
+  const { opts: redisOpts, redisProtocolUrl } = getRedisOptions()
   const queueConfig: any = redisProtocolUrl || { redis: redisOpts }
   let queue: any
   if (!env.isTest()) {

packages/backend-core/src/redis/index.ts

@@ -3,4 +3,4 @@
 export { default as Client } from "./redis"
 export * as utils from "./utils"
 export * as clients from "./init"
-export * as redlock from "./redlock"
+export * as locks from "./redlock"

packages/backend-core/src/redis/init.ts

@@ -20,13 +20,17 @@ async function init() {
   ).init()
 }
 
-process.on("exit", async () => {
+export async function shutdown() {
   if (userClient) await userClient.finish()
   if (sessionClient) await sessionClient.finish()
   if (appClient) await appClient.finish()
   if (cacheClient) await cacheClient.finish()
   if (writethroughClient) await writethroughClient.finish()
   if (lockClient) await lockClient.finish()
+}
+
+process.on("exit", async () => {
+  await shutdown()
 })
 
 export async function getUserClient() {

packages/backend-core/src/redis/redis.ts

@@ -1,6 +1,6 @@
 import env from "../environment"
 // ioredis mock is all in memory
-const Redis = env.isTest() ? require("ioredis-mock") : require("ioredis")
+const Redis = env.MOCK_REDIS ? require("ioredis-mock") : require("ioredis")
 import {
   addDbPrefix,
   removeDbPrefix,
@@ -17,8 +17,13 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT
 // for testing just generate the client once
 let CLOSED = false
 let CLIENTS: { [key: number]: any } = {}
-// if in test always connected
+
-let CONNECTED = env.isTest()
+let CONNECTED = false
+
+// mock redis always connected
+if (env.MOCK_REDIS) {
+  CONNECTED = true
+}
 
 function pickClient(selectDb: number): any {
   return CLIENTS[selectDb]
@@ -57,7 +62,7 @@ function init(selectDb = DEFAULT_SELECT_DB) {
     return
   }
   // testing uses a single in memory client
-  if (env.isTest()) {
+  if (env.MOCK_REDIS) {
     CLIENTS[selectDb] = new Redis(getRedisOptions())
   }
   // start the timer - only allowed 5 seconds to connect
@@ -86,6 +91,11 @@ function init(selectDb = DEFAULT_SELECT_DB) {
   }
   // attach handlers
   client.on("end", (err: Error) => {
+    if (env.isTest()) {
+      // don't try to re-connect in test env
+      // allow the process to exit
+      return
+    }
     connectionError(selectDb, timeout, err)
   })
   client.on("error", (err: Error) => {

packages/backend-core/src/redis/redlock.ts

@@ -1,29 +1,22 @@
 import Redlock, { Options } from "redlock"
 import { getLockClient } from "./init"
 import { LockOptions, LockType } from "@budibase/types"
-import * as tenancy from "../tenancy"
+import * as context from "../context"
-
+import env from "../environment"
-let noRetryRedlock: Redlock | undefined
 
 const getClient = async (type: LockType): Promise<Redlock> => {
+  if (env.isTest() && type !== LockType.TRY_ONCE) {
+    return newRedlock(OPTIONS.TEST)
+  }
   switch (type) {
     case LockType.TRY_ONCE: {
-      if (!noRetryRedlock) {
+      return newRedlock(OPTIONS.TRY_ONCE)
-        noRetryRedlock = await newRedlock(OPTIONS.TRY_ONCE)
-      }
-      return noRetryRedlock
     }
     case LockType.DEFAULT: {
-      if (!noRetryRedlock) {
+      return newRedlock(OPTIONS.DEFAULT)
-        noRetryRedlock = await newRedlock(OPTIONS.DEFAULT)
-      }
-      return noRetryRedlock
     }
     case LockType.DELAY_500: {
-      if (!noRetryRedlock) {
+      return newRedlock(OPTIONS.DELAY_500)
-        noRetryRedlock = await newRedlock(OPTIONS.DELAY_500)
-      }
-      return noRetryRedlock
     }
     default: {
       throw new Error(`Could not get redlock client: ${type}`)
@@ -36,6 +29,11 @@ export const OPTIONS = {
     // immediately throws an error if the lock is already held
     retryCount: 0,
   },
+  TEST: {
+    // higher retry count in unit tests
+    // due to high contention.
+    retryCount: 100,
+  },
   DEFAULT: {
     // the expected clock drift; for more details
     // see http://redis.io/topics/distlock
@@ -69,28 +67,38 @@ export const doWithLock = async (opts: LockOptions, task: any) => {
   const redlock = await getClient(opts.type)
   let lock
   try {
-    // aquire lock
+    // determine lock name
-    let name: string = `lock:${tenancy.getTenantId()}_${opts.name}`
+    // by default use the tenantId for uniqueness, unless using a system lock
+    const prefix = opts.systemLock ? "system" : context.getTenantId()
+    let name: string = `lock:${prefix}_${opts.name}`
+
+    // add additional unique name if required
     if (opts.nameSuffix) {
       name = name + `_${opts.nameSuffix}`
     }
+
+    // create the lock
     lock = await redlock.lock(name, opts.ttl)
+
     // perform locked task
     // need to await to ensure completion before unlocking
     const result = await task()
     return result
   } catch (e: any) {
-    console.log("lock error")
+    console.warn("lock error")
     // lock limit exceeded
     if (e.name === "LockError") {
       if (opts.type === LockType.TRY_ONCE) {
         // don't throw for try-once locks, they will always error
         // due to retry count (0) exceeded
+        console.warn(e)
         return
       } else {
+        console.error(e)
         throw e
       }
     } else {
+      console.error(e)
       throw e
     }
   } finally {

packages/backend-core/src/redis/utils.ts

@@ -2,8 +2,6 @@ import env from "../environment"
 
 const SLOT_REFRESH_MS = 2000
 const CONNECT_TIMEOUT_MS = 10000
-const REDIS_URL = !env.REDIS_URL ? "localhost:6379" : env.REDIS_URL
-const REDIS_PASSWORD = !env.REDIS_PASSWORD ? "budibase" : env.REDIS_PASSWORD
 export const SEPARATOR = "-"
 
 /**
@@ -60,8 +58,8 @@ export enum SelectableDatabase {
 }
 
 export function getRedisOptions(clustered = false) {
-  let password = REDIS_PASSWORD
+  let password = env.REDIS_PASSWORD
-  let url: string[] | string = REDIS_URL.split("//")
+  let url: string[] | string = env.REDIS_URL.split("//")
   // get rid of the protocol
   url = url.length > 1 ? url[1] : url[0]
   // check for a password etc
@@ -78,8 +76,8 @@ export function getRedisOptions(clustered = false) {
   let redisProtocolUrl
 
   // fully qualified redis URL
-  if (/rediss?:\/\//.test(REDIS_URL)) {
+  if (/rediss?:\/\//.test(env.REDIS_URL)) {
-    redisProtocolUrl = REDIS_URL
+    redisProtocolUrl = env.REDIS_URL
   }
 
   const opts: any = {

packages/backend-core/src/tenancy/db.ts

@@ -0,0 +1,6 @@
+import { getDB } from "../db/db"
+import { getGlobalDBName } from "../context"
+
+export function getTenantDB(tenantId: string) {
+  return getDB(getGlobalDBName(tenantId))
+}

packages/backend-core/src/tenancy/index.ts

@@ -1,2 +1,2 @@
-export * from "../context"
+export * from "./db"
 export * from "./tenancy"

packages/backend-core/src/tenancy/tenancy.ts

@@ -1,4 +1,3 @@
-import { doWithDB, getGlobalDBName } from "../db"
 import {
   DEFAULT_TENANT_ID,
   getTenantId,
@@ -11,10 +10,7 @@ import {
   TenantResolutionStrategy,
   GetTenantIdOptions,
 } from "@budibase/types"
-import { Header, StaticDatabases } from "../constants"
+import { Header } from "../constants"
-
-const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
-const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
 
 export function addTenantToUrl(url: string) {
   const tenantId = getTenantId()
@@ -27,89 +23,6 @@ export function addTenantToUrl(url: string) {
   return url
 }
 
-export async function doesTenantExist(tenantId: string) {
-  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
-    let tenants
-    try {
-      tenants = await db.get(TENANT_DOC)
-    } catch (err) {
-      // if theres an error the doc doesn't exist, no tenants exist
-      return false
-    }
-    return (
-      tenants &&
-      Array.isArray(tenants.tenantIds) &&
-      tenants.tenantIds.indexOf(tenantId) !== -1
-    )
-  })
-}
-
-export async function tryAddTenant(
-  tenantId: string,
-  userId: string,
-  email: string,
-  afterCreateTenant: () => Promise<void>
-) {
-  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
-    const getDoc = async (id: string) => {
-      if (!id) {
-        return null
-      }
-      try {
-        return await db.get(id)
-      } catch (err) {
-        return { _id: id }
-      }
-    }
-    let [tenants, userIdDoc, emailDoc] = await Promise.all([
-      getDoc(TENANT_DOC),
-      getDoc(userId),
-      getDoc(email),
-    ])
-    if (!Array.isArray(tenants.tenantIds)) {
-      tenants = {
-        _id: TENANT_DOC,
-        tenantIds: [],
-      }
-    }
-    let promises = []
-    if (userIdDoc) {
-      userIdDoc.tenantId = tenantId
-      promises.push(db.put(userIdDoc))
-    }
-    if (emailDoc) {
-      emailDoc.tenantId = tenantId
-      emailDoc.userId = userId
-      promises.push(db.put(emailDoc))
-    }
-    if (tenants.tenantIds.indexOf(tenantId) === -1) {
-      tenants.tenantIds.push(tenantId)
-      promises.push(db.put(tenants))
-      await afterCreateTenant()
-    }
-    await Promise.all(promises)
-  })
-}
-
-export function doWithGlobalDB(tenantId: string, cb: any) {
-  return doWithDB(getGlobalDBName(tenantId), cb)
-}
-
-export async function lookupTenantId(userId: string) {
-  return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: any) => {
-    let tenantId = env.MULTI_TENANCY ? DEFAULT_TENANT_ID : null
-    try {
-      const doc = await db.get(userId)
-      if (doc && doc.tenantId) {
-        tenantId = doc.tenantId
-      }
-    } catch (err) {
-      // just return the default
-    }
-    return tenantId
-  })
-}
-
 export const isUserInAppTenant = (appId: string, user?: any) => {
   let userTenantId
   if (user) {
@@ -121,19 +34,6 @@ export const isUserInAppTenant = (appId: string, user?: any) => {
   return tenantId === userTenantId
 }
 
-export async function getTenantIds() {
-  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
-    let tenants
-    try {
-      tenants = await db.get(TENANT_DOC)
-    } catch (err) {
-      // if theres an error the doc doesn't exist, no tenants exist
-      return []
-    }
-    return (tenants && tenants.tenantIds) || []
-  })
-}
-
 const ALL_STRATEGIES = Object.values(TenantResolutionStrategy)
 
 export const getTenantIDFromCtx = (

packages/backend-core/src/users.ts

@@ -8,6 +8,7 @@ import {
 } from "./db"
 import { BulkDocsResponse, User } from "@budibase/types"
 import { getGlobalDB } from "./context"
+import * as context from "./context"
 
 export const bulkGetGlobalUsersById = async (userIds: string[]) => {
   const db = getGlobalDB()
@@ -24,6 +25,11 @@ export const bulkUpdateGlobalUsers = async (users: User[]) => {
   return (await db.bulkDocs(users)) as BulkDocsResponse
 }
 
+export async function getById(id: string): Promise<User> {
+  const db = context.getGlobalDB()
+  return db.get(id)
+}
+
 /**
  * Given an email address this will use a view to search through
  * all the users to find one with this email address.

packages/backend-core/src/utils/tests/utils.spec.ts

@@ -1,21 +1,12 @@
-import { structures } from "../../../tests"
+import { structures, DBTestConfiguration } from "../../../tests"
 import * as utils from "../../utils"
-import * as events from "../../events"
 import * as db from "../../db"
 import { Header } from "../../constants"
-import { doInTenant } from "../../context"
 import { newid } from "../../utils"
+import env from "../../environment"
 
 describe("utils", () => {
-  describe("platformLogout", () => {
+  const config = new DBTestConfiguration()
-    it("should call platform logout", async () => {
-      await doInTenant(structures.tenant.id(), async () => {
-        const ctx = structures.koa.newContext()
-        await utils.platformLogout({ ctx, userId: "test" })
-        expect(events.auth.logout).toBeCalledTimes(1)
-      })
-    })
-  })
 
   describe("getAppIdFromCtx", () => {
     it("gets appId from header", async () => {
@@ -50,21 +41,28 @@ describe("utils", () => {
     })
 
     it("gets appId from url", async () => {
-      const ctx = structures.koa.newContext()
+      await config.doInTenant(async () => {
-      const expected = db.generateAppID()
+        const url = "http://test.com"
-      const app = structures.apps.app(expected)
+        env._set("PLATFORM_URL", url)
 
-      // set custom url
+        const ctx = structures.koa.newContext()
-      const appUrl = newid()
+        ctx.host = `${config.tenantId}.test.com`
-      app.url = `/${appUrl}`
-      ctx.path = `/app/${appUrl}`
 
-      // save the app
+        const expected = db.generateAppID(config.tenantId)
-      const database = db.getDB(expected)
+        const app = structures.apps.app(expected)
-      await database.put(app)
 
-      const actual = await utils.getAppIdFromCtx(ctx)
+        // set custom url
-      expect(actual).toBe(expected)
+        const appUrl = newid()
+        app.url = `/${appUrl}`
+        ctx.path = `/app/${appUrl}`
+
+        // save the app
+        const database = db.getDB(expected)
+        await database.put(app)
+
+        const actual = await utils.getAppIdFromCtx(ctx)
+        expect(actual).toBe(expected)
+      })
     })
 
     it("doesn't get appId from url when previewing", async () => {

packages/backend-core/src/utils/utils.ts

@@ -2,23 +2,15 @@ import { getAllApps, queryGlobalView } from "../db"
 import { options } from "../middleware/passport/jwt"
 import {
   Header,
-  Cookie,
   MAX_VALID_DATE,
   DocumentType,
   SEPARATOR,
   ViewName,
 } from "../constants"
 import env from "../environment"
-import * as userCache from "../cache/user"
-import { getSessionsForUser, invalidateSessions } from "../security/sessions"
-import * as events from "../events"
 import * as tenancy from "../tenancy"
-import {
+import * as context from "../context"
-  App,
+import { App, Ctx, TenantResolutionStrategy } from "@budibase/types"
-  Ctx,
-  PlatformLogoutOpts,
-  TenantResolutionStrategy,
-} from "@budibase/types"
 import { SetOption } from "cookies"
 const jwt = require("jsonwebtoken")
 
@@ -38,7 +30,7 @@ export async function resolveAppUrl(ctx: Ctx) {
   const appUrl = ctx.path.split("/")[2]
   let possibleAppUrl = `/${appUrl.toLowerCase()}`
 
-  let tenantId: string | null = tenancy.getTenantId()
+  let tenantId: string | null = context.getTenantId()
   if (env.MULTI_TENANCY) {
     // always use the tenant id from the subdomain in multi tenancy
     // this ensures the logged-in user tenant id doesn't overwrite
@@ -49,7 +41,7 @@ export async function resolveAppUrl(ctx: Ctx) {
   }
 
   // search prod apps for a url that matches
-  const apps: App[] = await tenancy.doInTenant(tenantId, () =>
+  const apps: App[] = await context.doInTenant(tenantId, () =>
     getAllApps({ dev: false })
   )
   const app = apps.filter(
@@ -222,35 +214,6 @@ export async function getBuildersCount() {
   return builders.length
 }
 
-/**
- * Logs a user out from budibase. Re-used across account portal and builder.
- */
-export async function platformLogout(opts: PlatformLogoutOpts) {
-  const ctx = opts.ctx
-  const userId = opts.userId
-  const keepActiveSession = opts.keepActiveSession
-
-  if (!ctx) throw new Error("Koa context must be supplied to logout.")
-
-  const currentSession = getCookie(ctx, Cookie.Auth)
-  let sessions = await getSessionsForUser(userId)
-
-  if (keepActiveSession) {
-    sessions = sessions.filter(
-      session => session.sessionId !== currentSession.sessionId
-    )
-  } else {
-    // clear cookies
-    clearCookie(ctx, Cookie.Auth)
-    clearCookie(ctx, Cookie.CurrentApp)
-  }
-
-  const sessionIds = sessions.map(({ sessionId }) => sessionId)
-  await invalidateSessions(userId, { sessionIds, reason: "logout" })
-  await events.auth.logout()
-  await userCache.invalidateUser(userId)
-}
-
 export function timeout(timeMs: number) {
   return new Promise(resolve => setTimeout(resolve, timeMs))
 }

packages/backend-core/tests/jestEnv.ts

@@ -1,23 +1,6 @@
-import env from "../src/environment"
+process.env.SELF_HOSTED = "1"
-import { mocks } from "./utilities"
+process.env.MULTI_TENANCY = "1"
-
+process.env.NODE_ENV = "jest"
-// must explicitly enable fetch mock
+process.env.MOCK_REDIS = "1"
-mocks.fetch.enable()
+process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
-
+process.env.ENABLE_4XX_HTTP_LOGGING = "0"
-// mock all dates to 2020-01-01T00:00:00.000Z
-// use tk.reset() to use real dates in individual tests
-import tk from "timekeeper"
-tk.freeze(mocks.date.MOCK_DATE)
-
-env._set("SELF_HOSTED", "1")
-env._set("NODE_ENV", "jest")
-
-if (!process.env.DEBUG) {
-  global.console.log = jest.fn() // console.log are ignored in tests
-}
-
-if (!process.env.CI) {
-  // set a longer timeout in dev for debugging
-  // 100 seconds
-  jest.setTimeout(100000)
-}

packages/backend-core/tests/jestSetup.ts

@@ -1,4 +1,23 @@
+import "./logging"
 import env from "../src/environment"
-import { testContainerUtils } from "./utilities"
+import { mocks, testContainerUtils } from "./utilities"
+
+// must explicitly enable fetch mock
+mocks.fetch.enable()
+
+// mock all dates to 2020-01-01T00:00:00.000Z
+// use tk.reset() to use real dates in individual tests
+import tk from "timekeeper"
+tk.freeze(mocks.date.MOCK_DATE)
+
+if (!process.env.DEBUG) {
+  console.log = jest.fn() // console.log are ignored in tests
+}
+
+if (!process.env.CI) {
+  // set a longer timeout in dev for debugging
+  // 100 seconds
+  jest.setTimeout(100000)
+}
 
 testContainerUtils.setupEnv(env)

packages/backend-core/tests/logging.ts

@@ -0,0 +1,34 @@
+export enum LogLevel {
+  TRACE = "trace",
+  DEBUG = "debug",
+  INFO = "info",
+  WARN = "warn",
+  ERROR = "error",
+}
+
+const LOG_INDEX: { [key in LogLevel]: number } = {
+  [LogLevel.TRACE]: 1,
+  [LogLevel.DEBUG]: 2,
+  [LogLevel.INFO]: 3,
+  [LogLevel.WARN]: 4,
+  [LogLevel.ERROR]: 5,
+}
+
+const setIndex = LOG_INDEX[process.env.LOG_LEVEL as LogLevel]
+
+if (setIndex > LOG_INDEX.trace) {
+  global.console.trace = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.debug) {
+  global.console.debug = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.info) {
+  global.console.info = jest.fn()
+  global.console.log = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.warn) {
+  global.console.warn = jest.fn()
+}

packages/backend-core/tests/utilities/DBTestConfiguration.ts

@@ -0,0 +1,36 @@
+import "./mocks"
+import * as structures from "./structures"
+import * as testEnv from "./testEnv"
+import * as context from "../../src/context"
+
+class DBTestConfiguration {
+  tenantId: string
+
+  constructor() {
+    // db tests need to be multi tenant to prevent conflicts
+    testEnv.multiTenant()
+    this.tenantId = structures.tenant.id()
+  }
+
+  newTenant() {
+    this.tenantId = structures.tenant.id()
+  }
+
+  // TENANCY
+
+  doInTenant(task: any) {
+    return context.doInTenant(this.tenantId, () => {
+      return task()
+    })
+  }
+
+  getTenantId() {
+    try {
+      return context.getTenantId()
+    } catch (e) {
+      return this.tenantId!
+    }
+  }
+}
+
+export default DBTestConfiguration

packages/backend-core/tests/utilities/db.ts

@@ -1,9 +0,0 @@
-import * as db from "../../src/db"
-
-const dbConfig = {
-  inMemory: true,
-}
-
-export const init = () => {
-  db.init(dbConfig)
-}

packages/backend-core/tests/utilities/index.ts

@@ -4,5 +4,4 @@ export { generator } from "./structures"
 export * as testEnv from "./testEnv"
 export * as testContainerUtils from "./testContainerUtils"
 
-import * as dbConfig from "./db"
+export { default as DBTestConfiguration } from "./DBTestConfiguration"
-dbConfig.init()

packages/backend-core/tests/utilities/mocks/accounts.ts

@@ -1,13 +0,0 @@
-const mockGetAccount = jest.fn()
-const mockGetAccountByTenantId = jest.fn()
-const mockGetStatus = jest.fn()
-
-jest.mock("../../../src/cloud/accounts", () => ({
-  getAccount: mockGetAccount,
-  getAccountByTenantId: mockGetAccountByTenantId,
-  getStatus: mockGetStatus,
-}))
-
-export const getAccount = mockGetAccount
-export const getAccountByTenantId = mockGetAccountByTenantId
-export const getStatus = mockGetStatus

packages/backend-core/tests/utilities/mocks/index.ts

@@ -1,4 +1,7 @@
-export * as accounts from "./accounts"
+jest.mock("../../../src/accounts")
+import * as _accounts from "../../../src/accounts"
+export const accounts = jest.mocked(_accounts)
+
 export * as date from "./date"
 export * as licenses from "./licenses"
 export { default as fetch } from "./fetch"

packages/backend-core/tests/utilities/structures/accounts.ts

@@ -1,6 +1,15 @@
 import { generator, uuid } from "."
 import * as db from "../../../src/db/utils"
-import { Account, AuthType, CloudAccount, Hosting } from "@budibase/types"
+import {
+  Account,
+  AccountSSOProvider,
+  AccountSSOProviderType,
+  AuthType,
+  CloudAccount,
+  Hosting,
+  SSOAccount,
+} from "@budibase/types"
+import _ from "lodash"
 
 export const account = (): Account => {
   return {
@@ -27,3 +36,28 @@ export const cloudAccount = (): CloudAccount => {
     budibaseUserId: db.generateGlobalUserID(),
   }
 }
+
+function providerType(): AccountSSOProviderType {
+  return _.sample(
+    Object.values(AccountSSOProviderType)
+  ) as AccountSSOProviderType
+}
+
+function provider(): AccountSSOProvider {
+  return _.sample(Object.values(AccountSSOProvider)) as AccountSSOProvider
+}
+
+export function ssoAccount(): SSOAccount {
+  return {
+    ...cloudAccount(),
+    authType: AuthType.SSO,
+    oauth2: {
+      accessToken: generator.string(),
+      refreshToken: generator.string(),
+    },
+    pictureUrl: generator.url(),
+    provider: provider(),
+    providerType: providerType(),
+    thirdPartyProfile: {},
+  }
+}

packages/backend-core/tests/utilities/structures/index.ts

@@ -5,8 +5,10 @@ export const generator = new Chance()
 
 export * as accounts from "./accounts"
 export * as apps from "./apps"
+export * as db from "./db"
 export * as koa from "./koa"
 export * as licenses from "./licenses"
 export * as plugins from "./plugins"
+export * as sso from "./sso"
 export * as tenant from "./tenants"
-export * as db from "./db"
+export * as users from "./users"

packages/backend-core/tests/utilities/structures/sso.ts

@@ -0,0 +1,100 @@
+import {
+  GoogleInnerConfig,
+  JwtClaims,
+  OIDCInnerConfig,
+  OIDCWellKnownConfig,
+  SSOAuthDetails,
+  SSOProfile,
+  SSOProviderType,
+  User,
+} from "@budibase/types"
+import { uuid, generator, users, email } from "./index"
+import _ from "lodash"
+
+export function providerType(): SSOProviderType {
+  return _.sample(Object.values(SSOProviderType)) as SSOProviderType
+}
+
+export function ssoProfile(user?: User): SSOProfile {
+  if (!user) {
+    user = users.user()
+  }
+  return {
+    id: user._id!,
+    name: {
+      givenName: user.firstName,
+      familyName: user.lastName,
+    },
+    _json: {
+      email: user.email,
+      picture: "http://test.com",
+    },
+    provider: generator.string(),
+  }
+}
+
+export function authDetails(user?: User): SSOAuthDetails {
+  if (!user) {
+    user = users.user()
+  }
+
+  const userId = user._id || uuid()
+  const provider = generator.string()
+
+  const profile = ssoProfile(user)
+  profile.provider = provider
+  profile.id = userId
+
+  return {
+    email: user.email,
+    oauth2: {
+      refreshToken: generator.string(),
+      accessToken: generator.string(),
+    },
+    profile,
+    provider,
+    providerType: providerType(),
+    userId,
+  }
+}
+
+// OIDC
+
+export function oidcConfig(): OIDCInnerConfig {
+  return {
+    uuid: uuid(),
+    activated: true,
+    logo: "",
+    name: generator.string(),
+    configUrl: "http://someconfigurl",
+    clientID: generator.string(),
+    clientSecret: generator.string(),
+  }
+}
+
+// response from .well-known/openid-configuration
+export function oidcWellKnownConfig(): OIDCWellKnownConfig {
+  return {
+    issuer: generator.string(),
+    authorization_endpoint: generator.url(),
+    token_endpoint: generator.url(),
+    userinfo_endpoint: generator.url(),
+  }
+}
+
+export function jwtClaims(): JwtClaims {
+  return {
+    email: email(),
+    preferred_username: email(),
+  }
+}
+
+// GOOGLE
+
+export function googleConfig(): GoogleInnerConfig {
+  return {
+    activated: true,
+    clientID: generator.string(),
+    clientSecret: generator.string(),
+  }
+}

packages/backend-core/tests/utilities/structures/users.ts

@@ -0,0 +1,70 @@
+import { generator } from "../"
+import {
+  AdminUser,
+  BuilderUser,
+  SSOAuthDetails,
+  SSOUser,
+  User,
+} from "@budibase/types"
+import { v4 as uuid } from "uuid"
+import * as sso from "./sso"
+
+export const newEmail = () => {
+  return `${uuid()}@test.com`
+}
+
+export const user = (userProps?: any): User => {
+  return {
+    email: newEmail(),
+    password: "test",
+    roles: { app_test: "admin" },
+    firstName: generator.first(),
+    lastName: generator.last(),
+    pictureUrl: "http://test.com",
+    ...userProps,
+  }
+}
+
+export const adminUser = (userProps?: any): AdminUser => {
+  return {
+    ...user(userProps),
+    admin: {
+      global: true,
+    },
+    builder: {
+      global: true,
+    },
+  }
+}
+
+export const builderUser = (userProps?: any): BuilderUser => {
+  return {
+    ...user(userProps),
+    builder: {
+      global: true,
+    },
+  }
+}
+
+export function ssoUser(
+  opts: { user?: any; details?: SSOAuthDetails } = {}
+): SSOUser {
+  const base = user(opts.user)
+  delete base.password
+
+  if (!opts.details) {
+    opts.details = sso.authDetails(base)
+  }
+
+  return {
+    ...base,
+    forceResetPassword: false,
+    oauth2: opts.details?.oauth2,
+    provider: opts.details?.provider!,
+    providerType: opts.details?.providerType!,
+    thirdPartyProfile: {
+      email: base.email,
+      picture: base.pictureUrl,
+    },
+  }
+}

packages/backend-core/tests/utilities/testContainerUtils.ts

@@ -34,12 +34,17 @@ function getMinioConfig() {
   return getContainerInfo("minio-service", 9000)
 }
 
+function getRedisConfig() {
+  return getContainerInfo("redis-service", 6379)
+}
+
 export function setupEnv(...envs: any[]) {
   const configs = [
     { key: "COUCH_DB_PORT", value: getCouchConfig().port },
     { key: "COUCH_DB_URL", value: getCouchConfig().url },
     { key: "MINIO_PORT", value: getMinioConfig().port },
     { key: "MINIO_URL", value: getMinioConfig().url },
+    { key: "REDIS_URL", value: getRedisConfig().url },
   ]
 
   for (const config of configs.filter(x => !!x.value)) {

packages/backend-core/tests/utilities/testEnv.ts

@@ -1,12 +1,12 @@
 import env from "../../src/environment"
-import * as tenancy from "../../src/tenancy"
+import * as context from "../../src/context"
-import { newid } from "../../src/utils"
+import * as structures from "./structures"
 
 // TENANCY
 
 export async function withTenant(task: (tenantId: string) => any) {
-  const tenantId = newid()
+  const tenantId = structures.tenant.id()
-  return tenancy.doInTenant(tenantId, async () => {
+  return context.doInTenant(tenantId, async () => {
     await task(tenantId)
   })
 }
@@ -19,6 +19,14 @@ export function multiTenant() {
   env._set("MULTI_TENANCY", 1)
 }
 
+export function selfHosted() {
+  env._set("SELF_HOSTED", 1)
+}
+
+export function cloudHosted() {
+  env._set("SELF_HOSTED", 0)
+}
+
 // NODE
 
 export function nodeDev() {

packages/backend-core/yarn.lock

@@ -1197,10 +1197,10 @@
   dependencies:
     "@types/istanbul-lib-report" "*"
 
-"@types/jest@27.5.1":
+"@types/jest@28.1.1":
-  version "27.5.1"
+  version "28.1.1"
-  resolved "https://registry.yarnpkg.com/@types/jest/-/jest-27.5.1.tgz#2c8b6dc6ff85c33bcd07d0b62cb3d19ddfdb3ab9"
+  resolved "https://registry.yarnpkg.com/@types/jest/-/jest-28.1.1.tgz#8c9ba63702a11f8c386ee211280e8b68cb093cd1"
-  integrity sha512-fUy7YRpT+rHXto1YlL+J9rs0uLGyiqVt3ZOTQR+4ROc47yNl8WLdVLgUloBRhOxP1PZvguHl44T3H0wAWxahYQ==
+  integrity sha512-C2p7yqleUKtCkVjlOur9BWVA4HgUQmEj/HWCt5WzZ5mLXrWnyIfl0wGuArc+kBXsy0ZZfLp+7dywB4HtSVYGVA==
   dependencies:
     jest-matcher-utils "^27.0.0"
     pretty-format "^27.0.0"

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.18-alpha.6",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/string-templates": "2.3.17-alpha.4",
+    "@budibase/string-templates": "2.3.18-alpha.6",
     "@spectrum-css/accordion": "3.0.24",
     "@spectrum-css/actionbutton": "1.0.1",
     "@spectrum-css/actiongroup": "1.0.1",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.18-alpha.6",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -58,10 +58,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.3.17-alpha.4",
+    "@budibase/bbui": "2.3.18-alpha.6",
-    "@budibase/client": "2.3.17-alpha.4",
+    "@budibase/client": "2.3.18-alpha.6",
-    "@budibase/frontend-core": "2.3.17-alpha.4",
+    "@budibase/frontend-core": "2.3.18-alpha.6",
-    "@budibase/string-templates": "2.3.17-alpha.4",
+    "@budibase/string-templates": "2.3.18-alpha.6",
     "@fortawesome/fontawesome-svg-core": "^6.2.1",
     "@fortawesome/free-brands-svg-icons": "^6.2.1",
     "@fortawesome/free-solid-svg-icons": "^6.2.1",

packages/builder/setup.js

@@ -19,6 +19,7 @@ process.env.COUCH_DB_USER = "budibase"
 process.env.COUCH_DB_PASSWORD = "budibase"
 process.env.INTERNAL_API_KEY = "budibase"
 process.env.ALLOW_DEV_AUTOMATIONS = 1
+process.env.MOCK_REDIS = 1
 
 // Stop info logs polluting test outputs
 process.env.LOG_LEVEL = "error"

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -183,6 +183,7 @@
     bind:this={popover}
     anchor={popoverAnchor}
     maxWidth={300}
+    dismissible={false}
   >
     <Layout gap="S">
       <div class="helper">

packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte

@@ -254,8 +254,8 @@
               {:else if filter.type === "datetime"}
                 <DatePicker
                   disabled={filter.noValue}
-                  enableTime={!getSchema(filter).dateOnly}
+                  enableTime={!getSchema(filter)?.dateOnly}
-                  timeOnly={getSchema(filter).timeOnly}
+                  timeOnly={getSchema(filter)?.timeOnly}
                   bind:value={filter.value}
                 />
               {:else}

packages/builder/src/components/start/CreateAppModal.svelte

@@ -26,7 +26,15 @@
 
   const values = writable({ name: "", url: null })
   const validation = createValidationStore()
-  $: validation.check($values)
+
+  $: {
+    const { name, url } = $values
+
+    validation.check({
+      name,
+      url: url?.[0] === "/" ? url.substring(1, url.length) : url,
+    })
+  }
 
   onMount(async () => {
     const lastChar = $auth.user?.firstName
@@ -87,7 +95,11 @@
     appValidation.url(validation, { apps: applications })
     appValidation.file(validation, { template })
     // init validation
-    validation.check($values)
+    const { name, url } = $values
+    validation.check({
+      name,
+      url: url?.[0] === "/" ? url.substring(1, url.length) : url,
+    })
   }
 
   async function createNewApp() {

packages/builder/src/components/start/UpdateAppModal.svelte

@@ -23,14 +23,25 @@
   })
   const validation = createValidationStore()
 
-  $: validation.check($values)
+  $: {
+    const { name, url } = $values
+
+    validation.check({
+      name,
+      url: url?.[0] === "/" ? url.substring(1, url.length) : url,
+    })
+  }
 
   const setupValidation = async () => {
     const applications = svelteGet(apps)
     appValidation.name(validation, { apps: applications, currentApp: app })
     appValidation.url(validation, { apps: applications, currentApp: app })
     // init validation
-    validation.check($values)
+    const { name, url } = $values
+    validation.check({
+      name,
+      url: url?.[0] === "/" ? url.substring(1, url.length) : url,
+    })
   }
 
   async function updateApp() {

packages/builder/src/constants/index.js

@@ -46,7 +46,7 @@ export const LAYOUT_NAMES = {
 // one or more word characters and whitespace
 export const APP_NAME_REGEX = /^[\w\s]+$/
 // zero or more non-whitespace characters
-export const APP_URL_REGEX = /^\S*$/
+export const APP_URL_REGEX = /^[0-9a-zA-Z-_]+$/
 
 export const DefaultAppTheme = {
   primaryColor: "var(--spectrum-global-color-blue-600)",

packages/builder/src/helpers/userInitials.js

@@ -0,0 +1,13 @@
+const getUserInitials = user => {
+  if (user.firstName && user.lastName) {
+    return user.firstName[0] + user.lastName[0]
+  } else if (user.firstName) {
+    return user.firstName[0]
+  } else if (user.email) {
+    return user.email[0]
+  }
+
+  return "U"
+}
+
+export default getUserInitials

packages/builder/src/helpers/validation/yup/app.js

@@ -62,11 +62,9 @@ export const url = (validation, { apps, currentApp } = { apps: [] }) => {
         }
         // make it clear that this is a url path and cannot be a full url
         return (
-          value.startsWith("/") &&
           !value.includes("http") &&
           !value.includes("www") &&
-          !value.includes(".") &&
+          !value.includes(".")
-          value.length > 1 // just '/' is not valid
         )
       })
   )

packages/builder/src/pages/builder/portal/_components/UserDropdown.svelte

@@ -30,9 +30,11 @@
     My profile
   </MenuItem>
   <MenuItem icon="Moon" on:click={() => themeModal.show()}>Theme</MenuItem>
-  <MenuItem icon="LockClosed" on:click={() => updatePasswordModal.show()}>
+  {#if !$auth.isSSO}
-    Update password
+    <MenuItem icon="LockClosed" on:click={() => updatePasswordModal.show()}>
-  </MenuItem>
+      Update password
+    </MenuItem>
+  {/if}
   <MenuItem icon="Key" on:click={() => apiKeyModal.show()}>
     View API key
   </MenuItem>

packages/builder/src/pages/builder/portal/apps/onboarding/_components/NamePanel.svelte

@@ -1,6 +1,7 @@
 <script>
   import { Button, FancyForm, FancyInput } from "@budibase/bbui"
   import PanelHeader from "./PanelHeader.svelte"
+  import { APP_URL_REGEX } from "constants"
 
   export let name = ""
   export let url = ""
@@ -25,6 +26,10 @@
     if (url.length < 1) {
       return "URL must be provided"
     }
+
+    if (!APP_URL_REGEX.test(url)) {
+      return "Invalid URL"
+    }
   }
 </script>
 

packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/ActionsRenderer.svelte

@@ -3,31 +3,26 @@
     ActionMenu,
     MenuItem,
     Icon,
-    Input,
     Heading,
     Body,
     Modal,
   } from "@budibase/bbui"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import CreateRestoreModal from "./CreateRestoreModal.svelte"
-  import { createEventDispatcher, onMount } from "svelte"
+  import { createEventDispatcher } from "svelte"
 
   export let row
 
   let deleteDialog
   let restoreDialog
-  let updateDialog
-  let name
   let restoreBackupModal
 
   const dispatch = createEventDispatcher()
 
-  const onClickRestore = name => {
+  const onClickRestore = () => {
     dispatch("buttonclick", {
       type: "backupRestore",
-      name,
       backupId: row._id,
-      restoreBackupName: name,
     })
   }
 
@@ -38,21 +33,9 @@
     })
   }
 
-  const onClickUpdate = () => {
-    dispatch("buttonclick", {
-      type: "backupUpdate",
-      backupId: row._id,
-      name,
-    })
-  }
-
   async function downloadExport() {
     window.open(`/api/apps/${row.appId}/backups/${row._id}/file`, "_blank")
   }
-
-  onMount(() => {
-    name = row.name
-  })
 </script>
 
 <div class="cell">
@@ -66,12 +49,11 @@
       <MenuItem on:click={deleteDialog.show} icon="Delete">Delete</MenuItem>
       <MenuItem on:click={downloadExport} icon="Download">Download</MenuItem>
     {/if}
-    <MenuItem on:click={updateDialog.show} icon="Edit">Rename</MenuItem>
   </ActionMenu>
 </div>
 
 <Modal bind:this={restoreBackupModal}>
-  <CreateRestoreModal confirm={name => onClickRestore(name)} />
+  <CreateRestoreModal confirm={onClickRestore} />
 </Modal>
 
 <ConfirmDialog
@@ -80,9 +62,7 @@
   onOk={onClickDelete}
   title="Confirm Deletion"
 >
-  Are you sure you wish to delete the backup
+  Are you sure you wish to delete this backup? This action cannot be undone.
-  <i>{row.name}?</i>
-  This action cannot be undone.
 </ConfirmDialog>
 
 <ConfirmDialog
@@ -92,21 +72,10 @@
   title="Confirm restore"
   warning={false}
 >
-  <Heading size="S">{row.name || "Backup"}</Heading>
+  <Heading size="S">Backup</Heading>
   <Body size="S">{new Date(row.timestamp).toLocaleString()}</Body>
 </ConfirmDialog>
 
-<ConfirmDialog
-  bind:this={updateDialog}
-  disabled={!name}
-  okText="Confirm"
-  onOk={onClickUpdate}
-  title="Update Backup"
-  warning={false}
->
-  <Input onlabel="Backup name" bind:value={name} />
-</ConfirmDialog>
-
 <style>
   .cell {
     display: flex;

packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/CreateBackupModal.svelte

@@ -1,22 +0,0 @@
-<script>
-  import { ModalContent, Input } from "@budibase/bbui"
-  import { auth } from "stores/portal"
-
-  export let createManualBackup
-
-  let templateName = $auth.user.firstName
-    ? `${$auth.user.firstName}'s Backup`
-    : "New Backup"
-  let name = templateName
-</script>
-
-<ModalContent
-  onConfirm={() => createManualBackup(name)}
-  title="Create new backup"
-  diabled={!name}
-  confirmText="Create"
-  ><Input label="Backup name" bind:value={name} /></ModalContent
->
-
-<style>
-</style>

packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/NameRenderer.svelte

@@ -1,8 +0,0 @@
-<script>
-  import { truncate } from "lodash"
-
-  export let value
-  $: truncatedValue = truncate(value, { length: 12 })
-</script>
-
-{truncatedValue}

packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/TimeAgoRenderer.svelte

@@ -0,0 +1,10 @@
+<script>
+  import dayjs from "dayjs"
+  import relativeTime from "dayjs/plugin/relativeTime"
+
+  dayjs.extend(relativeTime)
+
+  export let value
+</script>
+
+<span title={value}>{dayjs(value).fromNow()}</span>

packages/builder/src/pages/builder/portal/overview/[appId]/backups/_components/UserRenderer.svelte

@@ -1,17 +1,14 @@
 <script>
+  import getUserInitials from "helpers/userInitials.js"
+  import { Avatar } from "@budibase/bbui"
+
   export let value
 
-  let firstName = value?.firstName
+  $: initials = getUserInitials(value)
-  let lastName = value?.lastName || ""
-
-  $: username =
-    firstName && lastName ? `${firstName} ${lastName}` : value?.email
 </script>
 
-<div class="cell">
+<div title={value.email} class="cell">
-  {#if value != null}
+  <Avatar size="M" {initials} />
-    <div>{username}</div>
-  {/if}
 </div>
 
 <style>

packages/builder/src/pages/builder/portal/overview/[appId]/backups/index.svelte

@@ -4,7 +4,6 @@
     DatePicker,
     Divider,
     Layout,
-    Modal,
     notifications,
     Pagination,
     Select,
@@ -16,25 +15,22 @@
   } from "@budibase/bbui"
   import { backups, licensing, auth, admin, overview } from "stores/portal"
   import { createPaginationStore } from "helpers/pagination"
-  import DateRenderer from "components/common/renderers/DateTimeRenderer.svelte"
+  import TimeAgoRenderer from "./_components/TimeAgoRenderer.svelte"
   import AppSizeRenderer from "./_components/AppSizeRenderer.svelte"
-  import CreateBackupModal from "./_components/CreateBackupModal.svelte"
   import ActionsRenderer from "./_components/ActionsRenderer.svelte"
   import UserRenderer from "./_components/UserRenderer.svelte"
   import StatusRenderer from "./_components/StatusRenderer.svelte"
   import TypeRenderer from "./_components/TypeRenderer.svelte"
-  import NameRenderer from "./_components/NameRenderer.svelte"
   import BackupsDefault from "assets/backups-default.png"
   import { BackupTrigger, BackupType } from "constants/backend/backups"
   import { onMount } from "svelte"
 
+  let loading = true
   let backupData = null
-  let modal
   let pageInfo = createPaginationStore()
   let filterOpt = null
   let startDate = null
   let endDate = null
-  let loaded = false
   let filters = [
     {
       label: "Manual backup",
@@ -44,10 +40,6 @@
       label: "Published backup",
       value: { type: BackupType.BACKUP, trigger: BackupTrigger.PUBLISH },
     },
-    {
-      label: "Scheduled backup",
-      value: { type: BackupType.BACKUP, trigger: BackupTrigger.SCHEDULED },
-    },
     {
       label: "Pre-restore backup",
       value: { type: BackupType.BACKUP, trigger: BackupTrigger.RESTORING },
@@ -71,10 +63,6 @@
       displayName: "Date",
       width: "auto",
     },
-    name: {
-      displayName: "Name",
-      width: "auto",
-    },
     appSize: {
       displayName: "App size",
       width: "auto",
@@ -96,11 +84,10 @@
   const customRenderers = [
     { column: "appSize", component: AppSizeRenderer },
     { column: "actions", component: ActionsRenderer },
-    { column: "createdAt", component: DateRenderer },
+    { column: "createdAt", component: TimeAgoRenderer },
     { column: "createdBy", component: UserRenderer },
     { column: "status", component: StatusRenderer },
     { column: "type", component: TypeRenderer },
-    { column: "name", component: NameRenderer },
   ]
 
   function flattenBackups(backups) {
@@ -126,11 +113,11 @@
     backupData = flattenBackups(response.data)
   }
 
-  async function createManualBackup(name) {
+  async function createManualBackup() {
     try {
+      loading = true
       let response = await backups.createManualBackup({
         appId: app.instance._id,
-        name,
       })
       await fetchBackups(filterOpt, page)
       notifications.success(response.message)
@@ -139,6 +126,20 @@
     }
   }
 
+  const poll = backupData => {
+    if (backupData === null) {
+      return
+    }
+
+    if (backupData.some(datum => datum.status === "started")) {
+      setTimeout(() => fetchBackups(filterOpt, page), 2000)
+    } else {
+      loading = false
+    }
+  }
+
+  $: poll(backupData)
+
   async function handleButtonClick({ detail }) {
     if (detail.type === "backupDelete") {
       await backups.deleteBackup({
@@ -165,7 +166,7 @@
 
   onMount(async () => {
     await fetchBackups(filterOpt, page, startDate, endDate)
-    loaded = true
+    loading = false
   })
 </script>
 
@@ -206,7 +207,7 @@
         View plans
       </Button>
     </div>
-  {:else if !backupData?.length && loaded && !filterOpt && !startDate}
+  {:else if !backupData?.length && !loading && !filterOpt && !startDate}
     <div class="center">
       <Layout noPadding gap="S" justifyItems="center">
         <img height="130px" src={BackupsDefault} alt="BackupsDefault" />
@@ -215,11 +216,13 @@
           <Body>You can manually back up your app any time</Body>
         </Layout>
         <div>
-          <Button on:click={modal.show} cta>Create backup</Button>
+          <Button cta disabled={loading} on:click={createManualBackup}>
+            Create backup
+          </Button>
         </div>
       </Layout>
     </div>
-  {:else if loaded}
+  {:else}
     <Layout noPadding gap="M" alignContent="start">
       <div class="controls">
         <div class="search">
@@ -245,7 +248,9 @@
           />
         </div>
         <div>
-          <Button cta on:click={modal.show}>Create new backup</Button>
+          <Button cta disabled={loading} on:click={createManualBackup}
+            >Create new backup</Button
+          >
         </div>
       </div>
       <div class="table">
@@ -275,10 +280,6 @@
   {/if}
 </Layout>
 
-<Modal bind:this={modal}>
-  <CreateBackupModal {createManualBackup} />
-</Modal>
-
 <style>
   .title {
     display: flex;