Merge remote-tracking branch 'origin/develop' into test/fix_foreigndata_pg_test

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.17-alpha.5",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.17-alpha.5",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -18,13 +18,13 @@
     "build:pro": "../../scripts/pro/build.sh",
     "postbuild": "yarn run build:pro",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
-    "test": "jest --coverage --maxWorkers=2",
+    "test": "jest --coverage",
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
     "@budibase/nano": "10.1.1",
     "@budibase/pouchdb-replication-stream": "1.2.10",
-    "@budibase/types": "2.3.17-alpha.4",
+    "@budibase/types": "2.3.17-alpha.5",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",
@@ -62,7 +62,7 @@
     "@trendyol/jest-testcontainers": "^2.1.1",
     "@types/chance": "1.1.3",
     "@types/ioredis": "4.28.0",
-    "@types/jest": "27.5.1",
+    "@types/jest": "28.1.1",
     "@types/koa": "2.13.4",
     "@types/koa-pino-logger": "3.0.0",
     "@types/lodash": "4.14.180",

packages/backend-core/src/accounts/accounts.ts

@@ -1,13 +1,24 @@
 import API from "./api"
 import env from "../environment"
 import { Header } from "../constants"
-import { CloudAccount } from "@budibase/types"
+import { CloudAccount, HealthStatusResponse } from "@budibase/types"
 
 const api = new API(env.ACCOUNT_PORTAL_URL)
 
+/**
+ * This client is intended to be used in a cloud hosted deploy only.
+ * Rather than relying on each consumer to perform the necessary environmental checks
+ * we use the following check to exit early with a undefined response which should be
+ * handled by the caller.
+ */
+const EXIT_EARLY = env.SELF_HOSTED || env.DISABLE_ACCOUNT_PORTAL
+
 export const getAccount = async (
   email: string
 ): Promise<CloudAccount | undefined> => {
+  if (EXIT_EARLY) {
+    return
+  }
   const payload = {
     email,
   }
@@ -29,6 +40,9 @@ export const getAccount = async (
 export const getAccountByTenantId = async (
   tenantId: string
 ): Promise<CloudAccount | undefined> => {
+  if (EXIT_EARLY) {
+    return
+  }
   const payload = {
     tenantId,
   }
@@ -47,7 +61,12 @@ export const getAccountByTenantId = async (
   return json[0]
 }
 
-export const getStatus = async () => {
+export const getStatus = async (): Promise<
+  HealthStatusResponse | undefined
+> => {
+  if (EXIT_EARLY) {
+    return
+  }
   const response = await api.get(`/api/status`, {
     headers: {
       [Header.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,

packages/backend-core/src/accounts/index.ts

@@ -0,0 +1 @@
+export * from "./accounts"

packages/backend-core/src/auth/auth.ts

@@ -1,10 +1,11 @@
 const _passport = require("koa-passport")
 const LocalStrategy = require("passport-local").Strategy
 const JwtStrategy = require("passport-jwt").Strategy
-import { getGlobalDB } from "../tenancy"
+import { getGlobalDB } from "../context"
 const refresh = require("passport-oauth2-refresh")
-import { Config } from "../constants"
+import { Config, Cookie } from "../constants"
 import { getScopedConfig } from "../db"
+import { getSessionsForUser, invalidateSessions } from "../security/sessions"
 import {
   jwt as jwtPassport,
   local,
@@ -15,8 +16,11 @@ import {
   google,
 } from "../middleware"
 import { invalidateUser } from "../cache/user"
-import { User } from "@budibase/types"
+import { PlatformLogoutOpts, User } from "@budibase/types"
 import { logAlert } from "../logging"
+import * as events from "../events"
+import * as userCache from "../cache/user"
+import { clearCookie, getCookie } from "../utils"
 export {
   auditLog,
   authError,
@@ -29,6 +33,7 @@ export {
   google,
   oidc,
 } from "../middleware"
+import { ssoSaveUserNoOp } from "../middleware/passport/sso/sso"
 export const buildAuthMiddleware = authenticated
 export const buildTenancyMiddleware = tenancy
 export const buildCsrfMiddleware = csrf
@@ -71,7 +76,7 @@ async function refreshOIDCAccessToken(
     if (!enrichedConfig) {
       throw new Error("OIDC Config contents invalid")
     }
-    strategy = await oidc.strategyFactory(enrichedConfig)
+    strategy = await oidc.strategyFactory(enrichedConfig, ssoSaveUserNoOp)
   } catch (err) {
     console.error(err)
     throw new Error("Could not refresh OAuth Token")
@@ -103,7 +108,11 @@ async function refreshGoogleAccessToken(
 
   let strategy
   try {
-    strategy = await google.strategyFactory(config, callbackUrl)
+    strategy = await google.strategyFactory(
+      config,
+      callbackUrl,
+      ssoSaveUserNoOp
+    )
   } catch (err: any) {
     console.error(err)
     throw new Error(
@@ -161,6 +170,8 @@ export async function refreshOAuthToken(
   return refreshResponse
 }
 
+// TODO: Refactor to use user save function instead to prevent the need for
+// manually saving and invalidating on callback
 export async function updateUserOAuth(userId: string, oAuthConfig: any) {
   const details = {
     accessToken: oAuthConfig.accessToken,
@@ -188,3 +199,32 @@ export async function updateUserOAuth(userId: string, oAuthConfig: any) {
     console.error("Could not update OAuth details for current user", e)
   }
 }
+
+/**
+ * Logs a user out from budibase. Re-used across account portal and builder.
+ */
+export async function platformLogout(opts: PlatformLogoutOpts) {
+  const ctx = opts.ctx
+  const userId = opts.userId
+  const keepActiveSession = opts.keepActiveSession
+
+  if (!ctx) throw new Error("Koa context must be supplied to logout.")
+
+  const currentSession = getCookie(ctx, Cookie.Auth)
+  let sessions = await getSessionsForUser(userId)
+
+  if (keepActiveSession) {
+    sessions = sessions.filter(
+      session => session.sessionId !== currentSession.sessionId
+    )
+  } else {
+    // clear cookies
+    clearCookie(ctx, Cookie.Auth)
+    clearCookie(ctx, Cookie.CurrentApp)
+  }
+
+  const sessionIds = sessions.map(({ sessionId }) => sessionId)
+  await invalidateSessions(userId, { sessionIds, reason: "logout" })
+  await events.auth.logout()
+  await userCache.invalidateUser(userId)
+}

packages/backend-core/src/auth/tests/auth.spec.ts

@@ -0,0 +1,13 @@
+import { structures, testEnv } from "../../../tests"
+import * as auth from "../auth"
+import * as events from "../../events"
+
+describe("platformLogout", () => {
+  it("should call platform logout", async () => {
+    await testEnv.withTenant(async () => {
+      const ctx = structures.koa.newContext()
+      await auth.platformLogout({ ctx, userId: "test" })
+      expect(events.auth.logout).toBeCalledTimes(1)
+    })
+  })
+})

packages/backend-core/src/cache/tests/writethrough.spec.js

@@ -1,61 +0,0 @@
-require("../../../tests")
-const { Writethrough } = require("../writethrough")
-const { getDB } = require("../../db")
-const tk = require("timekeeper")
-const { structures } = require("../../../tests")
-
-const START_DATE = Date.now()
-tk.freeze(START_DATE)
-
-
-const DELAY = 5000
-
-const db = getDB(structures.db.id())
-const db2 = getDB(structures.db.id())
-const writethrough = new Writethrough(db, DELAY), writethrough2 = new Writethrough(db2, DELAY)
-
-describe("writethrough", () => {
-  describe("put", () => {
-    let first
-    it("should be able to store, will go to DB", async () => {
-      const response = await writethrough.put({ _id: "test", value: 1 })
-      const output = await db.get(response.id)
-      first = output
-      expect(output.value).toBe(1)
-    })
-
-    it("second put shouldn't update DB", async () => {
-      const response = await writethrough.put({ ...first, value: 2 })
-      const output = await db.get(response.id)
-      expect(first._rev).toBe(output._rev)
-      expect(output.value).toBe(1)
-    })
-
-    it("should put it again after delay period", async () => {
-      tk.freeze(START_DATE + DELAY + 1)
-      const response = await writethrough.put({ ...first, value: 3 })
-      const output = await db.get(response.id)
-      expect(response.rev).not.toBe(first._rev)
-      expect(output.value).toBe(3)
-    })
-  })
-
-  describe("get", () => {
-    it("should be able to retrieve", async () => {
-      const response = await writethrough.get("test")
-      expect(response.value).toBe(3)
-    })
-  })
-
-  describe("same doc, different databases (tenancy)", () => {
-    it("should be able to two different databases", async () => {
-      const resp1 = await writethrough.put({ _id: "db1", value: "first" })
-      const resp2 = await writethrough2.put({ _id: "db1", value: "second" })
-      expect(resp1.rev).toBeDefined()
-      expect(resp2.rev).toBeDefined()
-      expect((await db.get("db1")).value).toBe("first")
-      expect((await db2.get("db1")).value).toBe("second")
-    })
-  })
-})
-

packages/backend-core/src/cache/tests/writethrough.spec.ts

@@ -0,0 +1,73 @@
+import { structures, DBTestConfiguration } from "../../../tests"
+import { Writethrough } from "../writethrough"
+import { getDB } from "../../db"
+import tk from "timekeeper"
+
+const START_DATE = Date.now()
+tk.freeze(START_DATE)
+
+const DELAY = 5000
+
+describe("writethrough", () => {
+  const config = new DBTestConfiguration()
+
+  const db = getDB(structures.db.id())
+  const db2 = getDB(structures.db.id())
+
+  const writethrough = new Writethrough(db, DELAY)
+  const writethrough2 = new Writethrough(db2, DELAY)
+
+  describe("put", () => {
+    let first: any
+
+    it("should be able to store, will go to DB", async () => {
+      await config.doInTenant(async () => {
+        const response = await writethrough.put({ _id: "test", value: 1 })
+        const output = await db.get(response.id)
+        first = output
+        expect(output.value).toBe(1)
+      })
+    })
+
+    it("second put shouldn't update DB", async () => {
+      await config.doInTenant(async () => {
+        const response = await writethrough.put({ ...first, value: 2 })
+        const output = await db.get(response.id)
+        expect(first._rev).toBe(output._rev)
+        expect(output.value).toBe(1)
+      })
+    })
+
+    it("should put it again after delay period", async () => {
+      await config.doInTenant(async () => {
+        tk.freeze(START_DATE + DELAY + 1)
+        const response = await writethrough.put({ ...first, value: 3 })
+        const output = await db.get(response.id)
+        expect(response.rev).not.toBe(first._rev)
+        expect(output.value).toBe(3)
+      })
+    })
+  })
+
+  describe("get", () => {
+    it("should be able to retrieve", async () => {
+      await config.doInTenant(async () => {
+        const response = await writethrough.get("test")
+        expect(response.value).toBe(3)
+      })
+    })
+  })
+
+  describe("same doc, different databases (tenancy)", () => {
+    it("should be able to two different databases", async () => {
+      await config.doInTenant(async () => {
+        const resp1 = await writethrough.put({ _id: "db1", value: "first" })
+        const resp2 = await writethrough2.put({ _id: "db1", value: "second" })
+        expect(resp1.rev).toBeDefined()
+        expect(resp2.rev).toBeDefined()
+        expect((await db.get("db1")).value).toBe("first")
+        expect((await db2.get("db1")).value).toBe("second")
+      })
+    })
+  })
+})

packages/backend-core/src/cache/user.ts

@@ -1,8 +1,9 @@
 import * as redis from "../redis/init"
-import { getTenantId, lookupTenantId, doWithGlobalDB } from "../tenancy"
+import * as tenancy from "../tenancy"
+import * as context from "../context"
+import * as platform from "../platform"
 import env from "../environment"
-import * as accounts from "../cloud/accounts"
+import * as accounts from "../accounts"
-import { Database } from "@budibase/types"
 
 const EXPIRY_SECONDS = 3600
 
@@ -10,7 +11,8 @@ const EXPIRY_SECONDS = 3600
  * The default populate user function
  */
 async function populateFromDB(userId: string, tenantId: string) {
-  const user = await doWithGlobalDB(tenantId, (db: Database) => db.get(userId))
+  const db = tenancy.getTenantDB(tenantId)
+  const user = await db.get(userId)
   user.budibaseAccess = true
   if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     const account = await accounts.getAccount(user.email)
@@ -42,9 +44,9 @@ export async function getUser(
   }
   if (!tenantId) {
     try {
-      tenantId = getTenantId()
+      tenantId = context.getTenantId()
     } catch (err) {
-      tenantId = await lookupTenantId(userId)
+      tenantId = await platform.users.lookupTenantId(userId)
     }
   }
   const client = await redis.getUserClient()

packages/backend-core/src/context/deprovision.ts

@@ -1,108 +0,0 @@
-import {
-  getGlobalUserParams,
-  getAllApps,
-  doWithDB,
-  StaticDatabases,
-} from "../db"
-import { doWithGlobalDB } from "../tenancy"
-import { App, Tenants, User, Database } from "@budibase/types"
-
-const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
-const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
-
-async function removeTenantFromInfoDB(tenantId: string) {
-  try {
-    await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => {
-      const tenants = (await infoDb.get(TENANT_DOC)) as Tenants
-      tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)
-
-      await infoDb.put(tenants)
-    })
-  } catch (err) {
-    console.error(`Error removing tenant ${tenantId} from info db`, err)
-    throw err
-  }
-}
-
-export async function removeUserFromInfoDB(dbUser: User) {
-  await doWithDB(PLATFORM_INFO_DB, async (infoDb: Database) => {
-    const keys = [dbUser._id!, dbUser.email]
-    const userDocs = await infoDb.allDocs({
-      keys,
-      include_docs: true,
-    })
-    const toDelete = userDocs.rows.map((row: any) => {
-      return {
-        ...row.doc,
-        _deleted: true,
-      }
-    })
-    await infoDb.bulkDocs(toDelete)
-  })
-}
-
-async function removeUsersFromInfoDB(tenantId: string) {
-  return doWithGlobalDB(tenantId, async (db: any) => {
-    try {
-      const allUsers = await db.allDocs(
-        getGlobalUserParams(null, {
-          include_docs: true,
-        })
-      )
-      await doWithDB(PLATFORM_INFO_DB, async (infoDb: any) => {
-        const allEmails = allUsers.rows.map((row: any) => row.doc.email)
-        // get the id docs
-        let keys = allUsers.rows.map((row: any) => row.id)
-        // and the email docs
-        keys = keys.concat(allEmails)
-        // retrieve the docs and delete them
-        const userDocs = await infoDb.allDocs({
-          keys,
-          include_docs: true,
-        })
-        const toDelete = userDocs.rows.map((row: any) => {
-          return {
-            ...row.doc,
-            _deleted: true,
-          }
-        })
-        await infoDb.bulkDocs(toDelete)
-      })
-    } catch (err) {
-      console.error(`Error removing tenant ${tenantId} users from info db`, err)
-      throw err
-    }
-  })
-}
-
-async function removeGlobalDB(tenantId: string) {
-  return doWithGlobalDB(tenantId, async (db: Database) => {
-    try {
-      await db.destroy()
-    } catch (err) {
-      console.error(`Error removing tenant ${tenantId} users from info db`, err)
-      throw err
-    }
-  })
-}
-
-async function removeTenantApps(tenantId: string) {
-  try {
-    const apps = (await getAllApps({ all: true })) as App[]
-    const destroyPromises = apps.map(app =>
-      doWithDB(app.appId, (db: Database) => db.destroy())
-    )
-    await Promise.allSettled(destroyPromises)
-  } catch (err) {
-    console.error(`Error removing tenant ${tenantId} apps`, err)
-    throw err
-  }
-}
-
-// can't live in tenancy package due to circular dependency on db/utils
-export async function deleteTenant(tenantId: string) {
-  await removeTenantFromInfoDB(tenantId)
-  await removeUsersFromInfoDB(tenantId)
-  await removeGlobalDB(tenantId)
-  await removeTenantApps(tenantId)
-}

packages/backend-core/src/context/tests/index.spec.ts

@@ -1,11 +1,14 @@
-require("../../../tests")
+import { testEnv } from "../../../tests"
 const context = require("../")
 const { DEFAULT_TENANT_ID } = require("../../constants")
-import env from "../../environment"
 
 describe("context", () => {
   describe("doInTenant", () => {
     describe("single-tenancy", () => {
+      beforeAll(() => {
+        testEnv.singleTenant()
+      })
+
       it("defaults to the default tenant", () => {
         const tenantId = context.getTenantId()
         expect(tenantId).toBe(DEFAULT_TENANT_ID)
@@ -20,8 +23,8 @@ describe("context", () => {
     })
 
     describe("multi-tenancy", () => {
-      beforeEach(() => {
+      beforeAll(() => {
-        env._set("MULTI_TENANCY", 1)
+        testEnv.multiTenant()
       })
 
       it("fails when no tenant id is set", () => {

packages/backend-core/src/db/db.ts

@@ -1,7 +1,6 @@
 import env from "../environment"
-import { directCouchQuery, getPouchDB } from "./couch"
+import { directCouchQuery, DatabaseImpl } from "./couch"
 import { CouchFindOptions, Database } from "@budibase/types"
-import { DatabaseImpl } from "../db"
 
 const dbList = new Set()
 

packages/backend-core/src/db/tests/utils.seq.spec.ts

@@ -1,190 +0,0 @@
-require("../../../tests")
-const {
-  getDevelopmentAppID,
-  getProdAppID,
-  isDevAppID,
-  isProdAppID,
-} = require("../conversions")
-const { generateAppID, getPlatformUrl, getScopedConfig } = require("../utils")
-const tenancy = require("../../tenancy")
-const { Config, DEFAULT_TENANT_ID } = require("../../constants")
-import { generator } from "../../../tests"
-import env from "../../environment"
-
-describe("utils", () => {
-  describe("app ID manipulation", () => {
-    function getID() {
-      const appId = generateAppID()
-      const split = appId.split("_")
-      const uuid = split[split.length - 1]
-      const devAppId = `app_dev_${uuid}`
-      return { appId, devAppId, split, uuid }
-    }
-
-    it("should be able to generate a new app ID", () => {
-      expect(generateAppID().startsWith("app_")).toEqual(true)
-    })
-
-    it("should be able to convert a production app ID to development", () => {
-      const { appId, uuid } = getID()
-      expect(getDevelopmentAppID(appId)).toEqual(`app_dev_${uuid}`)
-    })
-
-    it("should be able to convert a development app ID to development", () => {
-      const { devAppId, uuid } = getID()
-      expect(getDevelopmentAppID(devAppId)).toEqual(`app_dev_${uuid}`)
-    })
-
-    it("should be able to convert a development ID to a production", () => {
-      const { devAppId, uuid } = getID()
-      expect(getProdAppID(devAppId)).toEqual(`app_${uuid}`)
-    })
-
-    it("should be able to convert a production ID to production", () => {
-      const { appId, uuid } = getID()
-      expect(getProdAppID(appId)).toEqual(`app_${uuid}`)
-    })
-
-    it("should be able to confirm dev app ID is development", () => {
-      const { devAppId } = getID()
-      expect(isDevAppID(devAppId)).toEqual(true)
-    })
-
-    it("should be able to confirm prod app ID is not development", () => {
-      const { appId } = getID()
-      expect(isDevAppID(appId)).toEqual(false)
-    })
-
-    it("should be able to confirm prod app ID is prod", () => {
-      const { appId } = getID()
-      expect(isProdAppID(appId)).toEqual(true)
-    })
-
-    it("should be able to confirm dev app ID is not prod", () => {
-      const { devAppId } = getID()
-      expect(isProdAppID(devAppId)).toEqual(false)
-    })
-  })
-})
-
-const DEFAULT_URL = "http://localhost:10000"
-const ENV_URL = "http://env.com"
-
-const setDbPlatformUrl = async (dbUrl: string) => {
-  const db = tenancy.getGlobalDB()
-  await db.put({
-    _id: "config_settings",
-    type: Config.SETTINGS,
-    config: {
-      platformUrl: dbUrl,
-    },
-  })
-}
-
-const clearSettingsConfig = async () => {
-  await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-    const db = tenancy.getGlobalDB()
-    try {
-      const config = await db.get("config_settings")
-      await db.remove("config_settings", config._rev)
-    } catch (e: any) {
-      if (e.status !== 404) {
-        throw e
-      }
-    }
-  })
-}
-
-describe("getPlatformUrl", () => {
-  describe("self host", () => {
-    beforeEach(async () => {
-      env._set("SELF_HOST", 1)
-      await clearSettingsConfig()
-    })
-
-    it("gets the default url", async () => {
-      await tenancy.doInTenant(null, async () => {
-        const url = await getPlatformUrl()
-        expect(url).toBe(DEFAULT_URL)
-      })
-    })
-
-    it("gets the platform url from the environment", async () => {
-      await tenancy.doInTenant(null, async () => {
-        env._set("PLATFORM_URL", ENV_URL)
-        const url = await getPlatformUrl()
-        expect(url).toBe(ENV_URL)
-      })
-    })
-
-    it("gets the platform url from the database", async () => {
-      await tenancy.doInTenant(null, async () => {
-        const dbUrl = generator.url()
-        await setDbPlatformUrl(dbUrl)
-        const url = await getPlatformUrl()
-        expect(url).toBe(dbUrl)
-      })
-    })
-  })
-
-  describe("cloud", () => {
-    const TENANT_AWARE_URL = "http://default.env.com"
-
-    beforeEach(async () => {
-      env._set("SELF_HOSTED", 0)
-      env._set("MULTI_TENANCY", 1)
-      env._set("PLATFORM_URL", ENV_URL)
-      await clearSettingsConfig()
-    })
-
-    it("gets the platform url from the environment without tenancy", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        const url = await getPlatformUrl({ tenantAware: false })
-        expect(url).toBe(ENV_URL)
-      })
-    })
-
-    it("gets the platform url from the environment with tenancy", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        const url = await getPlatformUrl()
-        expect(url).toBe(TENANT_AWARE_URL)
-      })
-    })
-
-    it("never gets the platform url from the database", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        await setDbPlatformUrl(generator.url())
-        const url = await getPlatformUrl()
-        expect(url).toBe(TENANT_AWARE_URL)
-      })
-    })
-  })
-})
-
-describe("getScopedConfig", () => {
-  describe("settings config", () => {
-    beforeEach(async () => {
-      env._set("SELF_HOSTED", 1)
-      env._set("PLATFORM_URL", "")
-      await clearSettingsConfig()
-    })
-
-    it("returns the platform url with an existing config", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        const dbUrl = generator.url()
-        await setDbPlatformUrl(dbUrl)
-        const db = tenancy.getGlobalDB()
-        const config = await getScopedConfig(db, { type: Config.SETTINGS })
-        expect(config.platformUrl).toBe(dbUrl)
-      })
-    })
-
-    it("returns the platform url without an existing config", async () => {
-      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
-        const db = tenancy.getGlobalDB()
-        const config = await getScopedConfig(db, { type: Config.SETTINGS })
-        expect(config.platformUrl).toBe(DEFAULT_URL)
-      })
-    })
-  })
-})

packages/backend-core/src/db/tests/utils.spec.ts

@@ -0,0 +1,192 @@
+import { generator, DBTestConfiguration, testEnv } from "../../../tests"
+import {
+  getDevelopmentAppID,
+  getProdAppID,
+  isDevAppID,
+  isProdAppID,
+} from "../conversions"
+import { generateAppID, getPlatformUrl, getScopedConfig } from "../utils"
+import * as context from "../../context"
+import { Config } from "../../constants"
+import env from "../../environment"
+
+describe("utils", () => {
+  const config = new DBTestConfiguration()
+
+  describe("app ID manipulation", () => {
+    function getID() {
+      const appId = generateAppID()
+      const split = appId.split("_")
+      const uuid = split[split.length - 1]
+      const devAppId = `app_dev_${uuid}`
+      return { appId, devAppId, split, uuid }
+    }
+
+    it("should be able to generate a new app ID", () => {
+      expect(generateAppID().startsWith("app_")).toEqual(true)
+    })
+
+    it("should be able to convert a production app ID to development", () => {
+      const { appId, uuid } = getID()
+      expect(getDevelopmentAppID(appId)).toEqual(`app_dev_${uuid}`)
+    })
+
+    it("should be able to convert a development app ID to development", () => {
+      const { devAppId, uuid } = getID()
+      expect(getDevelopmentAppID(devAppId)).toEqual(`app_dev_${uuid}`)
+    })
+
+    it("should be able to convert a development ID to a production", () => {
+      const { devAppId, uuid } = getID()
+      expect(getProdAppID(devAppId)).toEqual(`app_${uuid}`)
+    })
+
+    it("should be able to convert a production ID to production", () => {
+      const { appId, uuid } = getID()
+      expect(getProdAppID(appId)).toEqual(`app_${uuid}`)
+    })
+
+    it("should be able to confirm dev app ID is development", () => {
+      const { devAppId } = getID()
+      expect(isDevAppID(devAppId)).toEqual(true)
+    })
+
+    it("should be able to confirm prod app ID is not development", () => {
+      const { appId } = getID()
+      expect(isDevAppID(appId)).toEqual(false)
+    })
+
+    it("should be able to confirm prod app ID is prod", () => {
+      const { appId } = getID()
+      expect(isProdAppID(appId)).toEqual(true)
+    })
+
+    it("should be able to confirm dev app ID is not prod", () => {
+      const { devAppId } = getID()
+      expect(isProdAppID(devAppId)).toEqual(false)
+    })
+  })
+
+  const DEFAULT_URL = "http://localhost:10000"
+  const ENV_URL = "http://env.com"
+
+  const setDbPlatformUrl = async (dbUrl: string) => {
+    const db = context.getGlobalDB()
+    await db.put({
+      _id: "config_settings",
+      type: Config.SETTINGS,
+      config: {
+        platformUrl: dbUrl,
+      },
+    })
+  }
+
+  const clearSettingsConfig = async () => {
+    await config.doInTenant(async () => {
+      const db = context.getGlobalDB()
+      try {
+        const config = await db.get("config_settings")
+        await db.remove("config_settings", config._rev)
+      } catch (e: any) {
+        if (e.status !== 404) {
+          throw e
+        }
+      }
+    })
+  }
+
+  describe("getPlatformUrl", () => {
+    describe("self host", () => {
+      beforeEach(async () => {
+        testEnv.selfHosted()
+        await clearSettingsConfig()
+      })
+
+      it("gets the default url", async () => {
+        await config.doInTenant(async () => {
+          const url = await getPlatformUrl()
+          expect(url).toBe(DEFAULT_URL)
+        })
+      })
+
+      it("gets the platform url from the environment", async () => {
+        await config.doInTenant(async () => {
+          env._set("PLATFORM_URL", ENV_URL)
+          const url = await getPlatformUrl()
+          expect(url).toBe(ENV_URL)
+        })
+      })
+
+      it("gets the platform url from the database", async () => {
+        await config.doInTenant(async () => {
+          const dbUrl = generator.url()
+          await setDbPlatformUrl(dbUrl)
+          const url = await getPlatformUrl()
+          expect(url).toBe(dbUrl)
+        })
+      })
+    })
+
+    describe("cloud", () => {
+      const TENANT_AWARE_URL = `http://${config.tenantId}.env.com`
+
+      beforeEach(async () => {
+        testEnv.cloudHosted()
+        testEnv.multiTenant()
+
+        env._set("PLATFORM_URL", ENV_URL)
+        await clearSettingsConfig()
+      })
+
+      it("gets the platform url from the environment without tenancy", async () => {
+        await config.doInTenant(async () => {
+          const url = await getPlatformUrl({ tenantAware: false })
+          expect(url).toBe(ENV_URL)
+        })
+      })
+
+      it("gets the platform url from the environment with tenancy", async () => {
+        await config.doInTenant(async () => {
+          const url = await getPlatformUrl()
+          expect(url).toBe(TENANT_AWARE_URL)
+        })
+      })
+
+      it("never gets the platform url from the database", async () => {
+        await config.doInTenant(async () => {
+          await setDbPlatformUrl(generator.url())
+          const url = await getPlatformUrl()
+          expect(url).toBe(TENANT_AWARE_URL)
+        })
+      })
+    })
+  })
+
+  describe("getScopedConfig", () => {
+    describe("settings config", () => {
+      beforeEach(async () => {
+        env._set("SELF_HOSTED", 1)
+        env._set("PLATFORM_URL", "")
+        await clearSettingsConfig()
+      })
+
+      it("returns the platform url with an existing config", async () => {
+        await config.doInTenant(async () => {
+          const dbUrl = generator.url()
+          await setDbPlatformUrl(dbUrl)
+          const db = context.getGlobalDB()
+          const config = await getScopedConfig(db, { type: Config.SETTINGS })
+          expect(config.platformUrl).toBe(dbUrl)
+        })
+      })
+
+      it("returns the platform url without an existing config", async () => {
+        await config.doInTenant(async () => {
+          const db = context.getGlobalDB()
+          const config = await getScopedConfig(db, { type: Config.SETTINGS })
+          expect(config.platformUrl).toBe(DEFAULT_URL)
+        })
+      })
+    })
+  })
+})

packages/backend-core/src/db/views.ts

@@ -1,13 +1,14 @@
 import {
-  DocumentType,
-  ViewName,
   DeprecatedViews,
+  DocumentType,
   SEPARATOR,
   StaticDatabases,
+  ViewName,
 } from "../constants"
 import { getGlobalDB } from "../context"
 import { doWithDB } from "./"
 import { Database, DatabaseQueryOpts } from "@budibase/types"
+import env from "../environment"
 
 const DESIGN_DB = "_design/database"
 
@@ -69,17 +70,6 @@ export const createNewUserEmailView = async () => {
   await createView(db, viewJs, ViewName.USER_BY_EMAIL)
 }
 
-export const createAccountEmailView = async () => {
-  const viewJs = `function(doc) {
-    if (doc._id.startsWith("${DocumentType.ACCOUNT_METADATA}${SEPARATOR}")) {
-      emit(doc.email.toLowerCase(), doc._id)
-    }
-  }`
-  await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
-    await createView(db, viewJs, ViewName.ACCOUNT_BY_EMAIL)
-  })
-}
-
 export const createUserAppView = async () => {
   const db = getGlobalDB()
   const viewJs = `function(doc) {
@@ -113,17 +103,6 @@ export const createUserBuildersView = async () => {
   await createView(db, viewJs, ViewName.USER_BY_BUILDERS)
 }
 
-export const createPlatformUserView = async () => {
-  const viewJs = `function(doc) {
-    if (doc.tenantId) {
-      emit(doc._id.toLowerCase(), doc._id)
-    }
-  }`
-  await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
-    await createView(db, viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
-  })
-}
-
 export interface QueryViewOptions {
   arrayResponse?: boolean
 }
@@ -162,13 +141,48 @@ export const queryView = async <T>(
   }
 }
 
+// PLATFORM
+
+async function createPlatformView(viewJs: string, viewName: ViewName) {
+  try {
+    await doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: Database) => {
+      await createView(db, viewJs, viewName)
+    })
+  } catch (e: any) {
+    if (e.status === 409 && env.isTest()) {
+      // multiple tests can try to initialise platforms views
+      // at once - safe to exit on conflict
+      return
+    }
+    throw e
+  }
+}
+
+export const createPlatformAccountEmailView = async () => {
+  const viewJs = `function(doc) {
+    if (doc._id.startsWith("${DocumentType.ACCOUNT_METADATA}${SEPARATOR}")) {
+      emit(doc.email.toLowerCase(), doc._id)
+    }
+  }`
+  await createPlatformView(viewJs, ViewName.ACCOUNT_BY_EMAIL)
+}
+
+export const createPlatformUserView = async () => {
+  const viewJs = `function(doc) {
+    if (doc.tenantId) {
+      emit(doc._id.toLowerCase(), doc._id)
+    }
+  }`
+  await createPlatformView(viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
+}
+
 export const queryPlatformView = async <T>(
   viewName: ViewName,
   params: DatabaseQueryOpts,
   opts?: QueryViewOptions
 ): Promise<T[] | T | undefined> => {
   const CreateFuncByName: any = {
-    [ViewName.ACCOUNT_BY_EMAIL]: createAccountEmailView,
+    [ViewName.ACCOUNT_BY_EMAIL]: createPlatformAccountEmailView,
     [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView,
   }
 

packages/backend-core/src/environment.ts

@@ -44,8 +44,9 @@ const environment = {
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
   GOOGLE_CLIENT_SECRET: process.env.GOOGLE_CLIENT_SECRET,
   SALT_ROUNDS: process.env.SALT_ROUNDS,
-  REDIS_URL: process.env.REDIS_URL,
+  REDIS_URL: process.env.REDIS_URL || "localhost:6379",
-  REDIS_PASSWORD: process.env.REDIS_PASSWORD,
+  REDIS_PASSWORD: process.env.REDIS_PASSWORD || "budibase",
+  MOCK_REDIS: process.env.MOCK_REDIS,
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   AWS_REGION: process.env.AWS_REGION,
@@ -82,6 +83,7 @@ const environment = {
   SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD,
   DEPLOYMENT_ENVIRONMENT:
     process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose",
+  ENABLE_4XX_HTTP_LOGGING: process.env.ENABLE_4XX_HTTP_LOGGING || true,
   _set(key: any, value: any) {
     process.env[key] = value
     // @ts-ignore

packages/backend-core/src/events/analytics.ts

@@ -1,5 +1,5 @@
 import env from "../environment"
-import * as tenancy from "../tenancy"
+import * as context from "../context"
 import * as dbUtils from "../db/utils"
 import { Config } from "../constants"
 import { withCache, TTL, CacheKey } from "../cache"
@@ -42,7 +42,7 @@ export const enabled = async () => {
 }
 
 const getSettingsDoc = async () => {
-  const db = tenancy.getGlobalDB()
+  const db = context.getGlobalDB()
   let settings
   try {
     settings = await db.get(dbUtils.generateConfigID({ type: Config.SETTINGS }))

packages/backend-core/src/events/identification.ts

@@ -16,6 +16,7 @@ import {
   InstallationGroup,
   UserContext,
   Group,
+  isSSOUser,
 } from "@budibase/types"
 import { processors } from "./processors"
 import * as dbUtils from "../db/utils"
@@ -166,7 +167,10 @@ const identifyUser = async (
   const type = IdentityType.USER
   let builder = user.builder?.global || false
   let admin = user.admin?.global || false
-  let providerType = user.providerType
+  let providerType
+  if (isSSOUser(user)) {
+    providerType = user.providerType
+  }
   const accountHolder = account?.budibaseUserId === user._id || false
   const verified =
     account && account?.budibaseUserId === user._id ? account.verified : false

packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts

@@ -1,4 +1,4 @@
-import "../../../../../tests"
+import { testEnv } from "../../../../../tests"
 import PosthogProcessor from "../PosthogProcessor"
 import { Event, IdentityType, Hosting } from "@budibase/types"
 const tk = require("timekeeper")
@@ -16,6 +16,10 @@ const newIdentity = () => {
 }
 
 describe("PosthogProcessor", () => {
+  beforeAll(() => {
+    testEnv.singleTenant()
+  })
+
   beforeEach(async () => {
     jest.clearAllMocks()
     await cache.bustCache(

packages/backend-core/src/featureFlags/index.ts

@@ -1,5 +1,5 @@
 import env from "../environment"
-import * as tenancy from "../tenancy"
+import * as context from "../context"
 
 /**
  * Read the TENANT_FEATURE_FLAGS env var and return an array of features flags for each tenant.
@@ -28,7 +28,7 @@ export function buildFeatureFlags() {
 }
 
 export function isEnabled(featureFlag: string) {
-  const tenantId = tenancy.getTenantId()
+  const tenantId = context.getTenantId()
   const flags = getTenantFeatureFlags(tenantId)
   return flags.includes(featureFlag)
 }

packages/backend-core/src/index.ts

@@ -3,12 +3,11 @@ export * as migrations from "./migrations"
 export * as users from "./users"
 export * as roles from "./security/roles"
 export * as permissions from "./security/permissions"
-export * as accounts from "./cloud/accounts"
+export * as accounts from "./accounts"
 export * as installation from "./installation"
-export * as tenancy from "./tenancy"
 export * as featureFlags from "./featureFlags"
 export * as sessions from "./security/sessions"
-export * as deprovisioning from "./context/deprovision"
+export * as platform from "./platform"
 export * as auth from "./auth"
 export * as constants from "./constants"
 export * as logging from "./logging"
@@ -21,20 +20,27 @@ export * as context from "./context"
 export * as cache from "./cache"
 export * as objectStore from "./objectStore"
 export * as redis from "./redis"
+export * as locks from "./redis/redlock"
 export * as utils from "./utils"
 export * as errors from "./errors"
 export { default as env } from "./environment"
 
+// Add context to tenancy for backwards compatibility
+// only do this for external usages to prevent internal
+// circular dependencies
+import * as context from "./context"
+import * as _tenancy from "./tenancy"
+export const tenancy = {
+  ..._tenancy,
+  ...context,
+}
+
 // expose error classes directly
 export * from "./errors"
 
 // expose constants directly
 export * from "./constants"
 
-// expose inner locks from redis directly
-import * as redis from "./redis"
-export const locks = redis.redlock
-
 // expose package init function
 import * as db from "./db"
 export const init = (opts: any = {}) => {

packages/backend-core/src/middleware/authenticated.ts

@@ -4,7 +4,7 @@ import { getUser } from "../cache/user"
 import { getSession, updateSessionTTL } from "../security/sessions"
 import { buildMatcherRegex, matches } from "./matchers"
 import { SEPARATOR, queryGlobalView, ViewName } from "../db"
-import { getGlobalDB, doInTenant } from "../tenancy"
+import { getGlobalDB, doInTenant } from "../context"
 import { decrypt } from "../security/encryption"
 import * as identity from "../context/identity"
 import env from "../environment"

packages/backend-core/src/middleware/errorHandling.ts

@@ -0,0 +1,28 @@
+import { APIError } from "@budibase/types"
+import * as errors from "../errors"
+import env from "../environment"
+
+export async function errorHandling(ctx: any, next: any) {
+  try {
+    await next()
+  } catch (err: any) {
+    const status = err.status || err.statusCode || 500
+    ctx.status = status
+
+    if (status > 499 || env.ENABLE_4XX_HTTP_LOGGING) {
+      ctx.log.error(err)
+    }
+
+    const error = errors.getPublicError(err)
+    const body: APIError = {
+      message: err.message,
+      status: status,
+      validationErrors: err.validation,
+      error,
+    }
+
+    ctx.body = body
+  }
+}
+
+export default errorHandling

packages/backend-core/src/middleware/index.ts

@@ -1,7 +1,7 @@
 export * as jwt from "./passport/jwt"
 export * as local from "./passport/local"
-export * as google from "./passport/google"
+export * as google from "./passport/sso/google"
-export * as oidc from "./passport/oidc"
+export * as oidc from "./passport/sso/oidc"
 import * as datasourceGoogle from "./passport/datasource/google"
 export const datasource = {
   google: datasourceGoogle,
@@ -16,4 +16,5 @@ export { default as adminOnly } from "./adminOnly"
 export { default as builderOrAdmin } from "./builderOrAdmin"
 export { default as builderOnly } from "./builderOnly"
 export { default as logging } from "./logging"
+export { default as errorHandling } from "./errorHandling"
 export * as joiValidator from "./joi-validator"

packages/backend-core/src/middleware/passport/datasource/google.ts

@@ -1,10 +1,11 @@
-import * as google from "../google"
+import * as google from "../sso/google"
 import { Cookie, Config } from "../../../constants"
 import { clearCookie, getCookie } from "../../../utils"
 import { getScopedConfig, getPlatformUrl, doWithDB } from "../../../db"
 import environment from "../../../environment"
-import { getGlobalDB } from "../../../tenancy"
+import { getGlobalDB } from "../../../context"
 import { BBContext, Database, SSOProfile } from "@budibase/types"
+import { ssoSaveUserNoOp } from "../sso/sso"
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
 
 type Passport = {
@@ -36,7 +37,11 @@ export async function preAuth(
   const platformUrl = await getPlatformUrl({ tenantAware: false })
 
   let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback`
-  const strategy = await google.strategyFactory(googleConfig, callbackUrl)
+  const strategy = await google.strategyFactory(
+    googleConfig,
+    callbackUrl,
+    ssoSaveUserNoOp
+  )
 
   if (!ctx.query.appId || !ctx.query.datasourceId) {
     ctx.throw(400, "appId and datasourceId query params not present.")

packages/backend-core/src/middleware/passport/local.ts

@@ -1,15 +1,10 @@
 import { UserStatus } from "../../constants"
-import { compare, newid } from "../../utils"
+import { compare } from "../../utils"
-import env from "../../environment"
 import * as users from "../../users"
 import { authError } from "./utils"
-import { createASession } from "../../security/sessions"
-import { getTenantId } from "../../tenancy"
 import { BBContext } from "@budibase/types"
-const jwt = require("jsonwebtoken")
 
 const INVALID_ERR = "Invalid credentials"
-const SSO_NO_PASSWORD = "SSO user does not have a password set"
 const EXPIRED = "This account has expired. Please reset your password"
 
 export const options = {
@@ -35,50 +30,25 @@ export async function authenticate(
 
   const dbUser = await users.getGlobalUserByEmail(email)
   if (dbUser == null) {
-    return authError(done, `User not found: [${email}]`)
+    console.info(`user=${email} could not be found`)
-  }
-
-  // check that the user is currently inactive, if this is the case throw invalid
-  if (dbUser.status === UserStatus.INACTIVE) {
     return authError(done, INVALID_ERR)
   }
 
-  // check that the user has a stored password before proceeding
+  if (dbUser.status === UserStatus.INACTIVE) {
-  if (!dbUser.password) {
+    console.info(`user=${email} is inactive`, dbUser)
-    if (
+    return authError(done, INVALID_ERR)
-      (dbUser.account && dbUser.account.authType === "sso") || // root account sso
+  }
-      dbUser.thirdPartyProfile // internal sso
-    ) {
-      return authError(done, SSO_NO_PASSWORD)
-    }
 
-    console.error("Non SSO usser has no password set", dbUser)
+  if (!dbUser.password) {
+    console.info(`user=${email} has no password set`, dbUser)
     return authError(done, EXPIRED)
   }
 
-  // authenticate
+  if (!(await compare(password, dbUser.password))) {
-  if (await compare(password, dbUser.password)) {
-    const sessionId = newid()
-    const tenantId = getTenantId()
-
-    await createASession(dbUser._id!, { sessionId, tenantId })
-
-    const token = jwt.sign(
-      {
-        userId: dbUser._id,
-        sessionId,
-        tenantId,
-      },
-      env.JWT_SECRET
-    )
-    // Remove users password in payload
-    delete dbUser.password
-
-    return done(null, {
-      ...dbUser,
-      token,
-    })
-  } else {
     return authError(done, INVALID_ERR)
   }
+
+  // intentionally remove the users password in payload
+  delete dbUser.password
+  return done(null, dbUser)
 }

packages/backend-core/src/middleware/passport/sso/google.ts

@@ -1,18 +1,26 @@
-import { ssoCallbackUrl } from "./utils"
+import { ssoCallbackUrl } from "../utils"
-import { authenticateThirdParty, SaveUserFunction } from "./third-party-common"
+import * as sso from "./sso"
-import { ConfigType, GoogleConfig, Database, SSOProfile } from "@budibase/types"
+import {
+  ConfigType,
+  GoogleConfig,
+  Database,
+  SSOProfile,
+  SSOAuthDetails,
+  SSOProviderType,
+  SaveSSOUserFunction,
+} from "@budibase/types"
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
 
-export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
+export function buildVerifyFn(saveUserFn: SaveSSOUserFunction) {
   return (
     accessToken: string,
     refreshToken: string,
     profile: SSOProfile,
     done: Function
   ) => {
-    const thirdPartyUser = {
+    const details: SSOAuthDetails = {
-      provider: profile.provider, // should always be 'google'
+      provider: "google",
-      providerType: "google",
+      providerType: SSOProviderType.GOOGLE,
       userId: profile.id,
       profile: profile,
       email: profile._json.email,
@@ -22,8 +30,8 @@ export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
       },
     }
 
-    return authenticateThirdParty(
+    return sso.authenticate(
-      thirdPartyUser,
+      details,
       true, // require local accounts to exist
       done,
       saveUserFn
@@ -39,7 +47,7 @@ export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
 export async function strategyFactory(
   config: GoogleConfig["config"],
   callbackUrl: string,
-  saveUserFn?: SaveUserFunction
+  saveUserFn: SaveSSOUserFunction
 ) {
   try {
     const { clientID, clientSecret } = config

packages/backend-core/src/middleware/passport/sso/oidc.ts

@@ -1,22 +1,20 @@
 import fetch from "node-fetch"
-import { authenticateThirdParty, SaveUserFunction } from "./third-party-common"
+import * as sso from "./sso"
-import { ssoCallbackUrl } from "./utils"
+import { ssoCallbackUrl } from "../utils"
 import {
   ConfigType,
-  OIDCInnerCfg,
+  OIDCInnerConfig,
   Database,
   SSOProfile,
-  ThirdPartyUser,
+  OIDCStrategyConfiguration,
-  OIDCConfiguration,
+  SSOAuthDetails,
+  SSOProviderType,
+  JwtClaims,
+  SaveSSOUserFunction,
 } from "@budibase/types"
 const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy
 
-type JwtClaims = {
+export function buildVerifyFn(saveUserFn: SaveSSOUserFunction) {
-  preferred_username: string
-  email: string
-}
-
-export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
   /**
    * @param {*} issuer The identity provider base URL
    * @param {*} sub The user ID
@@ -39,10 +37,10 @@ export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
     params: any,
     done: Function
   ) => {
-    const thirdPartyUser: ThirdPartyUser = {
+    const details: SSOAuthDetails = {
       // store the issuer info to enable sync in future
       provider: issuer,
-      providerType: "oidc",
+      providerType: SSOProviderType.OIDC,
       userId: profile.id,
       profile: profile,
       email: getEmail(profile, jwtClaims),
@@ -52,8 +50,8 @@ export function buildVerifyFn(saveUserFn?: SaveUserFunction) {
       },
     }
 
-    return authenticateThirdParty(
+    return sso.authenticate(
-      thirdPartyUser,
+      details,
       false, // don't require local accounts to exist
       done,
       saveUserFn
@@ -104,8 +102,8 @@ function validEmail(value: string) {
  * @returns Dynamically configured Passport OIDC Strategy
  */
 export async function strategyFactory(
-  config: OIDCConfiguration,
+  config: OIDCStrategyConfiguration,
-  saveUserFn?: SaveUserFunction
+  saveUserFn: SaveSSOUserFunction
 ) {
   try {
     const verify = buildVerifyFn(saveUserFn)
@@ -119,14 +117,14 @@ export async function strategyFactory(
 }
 
 export async function fetchStrategyConfig(
-  enrichedConfig: OIDCInnerCfg,
+  oidcConfig: OIDCInnerConfig,
   callbackUrl?: string
-): Promise<OIDCConfiguration> {
+): Promise<OIDCStrategyConfiguration> {
   try {
-    const { clientID, clientSecret, configUrl } = enrichedConfig
+    const { clientID, clientSecret, configUrl } = oidcConfig
 
     if (!clientID || !clientSecret || !callbackUrl || !configUrl) {
-      //check for remote config and all required elements
+      // check for remote config and all required elements
       throw new Error(
         "Configuration invalid. Must contain clientID, clientSecret, callbackUrl and configUrl"
       )

packages/backend-core/src/middleware/passport/sso/sso.ts

@@ -0,0 +1,165 @@
+import { generateGlobalUserID } from "../../../db"
+import { authError } from "../utils"
+import * as users from "../../../users"
+import * as context from "../../../context"
+import fetch from "node-fetch"
+import {
+  SaveSSOUserFunction,
+  SaveUserOpts,
+  SSOAuthDetails,
+  SSOUser,
+  User,
+} from "@budibase/types"
+
+// no-op function for user save
+// - this allows datasource auth and access token refresh to work correctly
+// - prefer no-op over an optional argument to ensure function is provided to login flows
+export const ssoSaveUserNoOp: SaveSSOUserFunction = (
+  user: SSOUser,
+  opts: SaveUserOpts
+) => Promise.resolve(user)
+
+/**
+ * Common authentication logic for third parties. e.g. OAuth, OIDC.
+ */
+export async function authenticate(
+  details: SSOAuthDetails,
+  requireLocalAccount: boolean = true,
+  done: any,
+  saveUserFn: SaveSSOUserFunction
+) {
+  if (!saveUserFn) {
+    throw new Error("Save user function must be provided")
+  }
+  if (!details.userId) {
+    return authError(done, "sso user id required")
+  }
+  if (!details.email) {
+    return authError(done, "sso user email required")
+  }
+
+  // use the third party id
+  const userId = generateGlobalUserID(details.userId)
+
+  let dbUser: User | undefined
+
+  // try to load by id
+  try {
+    dbUser = await users.getById(userId)
+  } catch (err: any) {
+    // abort when not 404 error
+    if (!err.status || err.status !== 404) {
+      return authError(
+        done,
+        "Unexpected error when retrieving existing user",
+        err
+      )
+    }
+  }
+
+  // fallback to loading by email
+  if (!dbUser) {
+    dbUser = await users.getGlobalUserByEmail(details.email)
+  }
+
+  // exit early if there is still no user and auto creation is disabled
+  if (!dbUser && requireLocalAccount) {
+    return authError(
+      done,
+      "Email does not yet exist. You must set up your local budibase account first."
+    )
+  }
+
+  // first time creation
+  if (!dbUser) {
+    // setup a blank user using the third party id
+    dbUser = {
+      _id: userId,
+      email: details.email,
+      roles: {},
+      tenantId: context.getTenantId(),
+    }
+  }
+
+  let ssoUser = await syncUser(dbUser, details)
+  // never prompt for password reset
+  ssoUser.forceResetPassword = false
+
+  try {
+    // don't try to re-save any existing password
+    delete ssoUser.password
+    // create or sync the user
+    ssoUser = (await saveUserFn(ssoUser, {
+      hashPassword: false,
+      requirePassword: false,
+    })) as SSOUser
+  } catch (err: any) {
+    return authError(done, "Error saving user", err)
+  }
+
+  return done(null, ssoUser)
+}
+
+async function getProfilePictureUrl(user: User, details: SSOAuthDetails) {
+  const pictureUrl = details.profile?._json.picture
+  if (pictureUrl) {
+    const response = await fetch(pictureUrl)
+    if (response.status === 200) {
+      const type = response.headers.get("content-type") as string
+      if (type.startsWith("image/")) {
+        return pictureUrl
+      }
+    }
+  }
+}
+
+/**
+ * @returns a user that has been sync'd with third party information
+ */
+async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
+  let firstName
+  let lastName
+  let pictureUrl
+  let oauth2
+  let thirdPartyProfile
+
+  if (details.profile) {
+    const profile = details.profile
+
+    if (profile.name) {
+      const name = profile.name
+      // first name
+      if (name.givenName) {
+        firstName = name.givenName
+      }
+      // last name
+      if (name.familyName) {
+        lastName = name.familyName
+      }
+    }
+
+    pictureUrl = await getProfilePictureUrl(user, details)
+
+    thirdPartyProfile = {
+      ...profile._json,
+    }
+  }
+
+  // oauth tokens for future use
+  if (details.oauth2) {
+    oauth2 = {
+      ...details.oauth2,
+    }
+  }
+
+  return {
+    ...user,
+    provider: details.provider,
+    providerType: details.providerType,
+    firstName,
+    lastName,
+    thirdPartyProfile,
+    pictureUrl,
+    oauth2,
+  }
+}

packages/backend-core/src/middleware/passport/sso/tests/google.spec.ts

@@ -0,0 +1,67 @@
+import { generator, structures } from "../../../../../tests"
+import { SSOProviderType } from "@budibase/types"
+
+jest.mock("passport-google-oauth")
+const mockStrategy = require("passport-google-oauth").OAuth2Strategy
+
+jest.mock("../sso")
+import * as _sso from "../sso"
+const sso = jest.mocked(_sso)
+
+const mockSaveUserFn = jest.fn()
+const mockDone = jest.fn()
+
+import * as google from "../google"
+
+describe("google", () => {
+  describe("strategyFactory", () => {
+    const googleConfig = structures.sso.googleConfig()
+    const callbackUrl = generator.url()
+
+    it("should create successfully create a google strategy", async () => {
+      await google.strategyFactory(googleConfig, callbackUrl)
+
+      const expectedOptions = {
+        clientID: googleConfig.clientID,
+        clientSecret: googleConfig.clientSecret,
+        callbackURL: callbackUrl,
+      }
+
+      expect(mockStrategy).toHaveBeenCalledWith(
+        expectedOptions,
+        expect.anything()
+      )
+    })
+  })
+
+  describe("authenticate", () => {
+    const details = structures.sso.authDetails()
+    details.provider = "google"
+    details.providerType = SSOProviderType.GOOGLE
+
+    const profile = details.profile!
+    profile.provider = "google"
+
+    beforeEach(() => {
+      jest.clearAllMocks()
+    })
+
+    it("delegates authentication to third party common", async () => {
+      const authenticate = await google.buildVerifyFn(mockSaveUserFn)
+
+      await authenticate(
+        details.oauth2.accessToken,
+        details.oauth2.refreshToken!,
+        profile,
+        mockDone
+      )
+
+      expect(sso.authenticate).toHaveBeenCalledWith(
+        details,
+        true,
+        mockDone,
+        mockSaveUserFn
+      )
+    })
+  })
+})

packages/backend-core/src/middleware/passport/sso/tests/oidc.spec.ts

@@ -0,0 +1,152 @@
+import { generator, mocks, structures } from "../../../../../tests"
+import {
+  JwtClaims,
+  OIDCInnerConfig,
+  SSOAuthDetails,
+  SSOProviderType,
+} from "@budibase/types"
+import * as _sso from "../sso"
+import * as oidc from "../oidc"
+
+jest.mock("@techpass/passport-openidconnect")
+const mockStrategy = require("@techpass/passport-openidconnect").Strategy
+
+jest.mock("../sso")
+const sso = jest.mocked(_sso)
+
+const mockSaveUser = jest.fn()
+const mockDone = jest.fn()
+
+describe("oidc", () => {
+  const callbackUrl = generator.url()
+  const oidcConfig: OIDCInnerConfig = structures.sso.oidcConfig()
+  const wellKnownConfig = structures.sso.oidcWellKnownConfig()
+
+  function mockRetrieveWellKnownConfig() {
+    // mock the request to retrieve the oidc configuration
+    mocks.fetch.mockReturnValue({
+      ok: true,
+      json: () => wellKnownConfig,
+    })
+  }
+
+  beforeEach(() => {
+    mockRetrieveWellKnownConfig()
+  })
+
+  describe("strategyFactory", () => {
+    it("should create successfully create an oidc strategy", async () => {
+      const strategyConfiguration = await oidc.fetchStrategyConfig(
+        oidcConfig,
+        callbackUrl
+      )
+      await oidc.strategyFactory(strategyConfiguration, mockSaveUser)
+
+      expect(mocks.fetch).toHaveBeenCalledWith(oidcConfig.configUrl)
+
+      const expectedOptions = {
+        issuer: wellKnownConfig.issuer,
+        authorizationURL: wellKnownConfig.authorization_endpoint,
+        tokenURL: wellKnownConfig.token_endpoint,
+        userInfoURL: wellKnownConfig.userinfo_endpoint,
+        clientID: oidcConfig.clientID,
+        clientSecret: oidcConfig.clientSecret,
+        callbackURL: callbackUrl,
+      }
+      expect(mockStrategy).toHaveBeenCalledWith(
+        expectedOptions,
+        expect.anything()
+      )
+    })
+  })
+
+  describe("authenticate", () => {
+    const details: SSOAuthDetails = structures.sso.authDetails()
+    details.providerType = SSOProviderType.OIDC
+    const profile = details.profile!
+    const issuer = profile.provider
+
+    const sub = generator.string()
+    const idToken = generator.string()
+    const params = {}
+
+    let authenticateFn: any
+    let jwtClaims: JwtClaims
+
+    beforeEach(async () => {
+      jest.clearAllMocks()
+      authenticateFn = await oidc.buildVerifyFn(mockSaveUser)
+    })
+
+    async function authenticate() {
+      await authenticateFn(
+        issuer,
+        sub,
+        profile,
+        jwtClaims,
+        details.oauth2.accessToken,
+        details.oauth2.refreshToken,
+        idToken,
+        params,
+        mockDone
+      )
+    }
+
+    it("passes auth details to sso module", async () => {
+      await authenticate()
+
+      expect(sso.authenticate).toHaveBeenCalledWith(
+        details,
+        false,
+        mockDone,
+        mockSaveUser
+      )
+    })
+
+    it("uses JWT email to get email", async () => {
+      delete profile._json.email
+
+      jwtClaims = {
+        email: details.email,
+      }
+
+      await authenticate()
+
+      expect(sso.authenticate).toHaveBeenCalledWith(
+        details,
+        false,
+        mockDone,
+        mockSaveUser
+      )
+    })
+
+    it("uses JWT username to get email", async () => {
+      delete profile._json.email
+
+      jwtClaims = {
+        email: details.email,
+      }
+
+      await authenticate()
+
+      expect(sso.authenticate).toHaveBeenCalledWith(
+        details,
+        false,
+        mockDone,
+        mockSaveUser
+      )
+    })
+
+    it("uses JWT invalid username to get email", async () => {
+      delete profile._json.email
+
+      jwtClaims = {
+        preferred_username: "invalidUsername",
+      }
+
+      await expect(authenticate()).rejects.toThrow(
+        "Could not determine user email from profile"
+      )
+    })
+  })
+})

packages/backend-core/src/middleware/passport/sso/tests/sso.spec.ts

@@ -0,0 +1,196 @@
+import { structures, testEnv, mocks } from "../../../../../tests"
+import { SSOAuthDetails, User } from "@budibase/types"
+
+import { HTTPError } from "../../../../errors"
+import * as sso from "../sso"
+import * as context from "../../../../context"
+
+const mockDone = jest.fn()
+const mockSaveUser = jest.fn()
+
+jest.mock("../../../../users")
+import * as _users from "../../../../users"
+const users = jest.mocked(_users)
+
+const getErrorMessage = () => {
+  return mockDone.mock.calls[0][2].message
+}
+
+describe("sso", () => {
+  describe("authenticate", () => {
+    beforeEach(() => {
+      jest.clearAllMocks()
+      testEnv.singleTenant()
+    })
+
+    describe("validation", () => {
+      const testValidation = async (
+        details: SSOAuthDetails,
+        message: string
+      ) => {
+        await sso.authenticate(details, false, mockDone, mockSaveUser)
+
+        expect(mockDone.mock.calls.length).toBe(1)
+        expect(getErrorMessage()).toContain(message)
+      }
+
+      it("user id fails", async () => {
+        const details = structures.sso.authDetails()
+        details.userId = undefined!
+
+        await testValidation(details, "sso user id required")
+      })
+
+      it("email fails", async () => {
+        const details = structures.sso.authDetails()
+        details.email = undefined!
+
+        await testValidation(details, "sso user email required")
+      })
+    })
+
+    function mockGetProfilePicture() {
+      mocks.fetch.mockReturnValueOnce(
+        Promise.resolve({
+          status: 200,
+          headers: { get: () => "image/" },
+        })
+      )
+    }
+
+    describe("when the user doesn't exist", () => {
+      let user: User
+      let details: SSOAuthDetails
+
+      beforeEach(() => {
+        users.getById.mockImplementationOnce(() => {
+          throw new HTTPError("", 404)
+        })
+        mockGetProfilePicture()
+
+        user = structures.users.user()
+        delete user._rev
+        delete user._id
+
+        details = structures.sso.authDetails(user)
+        details.userId = structures.uuid()
+      })
+
+      describe("when a local account is required", () => {
+        it("returns an error message", async () => {
+          const details = structures.sso.authDetails()
+
+          await sso.authenticate(details, true, mockDone, mockSaveUser)
+
+          expect(mockDone.mock.calls.length).toBe(1)
+          expect(getErrorMessage()).toContain(
+            "Email does not yet exist. You must set up your local budibase account first."
+          )
+        })
+      })
+
+      describe("when a local account isn't required", () => {
+        it("creates and authenticates the user", async () => {
+          const ssoUser = structures.users.ssoUser({ user, details })
+          mockSaveUser.mockReturnValueOnce(ssoUser)
+
+          await sso.authenticate(details, false, mockDone, mockSaveUser)
+
+          // default roles for new user
+          ssoUser.roles = {}
+
+          // modified external id to match user format
+          ssoUser._id = "us_" + details.userId
+
+          // new sso user won't have a password
+          delete ssoUser.password
+
+          // new user isn't saved with rev
+          delete ssoUser._rev
+
+          // tenant id added
+          ssoUser.tenantId = context.getTenantId()
+
+          expect(mockSaveUser).toBeCalledWith(ssoUser, {
+            hashPassword: false,
+            requirePassword: false,
+          })
+          expect(mockDone).toBeCalledWith(null, ssoUser)
+        })
+      })
+    })
+
+    describe("when the user exists", () => {
+      let existingUser: User
+      let details: SSOAuthDetails
+
+      beforeEach(() => {
+        existingUser = structures.users.user()
+        existingUser._id = structures.uuid()
+        details = structures.sso.authDetails(existingUser)
+        mockGetProfilePicture()
+      })
+
+      describe("exists by email", () => {
+        beforeEach(() => {
+          users.getById.mockImplementationOnce(() => {
+            throw new HTTPError("", 404)
+          })
+          users.getGlobalUserByEmail.mockReturnValueOnce(
+            Promise.resolve(existingUser)
+          )
+        })
+
+        it("syncs and authenticates the user", async () => {
+          const ssoUser = structures.users.ssoUser({
+            user: existingUser,
+            details,
+          })
+          mockSaveUser.mockReturnValueOnce(ssoUser)
+
+          await sso.authenticate(details, true, mockDone, mockSaveUser)
+
+          // roles preserved
+          ssoUser.roles = existingUser.roles
+
+          // existing id preserved
+          ssoUser._id = existingUser._id
+
+          expect(mockSaveUser).toBeCalledWith(ssoUser, {
+            hashPassword: false,
+            requirePassword: false,
+          })
+          expect(mockDone).toBeCalledWith(null, ssoUser)
+        })
+      })
+
+      describe("exists by id", () => {
+        beforeEach(() => {
+          users.getById.mockReturnValueOnce(Promise.resolve(existingUser))
+        })
+
+        it("syncs and authenticates the user", async () => {
+          const ssoUser = structures.users.ssoUser({
+            user: existingUser,
+            details,
+          })
+          mockSaveUser.mockReturnValueOnce(ssoUser)
+
+          await sso.authenticate(details, true, mockDone, mockSaveUser)
+
+          // roles preserved
+          ssoUser.roles = existingUser.roles
+
+          // existing id preserved
+          ssoUser._id = existingUser._id
+
+          expect(mockSaveUser).toBeCalledWith(ssoUser, {
+            hashPassword: false,
+            requirePassword: false,
+          })
+          expect(mockDone).toBeCalledWith(null, ssoUser)
+        })
+      })
+    })
+  })
+})

packages/backend-core/src/middleware/passport/tests/google.spec.js

@@ -1,79 +0,0 @@
-// Mock data
-
-const { data } = require("./utilities/mock-data")
-
-const TENANT_ID = "default"
-
-const googleConfig = {
-  clientID: data.clientID,
-  clientSecret: data.clientSecret,
-}
-
-const profile = {
-  id: "mockId",
-  _json: {
-    email : data.email
-  },
-  provider: "google"
-}
-
-const user = data.buildThirdPartyUser("google", "google", profile)
-
-describe("google", () => {
-  describe("strategyFactory", () => {  
-    // mock passport strategy factory
-    jest.mock("passport-google-oauth")
-    const mockStrategy = require("passport-google-oauth").OAuth2Strategy
-  
-    it("should create successfully create a google strategy", async () => {
-      const google = require("../google")
-
-      const callbackUrl = `/api/global/auth/${TENANT_ID}/google/callback`
-      await google.strategyFactory(googleConfig, callbackUrl)
-  
-      const expectedOptions = {
-        clientID: googleConfig.clientID,
-        clientSecret: googleConfig.clientSecret,
-        callbackURL: callbackUrl,
-      }
-
-      expect(mockStrategy).toHaveBeenCalledWith(
-        expectedOptions,
-        expect.anything()
-      )
-    })
-  })
-  
-  describe("authenticate", () => {    
-    afterEach(() => {
-      jest.clearAllMocks();
-    });
-
-    // mock third party common authentication
-    jest.mock("../third-party-common")
-    const authenticateThirdParty = require("../third-party-common").authenticateThirdParty
-
-    // mock the passport callback
-    const mockDone = jest.fn()
-
-    it("delegates authentication to third party common", async () => {
-      const google = require("../google")
-      const mockSaveUserFn = jest.fn()
-      const authenticate = await google.buildVerifyFn(mockSaveUserFn)
-
-      await authenticate(
-        data.accessToken,
-        data.refreshToken,
-        profile,
-        mockDone
-      )
-
-      expect(authenticateThirdParty).toHaveBeenCalledWith(
-        user,
-        true, 
-        mockDone,
-        mockSaveUserFn)
-    })
-  })
-})
-

packages/backend-core/src/middleware/passport/tests/oidc.spec.js

@@ -1,144 +0,0 @@
-// Mock data
-const mockFetch = require("node-fetch")
-const { data } = require("./utilities/mock-data")
-const issuer = "mockIssuer"
-const sub = "mockSub"
-const profile = {
-  id: "mockId",
-  _json: {
-    email : data.email
-  }
-}
-let jwtClaims = {}  
-const idToken = "mockIdToken"
-const params = {}
-
-const callbackUrl = "http://somecallbackurl"
-
-// response from .well-known/openid-configuration
-const oidcConfigUrlResponse = {
-  issuer: issuer,
-  authorization_endpoint: "mockAuthorizationEndpoint",
-  token_endpoint: "mockTokenEndpoint",
-  userinfo_endpoint: "mockUserInfoEndpoint"
-}
-
-const oidcConfig = {
-  configUrl: "http://someconfigurl",
-  clientID: data.clientID,
-  clientSecret: data.clientSecret,
-}
-
-const user = data.buildThirdPartyUser(issuer, "oidc", profile)
-
-describe("oidc", () => {
-  describe("strategyFactory", () => {  
-    // mock passport strategy factory
-    jest.mock("@techpass/passport-openidconnect")
-    const mockStrategy = require("@techpass/passport-openidconnect").Strategy
-    
-    // mock the request to retrieve the oidc configuration
-    mockFetch.mockReturnValue({
-      ok: true,
-      json: () => oidcConfigUrlResponse
-    })
-  
-    it("should create successfully create an oidc strategy", async () => {
-      const oidc = require("../oidc")
-      const enrichedConfig = await oidc.fetchStrategyConfig(oidcConfig, callbackUrl)
-      await oidc.strategyFactory(enrichedConfig, callbackUrl)
-  
-      expect(mockFetch).toHaveBeenCalledWith(oidcConfig.configUrl)
-
-      const expectedOptions = {
-        issuer: oidcConfigUrlResponse.issuer,
-        authorizationURL: oidcConfigUrlResponse.authorization_endpoint,
-        tokenURL: oidcConfigUrlResponse.token_endpoint,
-        userInfoURL: oidcConfigUrlResponse.userinfo_endpoint,
-        clientID: oidcConfig.clientID,
-        clientSecret: oidcConfig.clientSecret,
-        callbackURL: callbackUrl,
-      }
-      expect(mockStrategy).toHaveBeenCalledWith(
-        expectedOptions,
-        expect.anything()
-      )
-    })
-  })
-  
-  describe("authenticate", () => {    
-    afterEach(() => {
-      jest.clearAllMocks()
-    });
-
-    // mock third party common authentication
-    jest.mock("../third-party-common")
-    const authenticateThirdParty = require("../third-party-common").authenticateThirdParty
-    
-    // mock the passport callback
-    const mockDone = jest.fn()
-    const mockSaveUserFn = jest.fn()
-
-    async function doAuthenticate() {
-      const oidc = require("../oidc")
-      const authenticate = await oidc.buildVerifyFn(mockSaveUserFn)
-
-      await authenticate(
-        issuer,
-        sub,
-        profile,
-        jwtClaims,
-        data.accessToken,
-        data.refreshToken,
-        idToken,
-        params,
-        mockDone
-      )
-    }
-
-    async function doTest() {
-      await doAuthenticate()
-
-      expect(authenticateThirdParty).toHaveBeenCalledWith(
-        user,
-        false, 
-        mockDone,
-        mockSaveUserFn,
-      )
-    }
-
-    it("delegates authentication to third party common", async () => {
-      await doTest()
-    })
-
-    it("uses JWT email to get email", async () => {
-      delete profile._json.email
-      jwtClaims = {
-        email : "mock@budibase.com"
-      }
-
-      await doTest()
-    })
-  
-    it("uses JWT username to get email", async () => {
-      delete profile._json.email
-      jwtClaims = {
-        preferred_username : "mock@budibase.com"
-      }
-
-      await doTest()
-    })
-
-    it("uses JWT invalid username to get email", async () => {
-      delete profile._json.email
-
-      jwtClaims = {
-        preferred_username : "invalidUsername"
-      }
-
-      await expect(doAuthenticate()).rejects.toThrow("Could not determine user email from profile");
-    })
-  
-  })
-})
-

packages/backend-core/src/middleware/passport/tests/third-party-common.seq.spec.js

@@ -1,178 +0,0 @@
-require("../../../../tests")
-const { authenticateThirdParty } = require("../third-party-common")
-const { data } = require("./utilities/mock-data")
-const { DEFAULT_TENANT_ID } = require("../../../constants")
-
-const { generateGlobalUserID } = require("../../../db/utils")
-const { newid } = require("../../../utils")
-const { doWithGlobalDB, doInTenant } = require("../../../tenancy")
-
-const done = jest.fn()
-
-const getErrorMessage = () => {
-  return done.mock.calls[0][2].message
-}
-
-const saveUser = async (user) => {
-  return doWithGlobalDB(DEFAULT_TENANT_ID, async db => {
-    return await db.put(user)
-  })
-}
-
-function authenticate(user, requireLocal, saveFn) {
-  return doInTenant(DEFAULT_TENANT_ID, () => {
-    return authenticateThirdParty(user, requireLocal, done, saveFn)
-  })
-}
-
-describe("third party common", () => {
-  describe("authenticateThirdParty", () => {
-    let thirdPartyUser
-
-    beforeEach(() => {
-      thirdPartyUser = data.buildThirdPartyUser()
-    })
-
-    afterEach(async () => {
-      return doWithGlobalDB(DEFAULT_TENANT_ID, async db => {
-        jest.clearAllMocks()
-        await db.destroy()
-      })
-    })
-
-    describe("validation", () => {
-      const testValidation = async (message) => {
-        await authenticate(thirdPartyUser, false, saveUser)
-        expect(done.mock.calls.length).toBe(1)
-        expect(getErrorMessage()).toContain(message)
-      }
-
-      it("provider fails", async () => {
-        delete thirdPartyUser.provider
-        await testValidation("third party user provider required")
-      })
-
-      it("user id fails", async () => {
-        delete thirdPartyUser.userId
-        await testValidation("third party user id required")
-      })
-
-      it("email fails", async () => {
-        delete thirdPartyUser.email
-        await testValidation("third party user email required")
-      })
-    })
-
-    const expectUserIsAuthenticated = () => {
-      const user = done.mock.calls[0][1]
-      expect(user).toBeDefined()
-      expect(user._id).toBeDefined()
-      expect(user._rev).toBeDefined()
-      expect(user.token).toBeDefined()
-      return user
-    }
-
-    const expectUserIsSynced = (user, thirdPartyUser) => {
-      expect(user.provider).toBe(thirdPartyUser.provider)
-      expect(user.firstName).toBe(thirdPartyUser.profile.name.givenName)
-      expect(user.lastName).toBe(thirdPartyUser.profile.name.familyName)
-      expect(user.thirdPartyProfile).toStrictEqual(thirdPartyUser.profile._json)
-      expect(user.oauth2).toStrictEqual(thirdPartyUser.oauth2)
-    }
-
-    describe("when the user doesn't exist", () => {
-      describe("when a local account is required", () => {
-        it("returns an error message", async () => {
-          await authenticate(thirdPartyUser, true, saveUser)
-          expect(done.mock.calls.length).toBe(1)
-          expect(getErrorMessage()).toContain("Email does not yet exist. You must set up your local budibase account first.")
-        })
-      })
-
-      describe("when a local account isn't required", () => {
-        it("creates and authenticates the user", async () => {
-          await authenticate(thirdPartyUser, false, saveUser)
-          const user = expectUserIsAuthenticated()
-          expectUserIsSynced(user, thirdPartyUser)
-          expect(user.roles).toStrictEqual({})
-        })
-      })
-    })
-
-    describe("when the user exists", () => {
-      let dbUser
-      let id
-      let email
-
-      const createUser = async () => {
-        return doWithGlobalDB(DEFAULT_TENANT_ID, async db => {
-          dbUser = {
-            _id: id,
-            email: email,
-          }
-          const response = await db.put(dbUser)
-          dbUser._rev = response.rev
-          return dbUser
-        })
-      }
-
-      const expectUserIsUpdated = (user) => {
-        // id is unchanged
-        expect(user._id).toBe(id)
-        // user is updated
-        expect(user._rev).not.toBe(dbUser._rev)
-      }
-
-      describe("exists by email", () => {
-        beforeEach(async () => {
-          id = generateGlobalUserID(newid()) // random id
-          email = thirdPartyUser.email //  matching email
-          await createUser()
-        })
-
-        it("syncs and authenticates the user", async () => {
-          await authenticate(thirdPartyUser, true, saveUser)
-
-          const user = expectUserIsAuthenticated()
-          expectUserIsSynced(user, thirdPartyUser)
-          expectUserIsUpdated(user)
-        })
-      })
-
-      describe("exists by email with different casing", () => {
-        beforeEach(async () => {
-          id = generateGlobalUserID(newid()) // random id
-          email = thirdPartyUser.email.toUpperCase() //  matching email except for casing
-          await createUser()
-        })
-
-        it("syncs and authenticates the user", async () => {
-          await authenticate(thirdPartyUser, true, saveUser)
-
-          const user = expectUserIsAuthenticated()
-          expectUserIsSynced(user, thirdPartyUser)
-          expectUserIsUpdated(user)
-          expect(user.email).toBe(thirdPartyUser.email.toUpperCase())
-        })
-      })
-
-
-      describe("exists by id", () => {
-        beforeEach(async () => {
-          id = generateGlobalUserID(thirdPartyUser.userId) // matching id
-          email = "test@test.com" // random email
-          await createUser()
-        })
-
-        it("syncs and authenticates the user", async () => {
-          await authenticate(thirdPartyUser, true, saveUser)
-
-          const user = expectUserIsAuthenticated()
-          expectUserIsSynced(user, thirdPartyUser)
-          expectUserIsUpdated(user)
-        })
-      })
-    })
-  })
-})
-

packages/backend-core/src/middleware/passport/tests/utilities/mock-data.js

@@ -1,54 +0,0 @@
-// Mock Data
-
-const mockClientID = "mockClientID"
-const mockClientSecret = "mockClientSecret"
-
-const mockEmail = "mock@budibase.com"
-const mockAccessToken = "mockAccessToken"
-const mockRefreshToken = "mockRefreshToken"
-
-const mockProvider = "mockProvider"
-const mockProviderType = "mockProviderType"
-
-const mockProfile = {
-  id: "mockId",
-  name: {
-    givenName: "mockGivenName",
-    familyName: "mockFamilyName",
-  },
-  _json: {
-    email: mockEmail,
-  },
-}
-
-const buildOauth2 = (
-  accessToken = mockAccessToken,
-  refreshToken = mockRefreshToken
-) => ({
-  accessToken: accessToken,
-  refreshToken: refreshToken,
-})
-
-const buildThirdPartyUser = (
-  provider = mockProvider,
-  providerType = mockProviderType,
-  profile = mockProfile,
-  email = mockEmail,
-  oauth2 = buildOauth2()
-) => ({
-  provider: provider,
-  providerType: providerType,
-  userId: profile.id,
-  profile: profile,
-  email: email,
-  oauth2: oauth2,
-})
-
-exports.data = {
-  clientID: mockClientID,
-  clientSecret: mockClientSecret,
-  email: mockEmail,
-  accessToken: mockAccessToken,
-  refreshToken: mockRefreshToken,
-  buildThirdPartyUser,
-}

packages/backend-core/src/middleware/passport/third-party-common.ts

@@ -1,177 +0,0 @@
-import env from "../../environment"
-import { generateGlobalUserID } from "../../db"
-import { authError } from "./utils"
-import { newid } from "../../utils"
-import { createASession } from "../../security/sessions"
-import * as users from "../../users"
-import { getGlobalDB, getTenantId } from "../../tenancy"
-import fetch from "node-fetch"
-import { ThirdPartyUser } from "@budibase/types"
-const jwt = require("jsonwebtoken")
-
-type SaveUserOpts = {
-  requirePassword?: boolean
-  hashPassword?: boolean
-  currentUserId?: string
-}
-
-export type SaveUserFunction = (
-  user: ThirdPartyUser,
-  opts: SaveUserOpts
-) => Promise<any>
-
-/**
- * Common authentication logic for third parties. e.g. OAuth, OIDC.
- */
-export async function authenticateThirdParty(
-  thirdPartyUser: ThirdPartyUser,
-  requireLocalAccount: boolean = true,
-  done: Function,
-  saveUserFn?: SaveUserFunction
-) {
-  if (!saveUserFn) {
-    throw new Error("Save user function must be provided")
-  }
-  if (!thirdPartyUser.provider) {
-    return authError(done, "third party user provider required")
-  }
-  if (!thirdPartyUser.userId) {
-    return authError(done, "third party user id required")
-  }
-  if (!thirdPartyUser.email) {
-    return authError(done, "third party user email required")
-  }
-
-  // use the third party id
-  const userId = generateGlobalUserID(thirdPartyUser.userId)
-  const db = getGlobalDB()
-
-  let dbUser
-
-  // try to load by id
-  try {
-    dbUser = await db.get(userId)
-  } catch (err: any) {
-    // abort when not 404 error
-    if (!err.status || err.status !== 404) {
-      return authError(
-        done,
-        "Unexpected error when retrieving existing user",
-        err
-      )
-    }
-  }
-
-  // fallback to loading by email
-  if (!dbUser) {
-    dbUser = await users.getGlobalUserByEmail(thirdPartyUser.email)
-  }
-
-  // exit early if there is still no user and auto creation is disabled
-  if (!dbUser && requireLocalAccount) {
-    return authError(
-      done,
-      "Email does not yet exist. You must set up your local budibase account first."
-    )
-  }
-
-  // first time creation
-  if (!dbUser) {
-    // setup a blank user using the third party id
-    dbUser = {
-      _id: userId,
-      email: thirdPartyUser.email,
-      roles: {},
-    }
-  }
-
-  dbUser = await syncUser(dbUser, thirdPartyUser)
-
-  // never prompt for password reset
-  dbUser.forceResetPassword = false
-
-  // create or sync the user
-  try {
-    await saveUserFn(dbUser, { hashPassword: false, requirePassword: false })
-  } catch (err: any) {
-    return authError(done, "Error saving user", err)
-  }
-
-  // now that we're sure user exists, load them from the db
-  dbUser = await db.get(dbUser._id)
-
-  // authenticate
-  const sessionId = newid()
-  const tenantId = getTenantId()
-  await createASession(dbUser._id, { sessionId, tenantId })
-
-  dbUser.token = jwt.sign(
-    {
-      userId: dbUser._id,
-      sessionId,
-    },
-    env.JWT_SECRET
-  )
-
-  return done(null, dbUser)
-}
-
-async function syncProfilePicture(
-  user: ThirdPartyUser,
-  thirdPartyUser: ThirdPartyUser
-) {
-  const pictureUrl = thirdPartyUser.profile?._json.picture
-  if (pictureUrl) {
-    const response = await fetch(pictureUrl)
-
-    if (response.status === 200) {
-      const type = response.headers.get("content-type") as string
-      if (type.startsWith("image/")) {
-        user.pictureUrl = pictureUrl
-      }
-    }
-  }
-
-  return user
-}
-
-/**
- * @returns a user that has been sync'd with third party information
- */
-async function syncUser(user: ThirdPartyUser, thirdPartyUser: ThirdPartyUser) {
-  // provider
-  user.provider = thirdPartyUser.provider
-  user.providerType = thirdPartyUser.providerType
-
-  if (thirdPartyUser.profile) {
-    const profile = thirdPartyUser.profile
-
-    if (profile.name) {
-      const name = profile.name
-      // first name
-      if (name.givenName) {
-        user.firstName = name.givenName
-      }
-      // last name
-      if (name.familyName) {
-        user.lastName = name.familyName
-      }
-    }
-
-    user = await syncProfilePicture(user, thirdPartyUser)
-
-    // profile
-    user.thirdPartyProfile = {
-      ...profile._json,
-    }
-  }
-
-  // oauth tokens for future use
-  if (thirdPartyUser.oauth2) {
-    user.oauth2 = {
-      ...thirdPartyUser.oauth2,
-    }
-  }
-
-  return user
-}

packages/backend-core/src/middleware/passport/utils.ts

@@ -1,6 +1,6 @@
-import { isMultiTenant, getTenantId } from "../../tenancy"
+import { isMultiTenant, getTenantId } from "../../context"
 import { getScopedConfig } from "../../db"
-import { ConfigType, Database, Config } from "@budibase/types"
+import { ConfigType, Database } from "@budibase/types"
 
 /**
  * Utility to handle authentication errors.

packages/backend-core/src/middleware/tenancy.ts

@@ -1,4 +1,5 @@
-import { doInTenant, getTenantIDFromCtx } from "../tenancy"
+import { doInTenant } from "../context"
+import { getTenantIDFromCtx } from "../tenancy"
 import { buildMatcherRegex, matches } from "./matchers"
 import { Header } from "../constants"
 import {

packages/backend-core/src/migrations/migrations.ts

@@ -7,7 +7,7 @@ import {
   doWithDB,
 } from "../db"
 import environment from "../environment"
-import { doInTenant, getTenantIds, getTenantId } from "../tenancy"
+import * as platform from "../platform"
 import * as context from "../context"
 import { DEFINITIONS } from "."
 import {
@@ -47,7 +47,7 @@ export const runMigration = async (
   const migrationType = migration.type
   let tenantId: string | undefined
   if (migrationType !== MigrationType.INSTALLATION) {
-    tenantId = getTenantId()
+    tenantId = context.getTenantId()
   }
   const migrationName = migration.name
   const silent = migration.silent
@@ -160,7 +160,7 @@ export const runMigrations = async (
       tenantIds = [options.noOp.tenantId]
     } else if (!options.tenantIds || !options.tenantIds.length) {
       // run for all tenants
-      tenantIds = await getTenantIds()
+      tenantIds = await platform.tenants.getTenantIds()
     } else {
       tenantIds = options.tenantIds
     }
@@ -185,7 +185,7 @@ export const runMigrations = async (
     // for all migrations
     for (const migration of migrations) {
       // run the migration
-      await doInTenant(tenantId, () => runMigration(migration, options))
+      await context.doInTenant(tenantId, () => runMigration(migration, options))
     }
   }
   console.log("Migrations complete")

packages/backend-core/src/objectStore/buckets/global.ts

@@ -1,5 +1,5 @@
 import env from "../../environment"
-import * as tenancy from "../../tenancy"
+import * as context from "../../context"
 import * as objectStore from "../objectStore"
 import * as cloudfront from "../cloudfront"
 
@@ -22,7 +22,7 @@ export const getGlobalFileUrl = (type: string, name: string, etag?: string) => {
 export const getGlobalFileS3Key = (type: string, name: string) => {
   let file = `${type}/${name}`
   if (env.MULTI_TENANCY) {
-    const tenantId = tenancy.getTenantId()
+    const tenantId = context.getTenantId()
     file = `${tenantId}/${file}`
   }
   return file

packages/backend-core/src/objectStore/buckets/plugins.ts

@@ -1,6 +1,6 @@
 import env from "../../environment"
 import * as objectStore from "../objectStore"
-import * as tenancy from "../../tenancy"
+import * as context from "../../context"
 import * as cloudfront from "../cloudfront"
 import { Plugin } from "@budibase/types"
 
@@ -61,7 +61,7 @@ const getPluginS3Key = (plugin: Plugin, fileName: string) => {
 export const getPluginS3Dir = (pluginName: string) => {
   let s3Key = `${pluginName}`
   if (env.MULTI_TENANCY) {
-    const tenantId = tenancy.getTenantId()
+    const tenantId = context.getTenantId()
     s3Key = `${tenantId}/${s3Key}`
   }
   if (env.CLOUDFRONT_CDN) {

packages/backend-core/src/platform/index.ts

@@ -0,0 +1,3 @@
+export * as users from "./users"
+export * as tenants from "./tenants"
+export * from "./platformDb"

packages/backend-core/src/platform/platformDb.ts

@@ -0,0 +1,6 @@
+import { StaticDatabases } from "../constants"
+import { getDB } from "../db/db"
+
+export function getPlatformDB() {
+  return getDB(StaticDatabases.PLATFORM_INFO.name)
+}

packages/backend-core/src/platform/tenants.ts

@@ -0,0 +1,101 @@
+import { StaticDatabases } from "../constants"
+import { getPlatformDB } from "./platformDb"
+import { LockName, LockOptions, LockType, Tenants } from "@budibase/types"
+import * as locks from "../redis/redlock"
+
+const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
+
+export const tenacyLockOptions: LockOptions = {
+  type: LockType.DEFAULT,
+  name: LockName.UPDATE_TENANTS_DOC,
+  ttl: 10 * 1000, // auto expire after 10 seconds
+  systemLock: true,
+}
+
+// READ
+
+export async function getTenantIds(): Promise<string[]> {
+  const tenants = await getTenants()
+  return tenants.tenantIds
+}
+
+async function getTenants(): Promise<Tenants> {
+  const db = getPlatformDB()
+  let tenants: Tenants
+
+  try {
+    tenants = await db.get(TENANT_DOC)
+  } catch (e: any) {
+    // doesn't exist yet - create
+    if (e.status === 404) {
+      tenants = await createTenantsDoc()
+    } else {
+      throw e
+    }
+  }
+
+  return tenants
+}
+
+export async function exists(tenantId: string) {
+  const tenants = await getTenants()
+  return tenants.tenantIds.indexOf(tenantId) !== -1
+}
+
+// CREATE / UPDATE
+
+function newTenantsDoc(): Tenants {
+  return {
+    _id: TENANT_DOC,
+    tenantIds: [],
+  }
+}
+
+async function createTenantsDoc(): Promise<Tenants> {
+  const db = getPlatformDB()
+  let tenants = newTenantsDoc()
+
+  try {
+    const response = await db.put(tenants)
+    tenants._rev = response.rev
+  } catch (e: any) {
+    // don't throw 409 is doc has already been created
+    if (e.status === 409) {
+      return db.get(TENANT_DOC)
+    }
+    throw e
+  }
+
+  return tenants
+}
+
+export async function addTenant(tenantId: string) {
+  const db = getPlatformDB()
+
+  // use a lock as tenant creation is conflict prone
+  await locks.doWithLock(tenacyLockOptions, async () => {
+    const tenants = await getTenants()
+
+    // write the new tenant if it doesn't already exist
+    if (tenants.tenantIds.indexOf(tenantId) === -1) {
+      tenants.tenantIds.push(tenantId)
+      await db.put(tenants)
+    }
+  })
+}
+
+// DELETE
+
+export async function removeTenant(tenantId: string) {
+  try {
+    await locks.doWithLock(tenacyLockOptions, async () => {
+      const db = getPlatformDB()
+      const tenants = await getTenants()
+      tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)
+      await db.put(tenants)
+    })
+  } catch (err) {
+    console.error(`Error removing tenant ${tenantId} from info db`, err)
+    throw err
+  }
+}

packages/backend-core/src/platform/tests/tenants.spec.ts

@@ -0,0 +1,25 @@
+import { DBTestConfiguration, structures } from "../../../tests"
+import * as tenants from "../tenants"
+
+describe("tenants", () => {
+  const config = new DBTestConfiguration()
+
+  describe("addTenant", () => {
+    it("concurrently adds multiple tenants safely", async () => {
+      const tenant1 = structures.tenant.id()
+      const tenant2 = structures.tenant.id()
+      const tenant3 = structures.tenant.id()
+
+      await Promise.all([
+        tenants.addTenant(tenant1),
+        tenants.addTenant(tenant2),
+        tenants.addTenant(tenant3),
+      ])
+
+      const tenantIds = await tenants.getTenantIds()
+      expect(tenantIds.includes(tenant1)).toBe(true)
+      expect(tenantIds.includes(tenant2)).toBe(true)
+      expect(tenantIds.includes(tenant3)).toBe(true)
+    })
+  })
+})

packages/backend-core/src/platform/users.ts

@@ -0,0 +1,90 @@
+import { getPlatformDB } from "./platformDb"
+import { DEFAULT_TENANT_ID } from "../constants"
+import env from "../environment"
+import {
+  PlatformUser,
+  PlatformUserByEmail,
+  PlatformUserById,
+  User,
+} from "@budibase/types"
+
+// READ
+
+export async function lookupTenantId(userId: string) {
+  if (!env.MULTI_TENANCY) {
+    return DEFAULT_TENANT_ID
+  }
+
+  const user = await getUserDoc(userId)
+  return user.tenantId
+}
+
+async function getUserDoc(emailOrId: string): Promise<PlatformUser> {
+  const db = getPlatformDB()
+  return db.get(emailOrId)
+}
+
+// CREATE
+
+function newUserIdDoc(id: string, tenantId: string): PlatformUserById {
+  return {
+    _id: id,
+    tenantId,
+  }
+}
+
+function newUserEmailDoc(
+  userId: string,
+  email: string,
+  tenantId: string
+): PlatformUserByEmail {
+  return {
+    _id: email,
+    userId,
+    tenantId,
+  }
+}
+
+/**
+ * Add a new user id or email doc if it doesn't exist.
+ */
+async function addUserDoc(emailOrId: string, newDocFn: () => PlatformUser) {
+  const db = getPlatformDB()
+  let user: PlatformUser
+
+  try {
+    await db.get(emailOrId)
+  } catch (e: any) {
+    if (e.status === 404) {
+      user = newDocFn()
+      await db.put(user)
+    } else {
+      throw e
+    }
+  }
+}
+
+export async function addUser(tenantId: string, userId: string, email: string) {
+  await Promise.all([
+    addUserDoc(userId, () => newUserIdDoc(userId, tenantId)),
+    addUserDoc(email, () => newUserEmailDoc(userId, email, tenantId)),
+  ])
+}
+
+// DELETE
+
+export async function removeUser(user: User) {
+  const db = getPlatformDB()
+  const keys = [user._id!, user.email]
+  const userDocs = await db.allDocs({
+    keys,
+    include_docs: true,
+  })
+  const toDelete = userDocs.rows.map((row: any) => {
+    return {
+      ...row.doc,
+      _deleted: true,
+    }
+  })
+  await db.bulkDocs(toDelete)
+}

packages/backend-core/src/queue/queue.ts

@@ -4,7 +4,6 @@ import { JobQueue } from "./constants"
 import InMemoryQueue from "./inMemoryQueue"
 import BullQueue from "bull"
 import { addListeners, StalledFn } from "./listeners"
-const { opts: redisOpts, redisProtocolUrl } = getRedisOptions()
 
 const CLEANUP_PERIOD_MS = 60 * 1000
 let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
@@ -20,6 +19,7 @@ export function createQueue<T>(
   jobQueue: JobQueue,
   opts: { removeStalledCb?: StalledFn } = {}
 ): BullQueue.Queue<T> {
+  const { opts: redisOpts, redisProtocolUrl } = getRedisOptions()
   const queueConfig: any = redisProtocolUrl || { redis: redisOpts }
   let queue: any
   if (!env.isTest()) {

packages/backend-core/src/redis/index.ts

@@ -3,4 +3,4 @@
 export { default as Client } from "./redis"
 export * as utils from "./utils"
 export * as clients from "./init"
-export * as redlock from "./redlock"
+export * as locks from "./redlock"

packages/backend-core/src/redis/redis.ts

@@ -1,6 +1,6 @@
 import env from "../environment"
 // ioredis mock is all in memory
-const Redis = env.isTest() ? require("ioredis-mock") : require("ioredis")
+const Redis = env.MOCK_REDIS ? require("ioredis-mock") : require("ioredis")
 import {
   addDbPrefix,
   removeDbPrefix,
@@ -17,8 +17,13 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT
 // for testing just generate the client once
 let CLOSED = false
 let CLIENTS: { [key: number]: any } = {}
-// if in test always connected
+
-let CONNECTED = env.isTest()
+let CONNECTED = false
+
+// mock redis always connected
+if (env.MOCK_REDIS) {
+  CONNECTED = true
+}
 
 function pickClient(selectDb: number): any {
   return CLIENTS[selectDb]
@@ -57,7 +62,7 @@ function init(selectDb = DEFAULT_SELECT_DB) {
     return
   }
   // testing uses a single in memory client
-  if (env.isTest()) {
+  if (env.MOCK_REDIS) {
     CLIENTS[selectDb] = new Redis(getRedisOptions())
   }
   // start the timer - only allowed 5 seconds to connect

packages/backend-core/src/redis/redlock.ts

@@ -1,29 +1,22 @@
 import Redlock, { Options } from "redlock"
 import { getLockClient } from "./init"
 import { LockOptions, LockType } from "@budibase/types"
-import * as tenancy from "../tenancy"
+import * as context from "../context"
-
+import env from "../environment"
-let noRetryRedlock: Redlock | undefined
 
 const getClient = async (type: LockType): Promise<Redlock> => {
+  if (env.isTest() && type !== LockType.TRY_ONCE) {
+    return newRedlock(OPTIONS.TEST)
+  }
   switch (type) {
     case LockType.TRY_ONCE: {
-      if (!noRetryRedlock) {
+      return newRedlock(OPTIONS.TRY_ONCE)
-        noRetryRedlock = await newRedlock(OPTIONS.TRY_ONCE)
-      }
-      return noRetryRedlock
     }
     case LockType.DEFAULT: {
-      if (!noRetryRedlock) {
+      return newRedlock(OPTIONS.DEFAULT)
-        noRetryRedlock = await newRedlock(OPTIONS.DEFAULT)
-      }
-      return noRetryRedlock
     }
     case LockType.DELAY_500: {
-      if (!noRetryRedlock) {
+      return newRedlock(OPTIONS.DELAY_500)
-        noRetryRedlock = await newRedlock(OPTIONS.DELAY_500)
-      }
-      return noRetryRedlock
     }
     default: {
       throw new Error(`Could not get redlock client: ${type}`)
@@ -36,6 +29,11 @@ export const OPTIONS = {
     // immediately throws an error if the lock is already held
     retryCount: 0,
   },
+  TEST: {
+    // higher retry count in unit tests
+    // due to high contention.
+    retryCount: 100,
+  },
   DEFAULT: {
     // the expected clock drift; for more details
     // see http://redis.io/topics/distlock
@@ -69,12 +67,19 @@ export const doWithLock = async (opts: LockOptions, task: any) => {
   const redlock = await getClient(opts.type)
   let lock
   try {
-    // aquire lock
+    // determine lock name
-    let name: string = `lock:${tenancy.getTenantId()}_${opts.name}`
+    // by default use the tenantId for uniqueness, unless using a system lock
+    const prefix = opts.systemLock ? "system" : context.getTenantId()
+    let name: string = `lock:${prefix}_${opts.name}`
+
+    // add additional unique name if required
     if (opts.nameSuffix) {
       name = name + `_${opts.nameSuffix}`
     }
+
+    // create the lock
     lock = await redlock.lock(name, opts.ttl)
+
     // perform locked task
     // need to await to ensure completion before unlocking
     const result = await task()

packages/backend-core/src/redis/utils.ts

@@ -2,8 +2,6 @@ import env from "../environment"
 
 const SLOT_REFRESH_MS = 2000
 const CONNECT_TIMEOUT_MS = 10000
-const REDIS_URL = !env.REDIS_URL ? "localhost:6379" : env.REDIS_URL
-const REDIS_PASSWORD = !env.REDIS_PASSWORD ? "budibase" : env.REDIS_PASSWORD
 export const SEPARATOR = "-"
 
 /**
@@ -60,8 +58,8 @@ export enum SelectableDatabase {
 }
 
 export function getRedisOptions(clustered = false) {
-  let password = REDIS_PASSWORD
+  let password = env.REDIS_PASSWORD
-  let url: string[] | string = REDIS_URL.split("//")
+  let url: string[] | string = env.REDIS_URL.split("//")
   // get rid of the protocol
   url = url.length > 1 ? url[1] : url[0]
   // check for a password etc
@@ -78,8 +76,8 @@ export function getRedisOptions(clustered = false) {
   let redisProtocolUrl
 
   // fully qualified redis URL
-  if (/rediss?:\/\//.test(REDIS_URL)) {
+  if (/rediss?:\/\//.test(env.REDIS_URL)) {
-    redisProtocolUrl = REDIS_URL
+    redisProtocolUrl = env.REDIS_URL
   }
 
   const opts: any = {

packages/backend-core/src/tenancy/db.ts

@@ -0,0 +1,6 @@
+import { getDB } from "../db/db"
+import { getGlobalDBName } from "../context"
+
+export function getTenantDB(tenantId: string) {
+  return getDB(getGlobalDBName(tenantId))
+}

packages/backend-core/src/tenancy/index.ts

@@ -1,2 +1,2 @@
-export * from "../context"
+export * from "./db"
 export * from "./tenancy"

packages/backend-core/src/tenancy/tenancy.ts

@@ -1,4 +1,3 @@
-import { doWithDB, getGlobalDBName } from "../db"
 import {
   DEFAULT_TENANT_ID,
   getTenantId,
@@ -11,10 +10,7 @@ import {
   TenantResolutionStrategy,
   GetTenantIdOptions,
 } from "@budibase/types"
-import { Header, StaticDatabases } from "../constants"
+import { Header } from "../constants"
-
-const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
-const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
 
 export function addTenantToUrl(url: string) {
   const tenantId = getTenantId()
@@ -27,89 +23,6 @@ export function addTenantToUrl(url: string) {
   return url
 }
 
-export async function doesTenantExist(tenantId: string) {
-  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
-    let tenants
-    try {
-      tenants = await db.get(TENANT_DOC)
-    } catch (err) {
-      // if theres an error the doc doesn't exist, no tenants exist
-      return false
-    }
-    return (
-      tenants &&
-      Array.isArray(tenants.tenantIds) &&
-      tenants.tenantIds.indexOf(tenantId) !== -1
-    )
-  })
-}
-
-export async function tryAddTenant(
-  tenantId: string,
-  userId: string,
-  email: string,
-  afterCreateTenant: () => Promise<void>
-) {
-  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
-    const getDoc = async (id: string) => {
-      if (!id) {
-        return null
-      }
-      try {
-        return await db.get(id)
-      } catch (err) {
-        return { _id: id }
-      }
-    }
-    let [tenants, userIdDoc, emailDoc] = await Promise.all([
-      getDoc(TENANT_DOC),
-      getDoc(userId),
-      getDoc(email),
-    ])
-    if (!Array.isArray(tenants.tenantIds)) {
-      tenants = {
-        _id: TENANT_DOC,
-        tenantIds: [],
-      }
-    }
-    let promises = []
-    if (userIdDoc) {
-      userIdDoc.tenantId = tenantId
-      promises.push(db.put(userIdDoc))
-    }
-    if (emailDoc) {
-      emailDoc.tenantId = tenantId
-      emailDoc.userId = userId
-      promises.push(db.put(emailDoc))
-    }
-    if (tenants.tenantIds.indexOf(tenantId) === -1) {
-      tenants.tenantIds.push(tenantId)
-      promises.push(db.put(tenants))
-      await afterCreateTenant()
-    }
-    await Promise.all(promises)
-  })
-}
-
-export function doWithGlobalDB(tenantId: string, cb: any) {
-  return doWithDB(getGlobalDBName(tenantId), cb)
-}
-
-export async function lookupTenantId(userId: string) {
-  return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: any) => {
-    let tenantId = env.MULTI_TENANCY ? DEFAULT_TENANT_ID : null
-    try {
-      const doc = await db.get(userId)
-      if (doc && doc.tenantId) {
-        tenantId = doc.tenantId
-      }
-    } catch (err) {
-      // just return the default
-    }
-    return tenantId
-  })
-}
-
 export const isUserInAppTenant = (appId: string, user?: any) => {
   let userTenantId
   if (user) {
@@ -121,19 +34,6 @@ export const isUserInAppTenant = (appId: string, user?: any) => {
   return tenantId === userTenantId
 }
 
-export async function getTenantIds() {
-  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
-    let tenants
-    try {
-      tenants = await db.get(TENANT_DOC)
-    } catch (err) {
-      // if theres an error the doc doesn't exist, no tenants exist
-      return []
-    }
-    return (tenants && tenants.tenantIds) || []
-  })
-}
-
 const ALL_STRATEGIES = Object.values(TenantResolutionStrategy)
 
 export const getTenantIDFromCtx = (

packages/backend-core/src/users.ts

@@ -8,6 +8,7 @@ import {
 } from "./db"
 import { BulkDocsResponse, User } from "@budibase/types"
 import { getGlobalDB } from "./context"
+import * as context from "./context"
 
 export const bulkGetGlobalUsersById = async (userIds: string[]) => {
   const db = getGlobalDB()
@@ -24,6 +25,11 @@ export const bulkUpdateGlobalUsers = async (users: User[]) => {
   return (await db.bulkDocs(users)) as BulkDocsResponse
 }
 
+export async function getById(id: string): Promise<User> {
+  const db = context.getGlobalDB()
+  return db.get(id)
+}
+
 /**
  * Given an email address this will use a view to search through
  * all the users to find one with this email address.

packages/backend-core/src/utils/tests/utils.spec.ts

@@ -1,21 +1,12 @@
-import { structures } from "../../../tests"
+import { structures, DBTestConfiguration } from "../../../tests"
 import * as utils from "../../utils"
-import * as events from "../../events"
 import * as db from "../../db"
 import { Header } from "../../constants"
-import { doInTenant } from "../../context"
 import { newid } from "../../utils"
+import env from "../../environment"
 
 describe("utils", () => {
-  describe("platformLogout", () => {
+  const config = new DBTestConfiguration()
-    it("should call platform logout", async () => {
-      await doInTenant(structures.tenant.id(), async () => {
-        const ctx = structures.koa.newContext()
-        await utils.platformLogout({ ctx, userId: "test" })
-        expect(events.auth.logout).toBeCalledTimes(1)
-      })
-    })
-  })
 
   describe("getAppIdFromCtx", () => {
     it("gets appId from header", async () => {
@@ -50,21 +41,28 @@ describe("utils", () => {
     })
 
     it("gets appId from url", async () => {
-      const ctx = structures.koa.newContext()
+      await config.doInTenant(async () => {
-      const expected = db.generateAppID()
+        const url = "http://test.com"
-      const app = structures.apps.app(expected)
+        env._set("PLATFORM_URL", url)
 
-      // set custom url
+        const ctx = structures.koa.newContext()
-      const appUrl = newid()
+        ctx.host = `${config.tenantId}.test.com`
-      app.url = `/${appUrl}`
-      ctx.path = `/app/${appUrl}`
 
-      // save the app
+        const expected = db.generateAppID(config.tenantId)
-      const database = db.getDB(expected)
+        const app = structures.apps.app(expected)
-      await database.put(app)
 
-      const actual = await utils.getAppIdFromCtx(ctx)
+        // set custom url
-      expect(actual).toBe(expected)
+        const appUrl = newid()
+        app.url = `/${appUrl}`
+        ctx.path = `/app/${appUrl}`
+
+        // save the app
+        const database = db.getDB(expected)
+        await database.put(app)
+
+        const actual = await utils.getAppIdFromCtx(ctx)
+        expect(actual).toBe(expected)
+      })
     })
 
     it("doesn't get appId from url when previewing", async () => {

packages/backend-core/src/utils/utils.ts

@@ -2,23 +2,15 @@ import { getAllApps, queryGlobalView } from "../db"
 import { options } from "../middleware/passport/jwt"
 import {
   Header,
-  Cookie,
   MAX_VALID_DATE,
   DocumentType,
   SEPARATOR,
   ViewName,
 } from "../constants"
 import env from "../environment"
-import * as userCache from "../cache/user"
-import { getSessionsForUser, invalidateSessions } from "../security/sessions"
-import * as events from "../events"
 import * as tenancy from "../tenancy"
-import {
+import * as context from "../context"
-  App,
+import { App, Ctx, TenantResolutionStrategy } from "@budibase/types"
-  Ctx,
-  PlatformLogoutOpts,
-  TenantResolutionStrategy,
-} from "@budibase/types"
 import { SetOption } from "cookies"
 const jwt = require("jsonwebtoken")
 
@@ -38,7 +30,7 @@ export async function resolveAppUrl(ctx: Ctx) {
   const appUrl = ctx.path.split("/")[2]
   let possibleAppUrl = `/${appUrl.toLowerCase()}`
 
-  let tenantId: string | null = tenancy.getTenantId()
+  let tenantId: string | null = context.getTenantId()
   if (env.MULTI_TENANCY) {
     // always use the tenant id from the subdomain in multi tenancy
     // this ensures the logged-in user tenant id doesn't overwrite
@@ -49,7 +41,7 @@ export async function resolveAppUrl(ctx: Ctx) {
   }
 
   // search prod apps for a url that matches
-  const apps: App[] = await tenancy.doInTenant(tenantId, () =>
+  const apps: App[] = await context.doInTenant(tenantId, () =>
     getAllApps({ dev: false })
   )
   const app = apps.filter(
@@ -222,35 +214,6 @@ export async function getBuildersCount() {
   return builders.length
 }
 
-/**
- * Logs a user out from budibase. Re-used across account portal and builder.
- */
-export async function platformLogout(opts: PlatformLogoutOpts) {
-  const ctx = opts.ctx
-  const userId = opts.userId
-  const keepActiveSession = opts.keepActiveSession
-
-  if (!ctx) throw new Error("Koa context must be supplied to logout.")
-
-  const currentSession = getCookie(ctx, Cookie.Auth)
-  let sessions = await getSessionsForUser(userId)
-
-  if (keepActiveSession) {
-    sessions = sessions.filter(
-      session => session.sessionId !== currentSession.sessionId
-    )
-  } else {
-    // clear cookies
-    clearCookie(ctx, Cookie.Auth)
-    clearCookie(ctx, Cookie.CurrentApp)
-  }
-
-  const sessionIds = sessions.map(({ sessionId }) => sessionId)
-  await invalidateSessions(userId, { sessionIds, reason: "logout" })
-  await events.auth.logout()
-  await userCache.invalidateUser(userId)
-}
-
 export function timeout(timeMs: number) {
   return new Promise(resolve => setTimeout(resolve, timeMs))
 }

packages/backend-core/tests/jestEnv.ts

@@ -1,23 +1,6 @@
-import env from "../src/environment"
+process.env.SELF_HOSTED = "1"
-import { mocks } from "./utilities"
+process.env.MULTI_TENANCY = "1"
-
+process.env.NODE_ENV = "jest"
-// must explicitly enable fetch mock
+process.env.MOCK_REDIS = "1"
-mocks.fetch.enable()
+process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
-
+process.env.ENABLE_4XX_HTTP_LOGGING = "0"
-// mock all dates to 2020-01-01T00:00:00.000Z
-// use tk.reset() to use real dates in individual tests
-import tk from "timekeeper"
-tk.freeze(mocks.date.MOCK_DATE)
-
-env._set("SELF_HOSTED", "1")
-env._set("NODE_ENV", "jest")
-
-if (!process.env.DEBUG) {
-  global.console.log = jest.fn() // console.log are ignored in tests
-}
-
-if (!process.env.CI) {
-  // set a longer timeout in dev for debugging
-  // 100 seconds
-  jest.setTimeout(100000)
-}

packages/backend-core/tests/jestSetup.ts

@@ -1,4 +1,23 @@
+import "./logging"
 import env from "../src/environment"
-import { testContainerUtils } from "./utilities"
+import { mocks, testContainerUtils } from "./utilities"
+
+// must explicitly enable fetch mock
+mocks.fetch.enable()
+
+// mock all dates to 2020-01-01T00:00:00.000Z
+// use tk.reset() to use real dates in individual tests
+import tk from "timekeeper"
+tk.freeze(mocks.date.MOCK_DATE)
+
+if (!process.env.DEBUG) {
+  console.log = jest.fn() // console.log are ignored in tests
+}
+
+if (!process.env.CI) {
+  // set a longer timeout in dev for debugging
+  // 100 seconds
+  jest.setTimeout(100000)
+}
 
 testContainerUtils.setupEnv(env)

packages/backend-core/tests/logging.ts

@@ -0,0 +1,34 @@
+export enum LogLevel {
+  TRACE = "trace",
+  DEBUG = "debug",
+  INFO = "info",
+  WARN = "warn",
+  ERROR = "error",
+}
+
+const LOG_INDEX: { [key in LogLevel]: number } = {
+  [LogLevel.TRACE]: 1,
+  [LogLevel.DEBUG]: 2,
+  [LogLevel.INFO]: 3,
+  [LogLevel.WARN]: 4,
+  [LogLevel.ERROR]: 5,
+}
+
+const setIndex = LOG_INDEX[process.env.LOG_LEVEL as LogLevel]
+
+if (setIndex > LOG_INDEX.trace) {
+  global.console.trace = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.debug) {
+  global.console.debug = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.info) {
+  global.console.info = jest.fn()
+  global.console.log = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.warn) {
+  global.console.warn = jest.fn()
+}

packages/backend-core/tests/utilities/DBTestConfiguration.ts

@@ -0,0 +1,32 @@
+import "./mocks"
+import * as structures from "./structures"
+import * as testEnv from "./testEnv"
+import * as context from "../../src/context"
+
+class DBTestConfiguration {
+  tenantId: string
+
+  constructor() {
+    // db tests need to be multi tenant to prevent conflicts
+    testEnv.multiTenant()
+    this.tenantId = structures.tenant.id()
+  }
+
+  // TENANCY
+
+  doInTenant(task: any) {
+    return context.doInTenant(this.tenantId, () => {
+      return task()
+    })
+  }
+
+  getTenantId() {
+    try {
+      return context.getTenantId()
+    } catch (e) {
+      return this.tenantId!
+    }
+  }
+}
+
+export default DBTestConfiguration

packages/backend-core/tests/utilities/db.ts

@@ -1,9 +0,0 @@
-import * as db from "../../src/db"
-
-const dbConfig = {
-  inMemory: true,
-}
-
-export const init = () => {
-  db.init(dbConfig)
-}

packages/backend-core/tests/utilities/index.ts

@@ -4,5 +4,4 @@ export { generator } from "./structures"
 export * as testEnv from "./testEnv"
 export * as testContainerUtils from "./testContainerUtils"
 
-import * as dbConfig from "./db"
+export { default as DBTestConfiguration } from "./DBTestConfiguration"
-dbConfig.init()

packages/backend-core/tests/utilities/mocks/accounts.ts

@@ -1,13 +0,0 @@
-const mockGetAccount = jest.fn()
-const mockGetAccountByTenantId = jest.fn()
-const mockGetStatus = jest.fn()
-
-jest.mock("../../../src/cloud/accounts", () => ({
-  getAccount: mockGetAccount,
-  getAccountByTenantId: mockGetAccountByTenantId,
-  getStatus: mockGetStatus,
-}))
-
-export const getAccount = mockGetAccount
-export const getAccountByTenantId = mockGetAccountByTenantId
-export const getStatus = mockGetStatus

packages/backend-core/tests/utilities/mocks/index.ts

@@ -1,4 +1,7 @@
-export * as accounts from "./accounts"
+jest.mock("../../../src/accounts")
+import * as _accounts from "../../../src/accounts"
+export const accounts = jest.mocked(_accounts)
+
 export * as date from "./date"
 export * as licenses from "./licenses"
 export { default as fetch } from "./fetch"

packages/backend-core/tests/utilities/structures/accounts.ts

@@ -1,6 +1,15 @@
 import { generator, uuid } from "."
 import * as db from "../../../src/db/utils"
-import { Account, AuthType, CloudAccount, Hosting } from "@budibase/types"
+import {
+  Account,
+  AccountSSOProvider,
+  AccountSSOProviderType,
+  AuthType,
+  CloudAccount,
+  Hosting,
+  SSOAccount,
+} from "@budibase/types"
+import _ from "lodash"
 
 export const account = (): Account => {
   return {
@@ -27,3 +36,28 @@ export const cloudAccount = (): CloudAccount => {
     budibaseUserId: db.generateGlobalUserID(),
   }
 }
+
+function providerType(): AccountSSOProviderType {
+  return _.sample(
+    Object.values(AccountSSOProviderType)
+  ) as AccountSSOProviderType
+}
+
+function provider(): AccountSSOProvider {
+  return _.sample(Object.values(AccountSSOProvider)) as AccountSSOProvider
+}
+
+export function ssoAccount(): SSOAccount {
+  return {
+    ...cloudAccount(),
+    authType: AuthType.SSO,
+    oauth2: {
+      accessToken: generator.string(),
+      refreshToken: generator.string(),
+    },
+    pictureUrl: generator.url(),
+    provider: provider(),
+    providerType: providerType(),
+    thirdPartyProfile: {},
+  }
+}

packages/backend-core/tests/utilities/structures/index.ts

@@ -5,8 +5,10 @@ export const generator = new Chance()
 
 export * as accounts from "./accounts"
 export * as apps from "./apps"
+export * as db from "./db"
 export * as koa from "./koa"
 export * as licenses from "./licenses"
 export * as plugins from "./plugins"
+export * as sso from "./sso"
 export * as tenant from "./tenants"
-export * as db from "./db"
+export * as users from "./users"

packages/backend-core/tests/utilities/structures/sso.ts

@@ -0,0 +1,100 @@
+import {
+  GoogleInnerConfig,
+  JwtClaims,
+  OIDCInnerConfig,
+  OIDCWellKnownConfig,
+  SSOAuthDetails,
+  SSOProfile,
+  SSOProviderType,
+  User,
+} from "@budibase/types"
+import { uuid, generator, users, email } from "./index"
+import _ from "lodash"
+
+export function providerType(): SSOProviderType {
+  return _.sample(Object.values(SSOProviderType)) as SSOProviderType
+}
+
+export function ssoProfile(user?: User): SSOProfile {
+  if (!user) {
+    user = users.user()
+  }
+  return {
+    id: user._id!,
+    name: {
+      givenName: user.firstName,
+      familyName: user.lastName,
+    },
+    _json: {
+      email: user.email,
+      picture: "http://test.com",
+    },
+    provider: generator.string(),
+  }
+}
+
+export function authDetails(user?: User): SSOAuthDetails {
+  if (!user) {
+    user = users.user()
+  }
+
+  const userId = user._id || uuid()
+  const provider = generator.string()
+
+  const profile = ssoProfile(user)
+  profile.provider = provider
+  profile.id = userId
+
+  return {
+    email: user.email,
+    oauth2: {
+      refreshToken: generator.string(),
+      accessToken: generator.string(),
+    },
+    profile,
+    provider,
+    providerType: providerType(),
+    userId,
+  }
+}
+
+// OIDC
+
+export function oidcConfig(): OIDCInnerConfig {
+  return {
+    uuid: uuid(),
+    activated: true,
+    logo: "",
+    name: generator.string(),
+    configUrl: "http://someconfigurl",
+    clientID: generator.string(),
+    clientSecret: generator.string(),
+  }
+}
+
+// response from .well-known/openid-configuration
+export function oidcWellKnownConfig(): OIDCWellKnownConfig {
+  return {
+    issuer: generator.string(),
+    authorization_endpoint: generator.url(),
+    token_endpoint: generator.url(),
+    userinfo_endpoint: generator.url(),
+  }
+}
+
+export function jwtClaims(): JwtClaims {
+  return {
+    email: email(),
+    preferred_username: email(),
+  }
+}
+
+// GOOGLE
+
+export function googleConfig(): GoogleInnerConfig {
+  return {
+    activated: true,
+    clientID: generator.string(),
+    clientSecret: generator.string(),
+  }
+}

packages/backend-core/tests/utilities/structures/users.ts

@@ -0,0 +1,70 @@
+import { generator } from "../"
+import {
+  AdminUser,
+  BuilderUser,
+  SSOAuthDetails,
+  SSOUser,
+  User,
+} from "@budibase/types"
+import { v4 as uuid } from "uuid"
+import * as sso from "./sso"
+
+export const newEmail = () => {
+  return `${uuid()}@test.com`
+}
+
+export const user = (userProps?: any): User => {
+  return {
+    email: newEmail(),
+    password: "test",
+    roles: { app_test: "admin" },
+    firstName: generator.first(),
+    lastName: generator.last(),
+    pictureUrl: "http://test.com",
+    ...userProps,
+  }
+}
+
+export const adminUser = (userProps?: any): AdminUser => {
+  return {
+    ...user(userProps),
+    admin: {
+      global: true,
+    },
+    builder: {
+      global: true,
+    },
+  }
+}
+
+export const builderUser = (userProps?: any): BuilderUser => {
+  return {
+    ...user(userProps),
+    builder: {
+      global: true,
+    },
+  }
+}
+
+export function ssoUser(
+  opts: { user?: any; details?: SSOAuthDetails } = {}
+): SSOUser {
+  const base = user(opts.user)
+  delete base.password
+
+  if (!opts.details) {
+    opts.details = sso.authDetails(base)
+  }
+
+  return {
+    ...base,
+    forceResetPassword: false,
+    oauth2: opts.details?.oauth2,
+    provider: opts.details?.provider!,
+    providerType: opts.details?.providerType!,
+    thirdPartyProfile: {
+      email: base.email,
+      picture: base.pictureUrl,
+    },
+  }
+}

packages/backend-core/tests/utilities/testContainerUtils.ts

@@ -34,12 +34,17 @@ function getMinioConfig() {
   return getContainerInfo("minio-service", 9000)
 }
 
+function getRedisConfig() {
+  return getContainerInfo("redis-service", 6379)
+}
+
 export function setupEnv(...envs: any[]) {
   const configs = [
     { key: "COUCH_DB_PORT", value: getCouchConfig().port },
     { key: "COUCH_DB_URL", value: getCouchConfig().url },
     { key: "MINIO_PORT", value: getMinioConfig().port },
     { key: "MINIO_URL", value: getMinioConfig().url },
+    { key: "REDIS_URL", value: getRedisConfig().url },
   ]
 
   for (const config of configs.filter(x => !!x.value)) {

packages/backend-core/tests/utilities/testEnv.ts

@@ -1,12 +1,12 @@
 import env from "../../src/environment"
-import * as tenancy from "../../src/tenancy"
+import * as context from "../../src/context"
-import { newid } from "../../src/utils"
+import * as structures from "./structures"
 
 // TENANCY
 
 export async function withTenant(task: (tenantId: string) => any) {
-  const tenantId = newid()
+  const tenantId = structures.tenant.id()
-  return tenancy.doInTenant(tenantId, async () => {
+  return context.doInTenant(tenantId, async () => {
     await task(tenantId)
   })
 }
@@ -19,6 +19,14 @@ export function multiTenant() {
   env._set("MULTI_TENANCY", 1)
 }
 
+export function selfHosted() {
+  env._set("SELF_HOSTED", 1)
+}
+
+export function cloudHosted() {
+  env._set("SELF_HOSTED", 0)
+}
+
 // NODE
 
 export function nodeDev() {

packages/backend-core/yarn.lock

@@ -1197,10 +1197,10 @@
   dependencies:
     "@types/istanbul-lib-report" "*"
 
-"@types/jest@27.5.1":
+"@types/jest@28.1.1":
-  version "27.5.1"
+  version "28.1.1"
-  resolved "https://registry.yarnpkg.com/@types/jest/-/jest-27.5.1.tgz#2c8b6dc6ff85c33bcd07d0b62cb3d19ddfdb3ab9"
+  resolved "https://registry.yarnpkg.com/@types/jest/-/jest-28.1.1.tgz#8c9ba63702a11f8c386ee211280e8b68cb093cd1"
-  integrity sha512-fUy7YRpT+rHXto1YlL+J9rs0uLGyiqVt3ZOTQR+4ROc47yNl8WLdVLgUloBRhOxP1PZvguHl44T3H0wAWxahYQ==
+  integrity sha512-C2p7yqleUKtCkVjlOur9BWVA4HgUQmEj/HWCt5WzZ5mLXrWnyIfl0wGuArc+kBXsy0ZZfLp+7dywB4HtSVYGVA==
   dependencies:
     jest-matcher-utils "^27.0.0"
     pretty-format "^27.0.0"

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.17-alpha.5",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/string-templates": "2.3.17-alpha.4",
+    "@budibase/string-templates": "2.3.17-alpha.5",
     "@spectrum-css/accordion": "3.0.24",
     "@spectrum-css/actionbutton": "1.0.1",
     "@spectrum-css/actiongroup": "1.0.1",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.17-alpha.5",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -58,10 +58,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.3.17-alpha.4",
+    "@budibase/bbui": "2.3.17-alpha.5",
-    "@budibase/client": "2.3.17-alpha.4",
+    "@budibase/client": "2.3.17-alpha.5",
-    "@budibase/frontend-core": "2.3.17-alpha.4",
+    "@budibase/frontend-core": "2.3.17-alpha.5",
-    "@budibase/string-templates": "2.3.17-alpha.4",
+    "@budibase/string-templates": "2.3.17-alpha.5",
     "@fortawesome/fontawesome-svg-core": "^6.2.1",
     "@fortawesome/free-brands-svg-icons": "^6.2.1",
     "@fortawesome/free-solid-svg-icons": "^6.2.1",

packages/builder/setup.js

@@ -19,6 +19,7 @@ process.env.COUCH_DB_USER = "budibase"
 process.env.COUCH_DB_PASSWORD = "budibase"
 process.env.INTERNAL_API_KEY = "budibase"
 process.env.ALLOW_DEV_AUTOMATIONS = 1
+process.env.MOCK_REDIS = 1
 
 // Stop info logs polluting test outputs
 process.env.LOG_LEVEL = "error"

packages/builder/src/pages/builder/portal/_components/UserDropdown.svelte

@@ -30,9 +30,11 @@
     My profile
   </MenuItem>
   <MenuItem icon="Moon" on:click={() => themeModal.show()}>Theme</MenuItem>
-  <MenuItem icon="LockClosed" on:click={() => updatePasswordModal.show()}>
+  {#if !$auth.isSSO}
-    Update password
+    <MenuItem icon="LockClosed" on:click={() => updatePasswordModal.show()}>
-  </MenuItem>
+      Update password
+    </MenuItem>
+  {/if}
   <MenuItem icon="Key" on:click={() => apiKeyModal.show()}>
     View API key
   </MenuItem>

packages/builder/src/pages/builder/portal/users/users/[userId].svelte

@@ -81,6 +81,7 @@
   let user
   let loaded = false
 
+  $: isSSO = !!user?.provider
   $: readonly = !$auth.isAdmin
   $: fullName = user?.firstName ? user?.firstName + " " + user?.lastName : ""
   $: privileged = user?.admin?.global || user?.builder?.global
@@ -246,9 +247,11 @@
             <span slot="control">
               <Icon hoverable name="More" />
             </span>
-            <MenuItem on:click={resetPasswordModal.show} icon="Refresh">
+            {#if !isSSO}
-              Force password reset
+              <MenuItem on:click={resetPasswordModal.show} icon="Refresh">
-            </MenuItem>
+                Force password reset
+              </MenuItem>
+            {/if}
             <MenuItem on:click={deleteModal.show} icon="Delete">
               Delete
             </MenuItem>

packages/builder/src/stores/portal/auth.js

@@ -41,6 +41,7 @@ export function createAuthStore() {
       initials,
       isAdmin,
       isBuilder,
+      isSSO: !!$store.user?.provider,
     }
   })
 

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.17-alpha.5",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -26,9 +26,9 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "2.3.17-alpha.4",
+    "@budibase/backend-core": "2.3.17-alpha.5",
-    "@budibase/string-templates": "2.3.17-alpha.4",
+    "@budibase/string-templates": "2.3.17-alpha.5",
-    "@budibase/types": "2.3.17-alpha.4",
+    "@budibase/types": "2.3.17-alpha.5",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.17-alpha.5",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "2.3.17-alpha.4",
+    "@budibase/bbui": "2.3.17-alpha.5",
-    "@budibase/frontend-core": "2.3.17-alpha.4",
+    "@budibase/frontend-core": "2.3.17-alpha.5",
-    "@budibase/string-templates": "2.3.17-alpha.4",
+    "@budibase/string-templates": "2.3.17-alpha.5",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.17-alpha.5",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "2.3.17-alpha.4",
+    "@budibase/bbui": "2.3.17-alpha.5",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/sdk",
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.17-alpha.5",
   "description": "Budibase Public API SDK",
   "author": "Budibase",
   "license": "MPL-2.0",

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.17-alpha.5",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -43,11 +43,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.3.17-alpha.4",
+    "@budibase/backend-core": "2.3.17-alpha.5",
-    "@budibase/client": "2.3.17-alpha.4",
+    "@budibase/client": "2.3.17-alpha.5",
-    "@budibase/pro": "2.3.17-alpha.4",
+    "@budibase/pro": "2.3.17-alpha.5",
-    "@budibase/string-templates": "2.3.17-alpha.4",
+    "@budibase/string-templates": "2.3.17-alpha.5",
-    "@budibase/types": "2.3.17-alpha.4",
+    "@budibase/types": "2.3.17-alpha.5",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -142,6 +142,7 @@
     "@types/pouchdb": "6.4.0",
     "@types/redis": "4.0.11",
     "@types/server-destroy": "1.0.1",
+    "@types/supertest": "2.0.12",
     "@types/tar": "6.1.3",
     "@typescript-eslint/parser": "5.45.0",
     "apidoc": "0.50.4",
@@ -160,7 +161,7 @@
     "path-to-regexp": "6.2.0",
     "prettier": "2.5.1",
     "rimraf": "3.0.2",
-    "supertest": "4.0.2",
+    "supertest": "6.2.2",
     "swagger-jsdoc": "6.1.0",
     "timekeeper": "2.2.0",
     "ts-jest": "28.0.4",

packages/server/specs/openapi.json

@@ -817,7 +817,6 @@
                       "type": "string",
                       "enum": [
                         "string",
-                        "barcodeqr",
                         "longform",
                         "options",
                         "number",
@@ -829,7 +828,8 @@
                         "formula",
                         "auto",
                         "json",
-                        "internal"
+                        "internal",
+                        "barcodeqr"
                       ],
                       "description": "Defines the type of the column, most explain themselves, a link column is a relationship."
                     },
@@ -1021,7 +1021,6 @@
                           "type": "string",
                           "enum": [
                             "string",
-                            "barcodeqr",
                             "longform",
                             "options",
                             "number",
@@ -1033,7 +1032,8 @@
                             "formula",
                             "auto",
                             "json",
-                            "internal"
+                            "internal",
+                            "barcodeqr"
                           ],
                           "description": "Defines the type of the column, most explain themselves, a link column is a relationship."
                         },
@@ -1236,7 +1236,6 @@
                             "type": "string",
                             "enum": [
                               "string",
-                              "barcodeqr",
                               "longform",
                               "options",
                               "number",
@@ -1248,7 +1247,8 @@
                               "formula",
                               "auto",
                               "json",
-                              "internal"
+                              "internal",
+                              "barcodeqr"
                             ],
                             "description": "Defines the type of the column, most explain themselves, a link column is a relationship."
                           },

packages/server/specs/openapi.yaml

@@ -603,7 +603,6 @@ components:
                     type: string
                     enum:
                       - string
-                      - barcodeqr
                       - longform
                       - options
                       - number
@@ -616,6 +615,7 @@ components:
                       - auto
                       - json
                       - internal
+                      - barcodeqr
                     description: Defines the type of the column, most explain themselves, a link
                       column is a relationship.
                   constraints:
@@ -766,7 +766,6 @@ components:
                         type: string
                         enum:
                           - string
-                          - barcodeqr
                           - longform
                           - options
                           - number
@@ -779,6 +778,7 @@ components:
                           - auto
                           - json
                           - internal
+                          - barcodeqr
                         description: Defines the type of the column, most explain themselves, a link
                           column is a relationship.
                       constraints:
@@ -936,7 +936,6 @@ components:
                           type: string
                           enum:
                             - string
-                            - barcodeqr
                             - longform
                             - options
                             - number
@@ -949,6 +948,7 @@ components:
                             - auto
                             - json
                             - internal
+                            - barcodeqr
                           description: Defines the type of the column, most explain themselves, a link
                             column is a relationship.
                         constraints:

packages/server/src/api/index.ts

@@ -1,5 +1,5 @@
 import Router from "@koa/router"
-import { errors, auth } from "@budibase/backend-core"
+import { auth, middleware } from "@budibase/backend-core"
 import currentApp from "../middleware/currentapp"
 import zlib from "zlib"
 import { mainRoutes, staticRoutes, publicRoutes } from "./routes"
@@ -14,6 +14,8 @@ export const router: Router = new Router()
 router.get("/health", ctx => (ctx.status = 200))
 router.get("/version", ctx => (ctx.body = pkg.version))
 
+router.use(middleware.errorHandling)
+
 router
   .use(
     compress({
@@ -54,27 +56,6 @@ router
   .use(currentApp)
   .use(auth.auditLog)
 
-// error handling middleware
-router.use(async (ctx, next) => {
-  try {
-    await next()
-  } catch (err: any) {
-    ctx.status = err.status || err.statusCode || 500
-    const error = errors.getPublicError(err)
-    ctx.body = {
-      message: err.message,
-      status: ctx.status,
-      validationErrors: err.validation,
-      error,
-    }
-    ctx.log.error(err)
-    // unauthorised errors don't provide a useful trace
-    if (!env.isTest()) {
-      console.trace(err)
-    }
-  }
-})
-
 // authenticated routes
 for (let route of mainRoutes) {
   router.use(route.routes())

packages/server/src/api/routes/tests/backup.spec.ts

@@ -19,7 +19,6 @@ describe("/backups", () => {
         .get(`/api/backups/export?appId=${config.getAppId()}&appname=test`)
         .set(config.defaultHeaders())
         .expect(200)
-      expect(res.text).toBeDefined()
       expect(res.headers["content-type"]).toEqual("application/gzip")
       expect(events.app.exported).toBeCalledTimes(1)
     })

packages/server/src/integrations/tests/couchdb.spec.ts

@@ -1,5 +1,3 @@
-import { DatabaseWithConnection } from "@budibase/backend-core/src/db"
-
 jest.mock("@budibase/backend-core", () => {
   const core = jest.requireActual("@budibase/backend-core")
   return {

packages/server/src/tests/jestEnv.ts

@@ -1,9 +1,10 @@
-import env from "../environment"
 import { tmpdir } from "os"
 
-env._set("SELF_HOSTED", "1")
+process.env.SELF_HOSTED = "1"
-env._set("NODE_ENV", "jest")
+process.env.NODE_ENV = "jest"
-env._set("MULTI_TENANCY", "1")
+process.env.MULTI_TENANCY = "1"
 // @ts-ignore
-env._set("BUDIBASE_DIR", tmpdir("budibase-unittests"))
+process.env.BUDIBASE_DIR = tmpdir("budibase-unittests")
-env._set("LOG_LEVEL", "silent")
+process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error"
+process.env.ENABLE_4XX_HTTP_LOGGING = "0"
+process.env.MOCK_REDIS = "1"

packages/server/src/tests/jestSetup.ts

@@ -1,3 +1,4 @@
+import "./logging"
 import env from "../environment"
 import { env as coreEnv } from "@budibase/backend-core"
 import { testContainerUtils } from "@budibase/backend-core/tests"

packages/server/src/tests/logging.ts

@@ -0,0 +1,34 @@
+export enum LogLevel {
+  TRACE = "trace",
+  DEBUG = "debug",
+  INFO = "info",
+  WARN = "warn",
+  ERROR = "error",
+}
+
+const LOG_INDEX: { [key in LogLevel]: number } = {
+  [LogLevel.TRACE]: 1,
+  [LogLevel.DEBUG]: 2,
+  [LogLevel.INFO]: 3,
+  [LogLevel.WARN]: 4,
+  [LogLevel.ERROR]: 5,
+}
+
+const setIndex = LOG_INDEX[process.env.LOG_LEVEL as LogLevel]
+
+if (setIndex > LOG_INDEX.trace) {
+  global.console.trace = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.debug) {
+  global.console.debug = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.info) {
+  global.console.info = jest.fn()
+  global.console.log = jest.fn()
+}
+
+if (setIndex > LOG_INDEX.warn) {
+  global.console.warn = jest.fn()
+}

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -236,42 +236,41 @@ class TestConfiguration {
     email = this.defaultUserValues.email,
     roles,
   }: any = {}) {
-    return tenancy.doWithGlobalDB(this.getTenantId(), async (db: Database) => {
+    const db = tenancy.getTenantDB(this.getTenantId())
-      let existing
+    let existing
-      try {
+    try {
-        existing = await db.get(id)
+      existing = await db.get(id)
-      } catch (err) {
+    } catch (err) {
-        existing = { email }
+      existing = { email }
-      }
+    }
-      const user = {
+    const user = {
-        _id: id,
+      _id: id,
-        ...existing,
+      ...existing,
-        roles: roles || {},
+      roles: roles || {},
-        tenantId: this.getTenantId(),
+      tenantId: this.getTenantId(),
-        firstName,
+      firstName,
-        lastName,
+      lastName,
-      }
+    }
-      await sessions.createASession(id, {
+    await sessions.createASession(id, {
-        sessionId: "sessionid",
+      sessionId: "sessionid",
-        tenantId: this.getTenantId(),
+      tenantId: this.getTenantId(),
-        csrfToken: this.defaultUserValues.csrfToken,
+      csrfToken: this.defaultUserValues.csrfToken,
-      })
-      if (builder) {
-        user.builder = { global: true }
-      } else {
-        user.builder = { global: false }
-      }
-      if (admin) {
-        user.admin = { global: true }
-      } else {
-        user.admin = { global: false }
-      }
-      const resp = await db.put(user)
-      return {
-        _rev: resp.rev,
-        ...user,
-      }
     })
+    if (builder) {
+      user.builder = { global: true }
+    } else {
+      user.builder = { global: false }
+    }
+    if (admin) {
+      user.admin = { global: true }
+    } else {
+      user.admin = { global: false }
+    }
+    const resp = await db.put(user)
+    return {
+      _rev: resp.rev,
+      ...user,
+    }
   }
 
   async createUser(
@@ -417,20 +416,19 @@ class TestConfiguration {
   // API
 
   async generateApiKey(userId = this.defaultUserValues.globalUserId) {
-    return tenancy.doWithGlobalDB(this.getTenantId(), async (db: any) => {
+    const db = tenancy.getTenantDB(this.getTenantId())
-      const id = dbCore.generateDevInfoID(userId)
+    const id = dbCore.generateDevInfoID(userId)
-      let devInfo
+    let devInfo
-      try {
+    try {
-        devInfo = await db.get(id)
+      devInfo = await db.get(id)
-      } catch (err) {
+    } catch (err) {
-        devInfo = { _id: id, userId }
+      devInfo = { _id: id, userId }
-      }
+    }
-      devInfo.apiKey = encryption.encrypt(
+    devInfo.apiKey = encryption.encrypt(
-        `${this.getTenantId()}${dbCore.SEPARATOR}${newid()}`
+      `${this.getTenantId()}${dbCore.SEPARATOR}${newid()}`
-      )
+    )
-      await db.put(devInfo)
+    await db.put(devInfo)
-      return devInfo.apiKey
+    return devInfo.apiKey
-    })
   }
 
   // APP

packages/server/yarn.lock

@@ -1278,14 +1278,14 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@2.3.17-alpha.4":
+"@budibase/backend-core@2.3.17-alpha.5":
-  version "2.3.17-alpha.4"
+  version "2.3.17-alpha.5"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.3.17-alpha.4.tgz#efe13cc2eb9a02223cff46228af8370a5a91db11"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.3.17-alpha.5.tgz#5bd380c9bf3835694729f1c68c44a77aec7b66dd"
-  integrity sha512-h9aCz+5uAQOvQiimULShKkXdQgyiwvgcxq3wqEvNHsAxmlslaJJouStWhvxBdsFiOW9K2CyI/8PeUBBRlAU0AQ==
+  integrity sha512-Nv/BsmItQcdi30oHvNEeRRkDxooMR3shZksb14NDWfdP9gxjVOkUltA2HELujL2ZEg4mismau/0oy9kZy5z3Og==
   dependencies:
     "@budibase/nano" "10.1.1"
     "@budibase/pouchdb-replication-stream" "1.2.10"
-    "@budibase/types" "2.3.17-alpha.4"
+    "@budibase/types" "2.3.17-alpha.5"
     "@shopify/jest-koa-mocks" "5.0.1"
     "@techpass/passport-openidconnect" "0.3.2"
     aws-cloudfront-sign "2.2.0"
@@ -1392,13 +1392,13 @@
     pouchdb-promise "^6.0.4"
     through2 "^2.0.0"
 
-"@budibase/pro@2.3.17-alpha.4":
+"@budibase/pro@2.3.17-alpha.5":
-  version "2.3.17-alpha.4"
+  version "2.3.17-alpha.5"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.3.17-alpha.4.tgz#56e945bd960eca8c8b2a04a3dd606df392bf5792"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.3.17-alpha.5.tgz#4007ab38a5a88cedc1c0ba48543fcbfdf0c58a2a"
-  integrity sha512-DJtKCc5/XXAnrvI5sS+joVuDYOLgwFmDYwLbssJgSQzGXNqepN/ikGq6eFCKn/99fRYBGREfwXwAlxXveZxPWA==
+  integrity sha512-pR7i7ehpoJ08Pq4Xs5vFJcZLvLkfCDS3AvJV1e0j9IDPD73kqiqCxgOF9rE27NTLf0BvbPK5Vj4Z7RPCxIJv7g==
   dependencies:
-    "@budibase/backend-core" "2.3.17-alpha.4"
+    "@budibase/backend-core" "2.3.17-alpha.5"
-    "@budibase/types" "2.3.17-alpha.4"
+    "@budibase/types" "2.3.17-alpha.5"
     "@koa/router" "8.0.8"
     bull "4.10.1"
     joi "17.6.0"
@@ -1424,10 +1424,10 @@
     svelte-apexcharts "^1.0.2"
     svelte-flatpickr "^3.1.0"
 
-"@budibase/types@2.3.17-alpha.4":
+"@budibase/types@2.3.17-alpha.5":
-  version "2.3.17-alpha.4"
+  version "2.3.17-alpha.5"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.3.17-alpha.4.tgz#63eb756160da7ace5459e303874f5e67425c6988"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.3.17-alpha.5.tgz#a8c31f49ac9f8b312317fd6fe547ea76d89d059b"
-  integrity sha512-us/gGZPHimHsYNAnJ5yGdxeThT065wofJ2sfg0aD81P8nq3F2mevBoUmci/KRYdrQ9EsgpZ3Ou6CqnhnV8WmfA==
+  integrity sha512-qp51pVnOaUf42plQjPlCXZVMTIFKrkkhiJ+XaClnekRmyAYaz7JUIex+QTDbmUF4i0WbZRc0MgUCVQxlKkWotA==
 
 "@bull-board/api@3.7.0":
   version "3.7.0"
@@ -3594,6 +3594,14 @@
   resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.1.tgz#20f18294f797f2209b5f65c8e3b5c8e8261d127c"
   integrity sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==
 
+"@types/superagent@*":
+  version "4.1.16"
+  resolved "https://registry.yarnpkg.com/@types/superagent/-/superagent-4.1.16.tgz#12c9c16f232f9d89beab91d69368f96ce8e2d881"
+  integrity sha512-tLfnlJf6A5mB6ddqF159GqcDizfzbMUB1/DeT59/wBNqzRTNNKsaw79A/1TZ84X+f/EwWH8FeuSkjlCLyqS/zQ==
+  dependencies:
+    "@types/cookiejar" "*"
+    "@types/node" "*"
+
 "@types/superagent@^4.1.12":
   version "4.1.15"
   resolved "https://registry.yarnpkg.com/@types/superagent/-/superagent-4.1.15.tgz#63297de457eba5e2bc502a7609426c4cceab434a"
@@ -3602,6 +3610,13 @@
     "@types/cookiejar" "*"
     "@types/node" "*"
 
+"@types/supertest@2.0.12":
+  version "2.0.12"
+  resolved "https://registry.yarnpkg.com/@types/supertest/-/supertest-2.0.12.tgz#ddb4a0568597c9aadff8dbec5b2e8fddbe8692fc"
+  integrity sha512-X3HPWTwXRerBZS7Mo1k6vMVR1Z6zmJcDVn5O/31whe0tnjE4te6ZJSJGq1RiqHPjzPdMTfjCFogDJmwng9xHaQ==
+  dependencies:
+    "@types/superagent" "*"
+
 "@types/tar@6.1.3":
   version "6.1.3"
   resolved "https://registry.yarnpkg.com/@types/tar/-/tar-6.1.3.tgz#46a2ce7617950c4852dfd7e9cd41aa8161b9d750"
@@ -4288,7 +4303,7 @@ arrify@^2.0.0:
   resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa"
   integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==
 
-asap@^2.0.3:
+asap@^2.0.0, asap@^2.0.3:
   version "2.0.6"
   resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46"
   integrity sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==
@@ -5463,7 +5478,7 @@ commoner@^0.10.1:
     q "^1.1.2"
     recast "^0.11.17"
 
-component-emitter@^1.2.0, component-emitter@^1.2.1:
+component-emitter@^1.2.1, component-emitter@^1.3.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.3.0.tgz#16e4070fba8ae29b679f2215853ee181ab2eabc0"
   integrity sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==
@@ -5565,7 +5580,7 @@ cookie@^0.5.0:
   resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b"
   integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==
 
-cookiejar@^2.1.0:
+cookiejar@^2.1.3:
   version "2.1.4"
   resolved "https://registry.yarnpkg.com/cookiejar/-/cookiejar-2.1.4.tgz#ee669c1fea2cf42dc31585469d193fef0d65771b"
   integrity sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==
@@ -6057,6 +6072,14 @@ detective@^4.3.1:
     acorn "^5.2.1"
     defined "^1.0.0"
 
+dezalgo@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/dezalgo/-/dezalgo-1.0.4.tgz#751235260469084c132157dfa857f386d4c33d81"
+  integrity sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==
+  dependencies:
+    asap "^2.0.0"
+    wrappy "1"
+
 diagnostics_channel@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/diagnostics_channel/-/diagnostics_channel-1.1.0.tgz#bd66c49124ce3bac697dff57466464487f57cea5"
@@ -6990,7 +7013,7 @@ extend-shallow@^3.0.0, extend-shallow@^3.0.2:
     assign-symbols "^1.0.0"
     is-extendable "^1.0.1"
 
-extend@^3.0.0, extend@^3.0.2, extend@~3.0.2:
+extend@^3.0.2, extend@~3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa"
   integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==
@@ -7079,7 +7102,7 @@ fast-redact@^3.0.0:
   resolved "https://registry.yarnpkg.com/fast-redact/-/fast-redact-3.1.1.tgz#790fcff8f808c2e12fabbfb2be5cb2deda448fa0"
   integrity sha512-odVmjC8x8jNeMZ3C+rPMESzXVSEU8tSWSHv9HFxP2mm89G/1WwqhrerJDQm9Zus8X6aoRgQDThKqptdNA6bt+A==
 
-fast-safe-stringify@^2.0.7, fast-safe-stringify@^2.0.8:
+fast-safe-stringify@^2.0.7, fast-safe-stringify@^2.0.8, fast-safe-stringify@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz#c406a83b6e70d9e35ce3b30a81141df30aeba884"
   integrity sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==
@@ -7371,7 +7394,7 @@ form-data@4.0.0, form-data@^4.0.0:
     combined-stream "^1.0.8"
     mime-types "^2.1.12"
 
-form-data@^2.3.1, form-data@^2.5.0:
+form-data@^2.5.0:
   version "2.5.1"
   resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.5.1.tgz#f2cbec57b5e59e23716e128fe44d4e5dd23895f4"
   integrity sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==
@@ -7398,11 +7421,21 @@ form-data@~2.3.2:
     combined-stream "^1.0.6"
     mime-types "^2.1.12"
 
-formidable@^1.1.1, formidable@^1.2.0:
+formidable@^1.1.1:
   version "1.2.6"
   resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.6.tgz#d2a51d60162bbc9b4a055d8457a7c75315d1a168"
   integrity sha512-KcpbcpuLNOwrEjnbpMC0gS+X8ciDoZE1kkqzat4a8vrprf+s9pKNQ/QIwWfbfs4ltgmFl3MD177SNTkve3BwGQ==
 
+formidable@^2.0.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/formidable/-/formidable-2.1.1.tgz#81269cbea1a613240049f5f61a9d97731517414f"
+  integrity sha512-0EcS9wCFEzLvfiks7omJ+SiYJAiD+TzK4Pcw1UlUoGnhUxDcMKjt0P7x8wEb0u6OHu8Nb98WG3nxtlF5C7bvUQ==
+  dependencies:
+    dezalgo "^1.0.4"
+    hexoid "^1.0.0"
+    once "^1.4.0"
+    qs "^6.11.0"
+
 forwarded-parse@^2.1.0:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/forwarded-parse/-/forwarded-parse-2.1.2.tgz#08511eddaaa2ddfd56ba11138eee7df117a09325"
@@ -8054,6 +8087,11 @@ has@^1.0.3:
   dependencies:
     function-bind "^1.1.1"
 
+hexoid@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/hexoid/-/hexoid-1.0.0.tgz#ad10c6573fb907de23d9ec63a711267d9dc9bc18"
+  integrity sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==
+
 homedir-polyfill@^1.0.0, homedir-polyfill@^1.0.1:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz#743298cef4e5af3e194161fbadcc2151d3a058e8"
@@ -10847,7 +10885,7 @@ merge2@^1.3.0, merge2@^1.4.1:
   resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae"
   integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==
 
-methods@^1.1.1, methods@^1.1.2:
+methods@^1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee"
   integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==
@@ -10899,7 +10937,12 @@ mime-types@^2.1.12, mime-types@^2.1.18, mime-types@^2.1.24, mime-types@^2.1.27,
   dependencies:
     mime-db "1.52.0"
 
-mime@^1.3.4, mime@^1.4.1:
+mime@2.6.0:
+  version "2.6.0"
+  resolved "https://registry.yarnpkg.com/mime/-/mime-2.6.0.tgz#a2a682a95cd4d0cb1d6257e28f83da7e35800367"
+  integrity sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==
+
+mime@^1.3.4:
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"
   integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==
@@ -12689,14 +12732,14 @@ q@^1.1.2:
   resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7"
   integrity sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==
 
-qs@^6.11.0:
+qs@^6.10.3, qs@^6.11.0:
   version "6.11.0"
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a"
   integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==
   dependencies:
     side-channel "^1.0.4"
 
-qs@^6.4.0, qs@^6.5.1:
+qs@^6.4.0:
   version "6.10.5"
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.10.5.tgz#974715920a80ff6a262264acd2c7e6c2a53282b4"
   integrity sha512-O5RlPh0VFtR78y79rgcgKK4wbAI0C5zGVLztOIdpWX6ep368q5Hv6XRxDvXuZ9q3C6v+e3n8UfZZJw7IIG27eQ==
@@ -14194,29 +14237,30 @@ sublevel-pouchdb@7.2.2:
     ltgt "2.2.1"
     readable-stream "1.1.14"
 
-superagent@^3.8.3:
+superagent@^7.1.0:
-  version "3.8.3"
+  version "7.1.6"
-  resolved "https://registry.yarnpkg.com/superagent/-/superagent-3.8.3.tgz#460ea0dbdb7d5b11bc4f78deba565f86a178e128"
+  resolved "https://registry.yarnpkg.com/superagent/-/superagent-7.1.6.tgz#64f303ed4e4aba1e9da319f134107a54cacdc9c6"
-  integrity sha512-GLQtLMCoEIK4eDv6OGtkOoSMt3D+oq0y3dsxMuYuDvaNUvuT8eFBuLmfR0iYYzHC1e8hpzC6ZsxbuP6DIalMFA==
+  integrity sha512-gZkVCQR1gy/oUXr+kxJMLDjla434KmSOKbx5iGD30Ql+AkJQ/YlPKECJy2nhqOsHLjGHzoDTXNSjhnvWhzKk7g==
   dependencies:
-    component-emitter "^1.2.0"
+    component-emitter "^1.3.0"
-    cookiejar "^2.1.0"
+    cookiejar "^2.1.3"
-    debug "^3.1.0"
+    debug "^4.3.4"
-    extend "^3.0.0"
+    fast-safe-stringify "^2.1.1"
-    form-data "^2.3.1"
+    form-data "^4.0.0"
-    formidable "^1.2.0"
+    formidable "^2.0.1"
-    methods "^1.1.1"
+    methods "^1.1.2"
-    mime "^1.4.1"
+    mime "2.6.0"
-    qs "^6.5.1"
+    qs "^6.10.3"
-    readable-stream "^2.3.5"
+    readable-stream "^3.6.0"
+    semver "^7.3.7"
 
-supertest@4.0.2:
+supertest@6.2.2:
-  version "4.0.2"
+  version "6.2.2"
-  resolved "https://registry.yarnpkg.com/supertest/-/supertest-4.0.2.tgz#c2234dbdd6dc79b6f15b99c8d6577b90e4ce3f36"
+  resolved "https://registry.yarnpkg.com/supertest/-/supertest-6.2.2.tgz#04a5998fd3efaff187cb69f07a169755d655b001"
-  integrity sha512-1BAbvrOZsGA3YTCWqbmh14L0YEq0EGICX/nBnfkfVJn7SrxQV1I3pMYjSzG9y/7ZU2V9dWqyqk2POwxlb09duQ==
+  integrity sha512-wCw9WhAtKJsBvh07RaS+/By91NNE0Wh0DN19/hWPlBOU8tAfOtbZoVSV4xXeoKoxgPx0rx2y+y+8660XtE7jzg==
   dependencies:
     methods "^1.1.2"
-    superagent "^3.8.3"
+    superagent "^7.1.0"
 
 supports-color@^5.3.0, supports-color@^5.5.0:
   version "5.5.0"

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.17-alpha.5",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/types",
-  "version": "2.3.17-alpha.4",
+  "version": "2.3.17-alpha.5",
   "description": "Budibase types",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

packages/types/src/api/account/index.ts

@@ -1,2 +1,3 @@
 export * from "./user"
 export * from "./license"
+export * from "./status"

packages/types/src/api/account/status.ts

@@ -0,0 +1,7 @@
+export interface HealthStatusResponse {
+  passing: boolean
+  checks: {
+    login: boolean
+    search: boolean
+  }
+}

packages/types/src/api/web/auth.ts

@@ -0,0 +1,25 @@
+export interface LoginRequest {
+  username: string
+  password: string
+}
+
+export interface PasswordResetRequest {
+  email: string
+}
+
+export interface PasswordResetUpdateRequest {
+  resetCode: string
+  password: string
+}
+
+export interface UpdateSelfRequest {
+  firstName?: string
+  lastName?: string
+  password?: string
+  forceResetPassword?: boolean
+}
+
+export interface UpdateSelfResponse {
+  _id: string
+  _rev: string
+}