Merge branch 'develop' of github.com:Budibase/budibase into fix/budi-6723

.github/ISSUE_TEMPLATE/bug_report.md

@@ -2,10 +2,11 @@
 name: Bug report
 about: Create a report to help us improve
 title: ''
-labels: bug
+labels: bug, linear
 assignees: ''
 
 ---
+
 **Checklist**
 - [ ] I have searched budibase discussions and github issues to check if my issue already exists
 

.github/workflows/deploy-preprod.yml

@@ -22,7 +22,6 @@ jobs:
             release_version=${{ github.event.inputs.version }}
           fi
           echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
-
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
@@ -37,7 +36,6 @@ jobs:
           -o values.preprod.yaml \
           -L https://api.github.com/repos/budibase/budibase-infra/contents/kubernetes/budibase-preprod/values.yaml
           wc -l values.preprod.yaml
-
       - name: Deploy to Preprod Environment
         uses: budibase/helm@v1.8.0
         with:

charts/budibase/templates/app-service-deployment.yaml

@@ -62,16 +62,22 @@ spec:
         {{ end }}
         - name: ENABLE_ANALYTICS
           value: {{ .Values.globals.enableAnalytics | quote }}
+        - name: API_ENCRYPTION_KEY
+          value: {{ .Values.globals.apiEncryptionKey | quote }}
         - name: INTERNAL_API_KEY
           valueFrom:
             secretKeyRef:
               name: {{ template "budibase.fullname" . }}
               key: internalApiKey
+        - name: INTERNAL_API_KEY_FALLBACK
+          value: {{ .Values.globals.internalApiKeyFallback | quote }}
         - name: JWT_SECRET
           valueFrom:
             secretKeyRef:
               name: {{ template "budibase.fullname" . }}
               key: jwtSecret
+        - name: JWT_SECRET_FALLBACK
+          value: {{ .Values.globals.jwtSecretFallback | quote }}
         {{ if .Values.services.objectStore.region }}
         - name: AWS_REGION
           value: {{ .Values.services.objectStore.region }}

charts/budibase/templates/worker-service-deployment.yaml

@@ -62,16 +62,22 @@ spec:
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
+        - name: API_ENCRYPTION_KEY
+          value: {{ .Values.globals.apiEncryptionKey | quote }}
         - name: INTERNAL_API_KEY
           valueFrom:
             secretKeyRef:
               name: {{ template "budibase.fullname" . }}
               key: internalApiKey
+        - name: INTERNAL_API_KEY_FALLBACK
+          value: {{ .Values.globals.internalApiKeyFallback | quote }}
         - name: JWT_SECRET
           valueFrom:
             secretKeyRef:
               name: {{ template "budibase.fullname" . }}
               key: jwtSecret
+        - name: JWT_SECRET_FALLBACK
+          value: {{ .Values.globals.jwtSecretFallback | quote }}
         {{ if .Values.services.objectStore.region }}
         - name: AWS_REGION
           value: {{ .Values.services.objectStore.region }}

charts/budibase/values.yaml

@@ -96,9 +96,13 @@ globals:
   createSecrets: true # creates an internal API key, JWT secrets and redis password for you
 
   # if createSecrets is set to false, you can hard-code your secrets here
+  apiEncryptionKey: ""
   internalApiKey: ""
   jwtSecret: ""
   cdnUrl: ""
+  # fallback values used during live rotation
+  internalApiKeyFallback: ""
+  jwtSecretFallback: ""
 
   smtp:
     enabled: false

hosting/.env

@@ -3,6 +3,7 @@ MAIN_PORT=10000
 
 # This section contains all secrets pertaining to the system
 # These should be updated
+API_ENCRYPTION_KEY=testsecret
 JWT_SECRET=testsecret
 MINIO_ACCESS_KEY=budibase
 MINIO_SECRET_KEY=budibase

hosting/docker-compose.yaml

@@ -17,6 +17,7 @@ services:
       INTERNAL_API_KEY: ${INTERNAL_API_KEY}
       BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT}
       PORT: 4002
+      API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY}
       JWT_SECRET: ${JWT_SECRET}
       LOG_LEVEL: info
       SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
@@ -40,6 +41,7 @@ services:
       SELF_HOSTED: 1
       PORT: 4003
       CLUSTER_PORT: ${MAIN_PORT}
+      API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY}
       JWT_SECRET: ${JWT_SECRET}
       MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
       MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}

hosting/hosting.properties

@@ -3,6 +3,7 @@ MAIN_PORT=10000
 
 # This section contains all secrets pertaining to the system
 # These should be updated
+API_ENCRYPTION_KEY=testsecret
 JWT_SECRET=testsecret
 MINIO_ACCESS_KEY=budibase
 MINIO_SECRET_KEY=budibase

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.4.20",
+  "version": "2.4.26",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.4.20",
+  "version": "2.4.26",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -24,7 +24,7 @@
   "dependencies": {
     "@budibase/nano": "10.1.2",
     "@budibase/pouchdb-replication-stream": "1.2.10",
-    "@budibase/types": "^2.4.20",
+    "@budibase/types": "^2.4.26",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",

packages/backend-core/src/auth/auth.ts

@@ -1,6 +1,5 @@
 const _passport = require("koa-passport")
 const LocalStrategy = require("passport-local").Strategy
-const JwtStrategy = require("passport-jwt").Strategy
 import { getGlobalDB } from "../context"
 import { Cookie } from "../constants"
 import { getSessionsForUser, invalidateSessions } from "../security/sessions"
@@ -8,7 +7,6 @@ import {
   authenticated,
   csrf,
   google,
-  jwt as jwtPassport,
   local,
   oidc,
   tenancy,
@@ -21,14 +19,11 @@ import {
   OIDCInnerConfig,
   PlatformLogoutOpts,
   SSOProviderType,
-  User,
 } from "@budibase/types"
-import { logAlert } from "../logging"
 import * as events from "../events"
 import * as configs from "../configs"
 import { clearCookie, getCookie } from "../utils"
 import { ssoSaveUserNoOp } from "../middleware/passport/sso/sso"
-import env from "../environment"
 
 const refresh = require("passport-oauth2-refresh")
 export {
@@ -51,25 +46,6 @@ export const jwt = require("jsonwebtoken")
 
 // Strategies
 _passport.use(new LocalStrategy(local.options, local.authenticate))
-if (jwtPassport.options.secretOrKey) {
-  _passport.use(new JwtStrategy(jwtPassport.options, jwtPassport.authenticate))
-} else if (!env.DISABLE_JWT_WARNING) {
-  logAlert("No JWT Secret supplied, cannot configure JWT strategy")
-}
-
-_passport.serializeUser((user: User, done: any) => done(null, user))
-
-_passport.deserializeUser(async (user: User, done: any) => {
-  const db = getGlobalDB()
-
-  try {
-    const dbUser = await db.get(user._id)
-    return done(null, dbUser)
-  } catch (err) {
-    console.error(`User not found`, err)
-    return done(null, false, { message: "User not found" })
-  }
-})
 
 async function refreshOIDCAccessToken(
   chosenConfig: OIDCInnerConfig,

packages/backend-core/src/cache/tests/writethrough.spec.ts

@@ -1,10 +1,13 @@
-import { structures, DBTestConfiguration } from "../../../tests"
+import {
+  structures,
+  DBTestConfiguration,
+  expectFunctionWasCalledTimesWith,
+} from "../../../tests"
 import { Writethrough } from "../writethrough"
 import { getDB } from "../../db"
 import tk from "timekeeper"
 
-const START_DATE = Date.now()
+tk.freeze(Date.now())
-tk.freeze(START_DATE)
 
 const DELAY = 5000
 
@@ -17,34 +20,99 @@ describe("writethrough", () => {
   const writethrough = new Writethrough(db, DELAY)
   const writethrough2 = new Writethrough(db2, DELAY)
 
+  const docId = structures.uuid()
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+  })
+
   describe("put", () => {
-    let first: any
+    let current: any
 
     it("should be able to store, will go to DB", async () => {
       await config.doInTenant(async () => {
-        const response = await writethrough.put({ _id: "test", value: 1 })
+        const response = await writethrough.put({
+          _id: docId,
+          value: 1,
+        })
         const output = await db.get(response.id)
-        first = output
+        current = output
         expect(output.value).toBe(1)
       })
     })
 
     it("second put shouldn't update DB", async () => {
       await config.doInTenant(async () => {
-        const response = await writethrough.put({ ...first, value: 2 })
+        const response = await writethrough.put({ ...current, value: 2 })
         const output = await db.get(response.id)
-        expect(first._rev).toBe(output._rev)
+        expect(current._rev).toBe(output._rev)
         expect(output.value).toBe(1)
       })
     })
 
     it("should put it again after delay period", async () => {
       await config.doInTenant(async () => {
-        tk.freeze(START_DATE + DELAY + 1)
+        tk.freeze(Date.now() + DELAY + 1)
-        const response = await writethrough.put({ ...first, value: 3 })
+        const response = await writethrough.put({ ...current, value: 3 })
         const output = await db.get(response.id)
-        expect(response.rev).not.toBe(first._rev)
+        expect(response.rev).not.toBe(current._rev)
         expect(output.value).toBe(3)
+
+        current = output
+      })
+    })
+
+    it("should handle parallel DB updates ignoring conflicts", async () => {
+      await config.doInTenant(async () => {
+        tk.freeze(Date.now() + DELAY + 1)
+        const responses = await Promise.all([
+          writethrough.put({ ...current, value: 4 }),
+          writethrough.put({ ...current, value: 4 }),
+          writethrough.put({ ...current, value: 4 }),
+        ])
+
+        const newRev = responses.map(x => x.rev).find(x => x !== current._rev)
+        expect(newRev).toBeDefined()
+        expect(responses.map(x => x.rev)).toEqual(
+          expect.arrayContaining([current._rev, current._rev, newRev])
+        )
+        expectFunctionWasCalledTimesWith(
+          console.warn,
+          2,
+          "bb-warn: Ignoring redlock conflict in write-through cache"
+        )
+
+        const output = await db.get(current._id)
+        expect(output.value).toBe(4)
+        expect(output._rev).toBe(newRev)
+
+        current = output
+      })
+    })
+
+    it("should handle updates with documents falling behind", async () => {
+      await config.doInTenant(async () => {
+        tk.freeze(Date.now() + DELAY + 1)
+
+        const id = structures.uuid()
+        await writethrough.put({ _id: id, value: 1 })
+        const doc = await writethrough.get(id)
+
+        // Updating document
+        tk.freeze(Date.now() + DELAY + 1)
+        await writethrough.put({ ...doc, value: 2 })
+
+        // Update with the old rev value
+        tk.freeze(Date.now() + DELAY + 1)
+        const res = await writethrough.put({
+          ...doc,
+          value: 3,
+        })
+        expect(res.ok).toBe(true)
+
+        const output = await db.get(id)
+        expect(output.value).toBe(3)
+        expect(output._rev).toBe(res.rev)
       })
     })
   })
@@ -52,8 +120,8 @@ describe("writethrough", () => {
   describe("get", () => {
     it("should be able to retrieve", async () => {
       await config.doInTenant(async () => {
-        const response = await writethrough.get("test")
+        const response = await writethrough.get(docId)
-        expect(response.value).toBe(3)
+        expect(response.value).toBe(4)
       })
     })
   })

packages/backend-core/src/cache/writethrough.ts

@@ -1,7 +1,8 @@
 import BaseCache from "./base"
 import { getWritethroughClient } from "../redis/init"
 import { logWarn } from "../logging"
-import { Database } from "@budibase/types"
+import { Database, Document, LockName, LockType } from "@budibase/types"
+import * as locks from "../redis/redlockImpl"
 
 const DEFAULT_WRITE_RATE_MS = 10000
 let CACHE: BaseCache | null = null
@@ -27,20 +28,31 @@ function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem {
   return { doc, lastWrite: lastWrite || Date.now() }
 }
 
-export async function put(
+async function put(
   db: Database,
-  doc: any,
+  doc: Document,
   writeRateMs: number = DEFAULT_WRITE_RATE_MS
 ) {
   const cache = await getCache()
   const key = doc._id
-  let cacheItem: CacheItem | undefined = await cache.get(makeCacheKey(db, key))
+  let cacheItem: CacheItem | undefined
+  if (key) {
+    cacheItem = await cache.get(makeCacheKey(db, key))
+  }
   const updateDb = !cacheItem || cacheItem.lastWrite < Date.now() - writeRateMs
   let output = doc
   if (updateDb) {
+    const lockResponse = await locks.doWithLock(
+      {
+        type: LockType.TRY_ONCE,
+        name: LockName.PERSIST_WRITETHROUGH,
+        resource: key,
+        ttl: 1000,
+      },
+      async () => {
         const writeDb = async (toWrite: any) => {
           // doc should contain the _id and _rev
-      const response = await db.put(toWrite)
+          const response = await db.put(toWrite, { force: true })
           output = {
             ...doc,
             _id: response.id,
@@ -58,13 +70,20 @@ export async function put(
           }
         }
       }
+    )
+    if (!lockResponse.executed) {
+      logWarn(`Ignoring redlock conflict in write-through cache`)
+    }
+  }
   // if we are updating the DB then need to set the lastWrite to now
   cacheItem = makeCacheItem(output, updateDb ? null : cacheItem?.lastWrite)
-  await cache.store(makeCacheKey(db, key), cacheItem)
+  if (output._id) {
+    await cache.store(makeCacheKey(db, output._id), cacheItem)
+  }
   return { ok: true, id: output._id, rev: output._rev }
 }
 
-export async function get(db: Database, id: string): Promise<any> {
+async function get(db: Database, id: string): Promise<any> {
   const cache = await getCache()
   const cacheKey = makeCacheKey(db, id)
   let cacheItem: CacheItem = await cache.get(cacheKey)
@@ -76,11 +95,7 @@ export async function get(db: Database, id: string): Promise<any> {
   return cacheItem.doc
 }
 
-export async function remove(
+async function remove(db: Database, docOrId: any, rev?: any): Promise<void> {
-  db: Database,
-  docOrId: any,
-  rev?: any
-): Promise<void> {
   const cache = await getCache()
   if (!docOrId) {
     throw new Error("No ID/Rev provided.")

packages/backend-core/src/db/lucene.ts

@@ -199,6 +199,10 @@ export class QueryBuilder<T> {
     return this
   }
 
+  setAllOr() {
+    this.query.allOr = true
+  }
+
   handleSpaces(input: string) {
     if (this.noEscaping) {
       return input
@@ -236,6 +240,36 @@ export class QueryBuilder<T> {
     return value
   }
 
+  isMultiCondition() {
+    let count = 0
+    for (let filters of Object.values(this.query)) {
+      // not contains is one massive filter in allOr mode
+      if (typeof filters === "object") {
+        count += Object.keys(filters).length
+      }
+    }
+    return count > 1
+  }
+
+  compressFilters(filters: Record<string, string[]>) {
+    const compressed: typeof filters = {}
+    for (let key of Object.keys(filters)) {
+      const finalKey = removeKeyNumbering(key)
+      if (compressed[finalKey]) {
+        compressed[finalKey] = compressed[finalKey].concat(filters[key])
+      } else {
+        compressed[finalKey] = filters[key]
+      }
+    }
+    // add prefixes back
+    const final: typeof filters = {}
+    let count = 1
+    for (let [key, value] of Object.entries(compressed)) {
+      final[`${count++}:${key}`] = value
+    }
+    return final
+  }
+
   buildSearchQuery() {
     const builder = this
     let allOr = this.query && this.query.allOr
@@ -272,9 +306,9 @@ export class QueryBuilder<T> {
     }
 
     const notContains = (key: string, value: any) => {
-      // @ts-ignore
+      const allPrefix = allOr ? "*:* AND " : ""
-      const allPrefix = allOr === "" ? "*:* AND" : ""
+      const mode = allOr ? "AND" : undefined
-      return allPrefix + "NOT " + contains(key, value)
+      return allPrefix + "NOT " + contains(key, value, mode)
     }
 
     const containsAny = (key: string, value: any) => {
@@ -299,21 +333,32 @@ export class QueryBuilder<T> {
       return `${key}:(${orStatement})`
     }
 
-    function build(structure: any, queryFn: any) {
+    function build(
+      structure: any,
+      queryFn: (key: string, value: any) => string | null,
+      opts?: { returnBuilt?: boolean; mode?: string }
+    ) {
+      let built = ""
       for (let [key, value] of Object.entries(structure)) {
         // check for new format - remove numbering if needed
         key = removeKeyNumbering(key)
         key = builder.preprocess(builder.handleSpaces(key), {
           escape: true,
         })
-        const expression = queryFn(key, value)
+        let expression = queryFn(key, value)
         if (expression == null) {
           continue
         }
-        if (query.length > 0) {
+        if (built.length > 0 || query.length > 0) {
-          query += ` ${allOr ? "OR" : "AND"} `
+          const mode = opts?.mode ? opts.mode : allOr ? "OR" : "AND"
+          built += ` ${mode} `
         }
-        query += expression
+        built += expression
+      }
+      if (opts?.returnBuilt) {
+        return built
+      } else {
+        query += built
       }
     }
 
@@ -384,14 +429,14 @@ export class QueryBuilder<T> {
       build(this.query.contains, contains)
     }
     if (this.query.notContains) {
-      build(this.query.notContains, notContains)
+      build(this.compressFilters(this.query.notContains), notContains)
     }
     if (this.query.containsAny) {
       build(this.query.containsAny, containsAny)
     }
     // make sure table ID is always added as an AND
     if (tableId) {
-      query = `(${query})`
+      query = this.isMultiCondition() ? `(${query})` : query
       allOr = false
       build({ tableId }, equal)
     }

packages/backend-core/src/db/tests/lucene.spec.ts

@@ -6,9 +6,13 @@ import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene"
 const INDEX_NAME = "main"
 
 const index = `function(doc) {
-  let props = ["property", "number"]
+  let props = ["property", "number", "array"]
   for (let key of props) {
-    if (doc[key]) {
+    if (Array.isArray(doc[key])) {
+      for (let val of doc[key]) {
+        index(key, val)
+      }
+    } else if (doc[key]) {
       index(key, doc[key])
     }
   }
@@ -21,9 +25,14 @@ describe("lucene", () => {
     dbName = `db-${newid()}`
     // create the DB for testing
     db = getDB(dbName)
-    await db.put({ _id: newid(), property: "word" })
+    await db.put({ _id: newid(), property: "word", array: ["1", "4"] })
-    await db.put({ _id: newid(), property: "word2" })
+    await db.put({ _id: newid(), property: "word2", array: ["3", "1"] })
-    await db.put({ _id: newid(), property: "word3", number: 1 })
+    await db.put({
+      _id: newid(),
+      property: "word3",
+      number: 1,
+      array: ["1", "2"],
+    })
   })
 
   it("should be able to create a lucene index", async () => {
@@ -118,6 +127,15 @@ describe("lucene", () => {
       const resp = await builder.run()
       expect(resp.rows.length).toBe(2)
     })
+
+    it("should be able to perform an or not contains search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addNotContains("array", ["1"])
+      builder.addNotContains("array", ["2"])
+      builder.setAllOr()
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(2)
+    })
   })
 
   describe("paginated search", () => {

packages/backend-core/src/environment.ts

@@ -30,6 +30,12 @@ const DefaultBucketName = {
 
 const selfHosted = !!parseInt(process.env.SELF_HOSTED || "")
 
+function getAPIEncryptionKey() {
+  return process.env.API_ENCRYPTION_KEY
+    ? process.env.API_ENCRYPTION_KEY
+    : process.env.JWT_SECRET // fallback to the JWT_SECRET used historically
+}
+
 const environment = {
   isTest,
   isJest,
@@ -39,7 +45,9 @@ const environment = {
   },
   JS_BCRYPT: process.env.JS_BCRYPT,
   JWT_SECRET: process.env.JWT_SECRET,
+  JWT_SECRET_FALLBACK: process.env.JWT_SECRET_FALLBACK,
   ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
+  API_ENCRYPTION_KEY: getAPIEncryptionKey(),
   COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
   COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
   COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
@@ -55,6 +63,7 @@ const environment = {
   MINIO_URL: process.env.MINIO_URL,
   MINIO_ENABLED: process.env.MINIO_ENABLED || 1,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
+  INTERNAL_API_KEY_FALLBACK: process.env.INTERNAL_API_KEY_FALLBACK,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   ACCOUNT_PORTAL_URL:
     process.env.ACCOUNT_PORTAL_URL || "https://account.budibase.app",

packages/backend-core/src/middleware/authenticated.ts

@@ -1,5 +1,10 @@
 import { Cookie, Header } from "../constants"
-import { getCookie, clearCookie, openJwt } from "../utils"
+import {
+  getCookie,
+  clearCookie,
+  openJwt,
+  isValidInternalAPIKey,
+} from "../utils"
 import { getUser } from "../cache/user"
 import { getSession, updateSessionTTL } from "../security/sessions"
 import { buildMatcherRegex, matches } from "./matchers"
@@ -35,7 +40,9 @@ function finalise(ctx: any, opts: FinaliseOpts = {}) {
 }
 
 async function checkApiKey(apiKey: string, populateUser?: Function) {
-  if (apiKey === env.INTERNAL_API_KEY) {
+  // check both the primary and the fallback internal api keys
+  // this allows for rotation
+  if (isValidInternalAPIKey(apiKey)) {
     return { valid: true }
   }
   const decrypted = decrypt(apiKey)

packages/backend-core/src/middleware/index.ts

@@ -1,4 +1,3 @@
-export * as jwt from "./passport/jwt"
 export * as local from "./passport/local"
 export * as google from "./passport/sso/google"
 export * as oidc from "./passport/sso/oidc"

packages/backend-core/src/middleware/internalApi.ts

@@ -1,13 +1,21 @@
-import env from "../environment"
 import { Header } from "../constants"
 import { BBContext } from "@budibase/types"
+import { isValidInternalAPIKey } from "../utils"
 
 /**
  * API Key only endpoint.
  */
 export default async (ctx: BBContext, next: any) => {
   const apiKey = ctx.request.headers[Header.API_KEY]
-  if (apiKey !== env.INTERNAL_API_KEY) {
+  if (!apiKey) {
+    ctx.throw(403, "Unauthorized")
+  }
+
+  if (Array.isArray(apiKey)) {
+    ctx.throw(403, "Unauthorized")
+  }
+
+  if (!isValidInternalAPIKey(apiKey)) {
     ctx.throw(403, "Unauthorized")
   }
 

packages/backend-core/src/middleware/passport/jwt.ts

@@ -1,19 +0,0 @@
-import { Cookie } from "../../constants"
-import env from "../../environment"
-import { authError } from "./utils"
-import { BBContext } from "@budibase/types"
-
-export const options = {
-  secretOrKey: env.JWT_SECRET,
-  jwtFromRequest: function (ctx: BBContext) {
-    return ctx.cookies.get(Cookie.Auth)
-  },
-}
-
-export async function authenticate(jwt: Function, done: Function) {
-  try {
-    return done(null, jwt)
-  } catch (err) {
-    return authError(done, "JWT invalid", err)
-  }
-}

packages/backend-core/src/redis/redlockImpl.ts

@@ -24,7 +24,7 @@ const getClient = async (type: LockType): Promise<Redlock> => {
   }
 }
 
-export const OPTIONS = {
+const OPTIONS = {
   TRY_ONCE: {
     // immediately throws an error if the lock is already held
     retryCount: 0,
@@ -56,14 +56,29 @@ export const OPTIONS = {
   },
 }
 
-export const newRedlock = async (opts: Options = {}) => {
+const newRedlock = async (opts: Options = {}) => {
   let options = { ...OPTIONS.DEFAULT, ...opts }
   const redisWrapper = await getLockClient()
   const client = redisWrapper.getClient()
   return new Redlock([client], options)
 }
 
-export const doWithLock = async (opts: LockOptions, task: any) => {
+type SuccessfulRedlockExecution<T> = {
+  executed: true
+  result: T
+}
+type UnsuccessfulRedlockExecution = {
+  executed: false
+}
+
+type RedlockExecution<T> =
+  | SuccessfulRedlockExecution<T>
+  | UnsuccessfulRedlockExecution
+
+export const doWithLock = async <T>(
+  opts: LockOptions,
+  task: () => Promise<T>
+): Promise<RedlockExecution<T>> => {
   const redlock = await getClient(opts.type)
   let lock
   try {
@@ -73,8 +88,8 @@ export const doWithLock = async (opts: LockOptions, task: any) => {
     let name: string = `lock:${prefix}_${opts.name}`
 
     // add additional unique name if required
-    if (opts.nameSuffix) {
+    if (opts.resource) {
-      name = name + `_${opts.nameSuffix}`
+      name = name + `_${opts.resource}`
     }
 
     // create the lock
@@ -83,7 +98,7 @@ export const doWithLock = async (opts: LockOptions, task: any) => {
     // perform locked task
     // need to await to ensure completion before unlocking
     const result = await task()
-    return result
+    return { executed: true, result }
   } catch (e: any) {
     console.warn("lock error")
     // lock limit exceeded
@@ -92,7 +107,7 @@ export const doWithLock = async (opts: LockOptions, task: any) => {
         // don't throw for try-once locks, they will always error
         // due to retry count (0) exceeded
         console.warn(e)
-        return
+        return { executed: false }
       } else {
         console.error(e)
         throw e

packages/backend-core/src/security/encryption.ts

@@ -8,7 +8,7 @@ const RANDOM_BYTES = 16
 const STRETCH_LENGTH = 32
 
 export enum SecretOption {
-  JWT = "jwt",
+  API = "api",
   ENCRYPTION = "encryption",
 }
 
@@ -19,10 +19,10 @@ function getSecret(secretOption: SecretOption): string {
       secret = env.ENCRYPTION_KEY
       secretName = "ENCRYPTION_KEY"
       break
-    case SecretOption.JWT:
+    case SecretOption.API:
     default:
-      secret = env.JWT_SECRET
+      secret = env.API_ENCRYPTION_KEY
-      secretName = "JWT_SECRET"
+      secretName = "API_ENCRYPTION_KEY"
       break
   }
   if (!secret) {
@@ -37,7 +37,7 @@ function stretchString(string: string, salt: Buffer) {
 
 export function encrypt(
   input: string,
-  secretOption: SecretOption = SecretOption.JWT
+  secretOption: SecretOption = SecretOption.API
 ) {
   const salt = crypto.randomBytes(RANDOM_BYTES)
   const stretched = stretchString(getSecret(secretOption), salt)
@@ -50,7 +50,7 @@ export function encrypt(
 
 export function decrypt(
   input: string,
-  secretOption: SecretOption = SecretOption.JWT
+  secretOption: SecretOption = SecretOption.API
 ) {
   const [salt, encrypted] = input.split(SEPARATOR)
   const saltBuffer = Buffer.from(salt, "hex")

packages/backend-core/src/users.ts

@@ -5,6 +5,8 @@ import {
   generateAppUserID,
   queryGlobalView,
   UNICODE_MAX,
+  DocumentType,
+  SEPARATOR,
   directCouchFind,
 } from "./db"
 import { BulkDocsResponse, User } from "@budibase/types"
@@ -45,6 +47,16 @@ export const bulkGetGlobalUsersById = async (
   return users
 }
 
+export const getAllUserIds = async () => {
+  const db = getGlobalDB()
+  const startKey = `${DocumentType.USER}${SEPARATOR}`
+  const response = await db.allDocs({
+    startkey: startKey,
+    endkey: `${startKey}${UNICODE_MAX}`,
+  })
+  return response.rows.map(row => row.id)
+}
+
 export const bulkUpdateGlobalUsers = async (users: User[]) => {
   const db = getGlobalDB()
   return (await db.bulkDocs(users)) as BulkDocsResponse

packages/backend-core/src/utils/utils.ts

@@ -1,5 +1,4 @@
 import { getAllApps, queryGlobalView } from "../db"
-import { options } from "../middleware/passport/jwt"
 import {
   Header,
   MAX_VALID_DATE,
@@ -133,7 +132,30 @@ export function openJwt(token: string) {
   if (!token) {
     return token
   }
-  return jwt.verify(token, options.secretOrKey)
+  try {
+    return jwt.verify(token, env.JWT_SECRET)
+  } catch (e) {
+    if (env.JWT_SECRET_FALLBACK) {
+      // fallback to enable rotation
+      return jwt.verify(token, env.JWT_SECRET_FALLBACK)
+    } else {
+      throw e
+    }
+  }
+}
+
+export function isValidInternalAPIKey(apiKey: string) {
+  if (env.INTERNAL_API_KEY && env.INTERNAL_API_KEY === apiKey) {
+    return true
+  }
+  // fallback to enable rotation
+  if (
+    env.INTERNAL_API_KEY_FALLBACK &&
+    env.INTERNAL_API_KEY_FALLBACK === apiKey
+  ) {
+    return true
+  }
+  return false
 }
 
 /**
@@ -165,7 +187,7 @@ export function setCookie(
   opts = { sign: true }
 ) {
   if (value && opts && opts.sign) {
-    value = jwt.sign(value, options.secretOrKey)
+    value = jwt.sign(value, env.JWT_SECRET)
   }
 
   const config: SetOption = {

packages/backend-core/tests/utilities/index.ts

@@ -4,4 +4,6 @@ export { generator } from "./structures"
 export * as testEnv from "./testEnv"
 export * as testContainerUtils from "./testContainerUtils"
 
+export * from "./jestUtils"
+
 export { default as DBTestConfiguration } from "./DBTestConfiguration"

packages/backend-core/tests/utilities/jestUtils.ts

@@ -0,0 +1,9 @@
+export function expectFunctionWasCalledTimesWith(
+  jestFunction: any,
+  times: number,
+  argument: any
+) {
+  expect(
+    jestFunction.mock.calls.filter((call: any) => call[0] === argument).length
+  ).toBe(times)
+}

packages/backend-core/tests/utilities/structures/db.ts

@@ -1,5 +1,12 @@
+import { structures } from ".."
 import { newid } from "../../../src/newid"
 
 export function id() {
   return `db_${newid()}`
 }
+
+export function rev() {
+  return `${structures.generator.character({
+    numeric: true,
+  })}-${structures.uuid().replace(/-/, "")}`
+}

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.4.20",
+  "version": "2.4.26",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,8 +38,8 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/shared-core": "^2.4.20",
+    "@budibase/shared-core": "^2.4.26",
-    "@budibase/string-templates": "^2.4.20",
+    "@budibase/string-templates": "^2.4.26",
     "@spectrum-css/accordion": "3.0.24",
     "@spectrum-css/actionbutton": "1.0.1",
     "@spectrum-css/actiongroup": "1.0.1",

packages/bbui/src/ActionButton/ActionButton.svelte

@@ -113,6 +113,9 @@
   .spectrum-ActionButton--quiet {
     padding: 0 8px;
   }
+  .spectrum-ActionButton--quiet.is-selected {
+    color: var(--spectrum-global-color-gray-900);
+  }
   .is-selected:not(.emphasized) .spectrum-Icon {
     color: var(--spectrum-global-color-gray-900);
   }

packages/bbui/src/Actions/position_dropdown.js

@@ -31,6 +31,7 @@ export default function positionDropdown(element, opts) {
       styles.top = anchorBounds.top
     } else if (window.innerHeight - anchorBounds.bottom < 100) {
       styles.top = anchorBounds.top - elementBounds.height - offset
+      styles.maxHeight = 240
     } else {
       styles.top = anchorBounds.bottom + offset
       styles.maxHeight = window.innerHeight - anchorBounds.bottom - 20

packages/bbui/src/Input/CopyInput.svelte

@@ -1,5 +1,7 @@
 <script>
-  import { Input, Icon, notifications } from "@budibase/bbui"
+  import Input from "../Form/Input.svelte"
+  import Icon from "../Icon/Icon.svelte"
+  import { notifications } from "../Stores/notifications"
 
   export let label = null
   export let value

packages/bbui/src/Modal/Modal.svelte

@@ -29,6 +29,14 @@
     visible = false
   }
 
+  export function toggle() {
+    if (visible) {
+      hide()
+    } else {
+      show()
+    }
+  }
+
   export function cancel() {
     if (!visible) {
       return
@@ -61,7 +69,7 @@
     }
   }
 
-  setContext(Context.Modal, { show, hide, cancel })
+  setContext(Context.Modal, { show, hide, toggle, cancel })
 
   onMount(() => {
     document.addEventListener("keydown", handleKey)

packages/bbui/src/index.js

@@ -67,6 +67,7 @@ export { default as ColorPicker } from "./ColorPicker/ColorPicker.svelte"
 export { default as IconPicker } from "./IconPicker/IconPicker.svelte"
 export { default as InlineAlert } from "./InlineAlert/InlineAlert.svelte"
 export { default as Banner } from "./Banner/Banner.svelte"
+export { default as CopyInput } from "./Input/CopyInput.svelte"
 export { default as BannerDisplay } from "./Banner/BannerDisplay.svelte"
 export { default as MarkdownEditor } from "./Markdown/MarkdownEditor.svelte"
 export { default as MarkdownViewer } from "./Markdown/MarkdownViewer.svelte"

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.4.20",
+  "version": "2.4.26",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -58,11 +58,11 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^2.4.20",
+    "@budibase/bbui": "^2.4.26",
-    "@budibase/client": "^2.4.20",
+    "@budibase/client": "^2.4.26",
-    "@budibase/frontend-core": "^2.4.20",
+    "@budibase/frontend-core": "^2.4.26",
-    "@budibase/shared-core": "^2.4.20",
+    "@budibase/shared-core": "^2.4.26",
-    "@budibase/string-templates": "^2.4.20",
+    "@budibase/string-templates": "^2.4.26",
     "@fortawesome/fontawesome-svg-core": "^6.2.1",
     "@fortawesome/free-brands-svg-icons": "^6.2.1",
     "@fortawesome/free-solid-svg-icons": "^6.2.1",

packages/builder/src/components/automation/Shared/WebhookDisplay.svelte

@@ -1,5 +1,5 @@
 <script>
-  import CopyInput from "components/common/inputs/CopyInput.svelte"
+  import { CopyInput } from "@budibase/bbui"
 
   export let value
 

packages/builder/src/components/commandPalette/CommandPalette.svelte

@@ -0,0 +1,333 @@
+<script>
+  import {
+    Context,
+    Icon,
+    Input,
+    ModalContent,
+    Detail,
+    notifications,
+  } from "@budibase/bbui"
+  import { API } from "api"
+  import { goto } from "@roxi/routify"
+  import {
+    store,
+    sortedScreens,
+    automationStore,
+    themeStore,
+  } from "builderStore"
+  import { datasources, queries, tables, views } from "stores/backend"
+  import { getContext } from "svelte"
+  import { Constants } from "@budibase/frontend-core"
+
+  const modalContext = getContext(Context.Modal)
+  const commands = [
+    {
+      type: "Access",
+      name: "Invite users and manage app access",
+      description: "",
+      icon: "User",
+      action: () =>
+        store.update(state => ({ ...state, builderSidePanel: true })),
+    },
+    {
+      type: "Navigate",
+      name: "Portal",
+      description: "",
+      icon: "Compass",
+      action: () => $goto("../../portal"),
+    },
+    {
+      type: "Navigate",
+      name: "Data",
+      description: "",
+      icon: "Compass",
+      action: () => $goto("./data"),
+    },
+    {
+      type: "Navigate",
+      name: "Design",
+      description: "",
+      icon: "Compass",
+      action: () => $goto("./design"),
+    },
+    {
+      type: "Navigate",
+      name: "Automations",
+      description: "",
+      icon: "Compass",
+      action: () => $goto("./automate"),
+    },
+    {
+      type: "Publish",
+      name: "App",
+      description: "Deploy your application",
+      icon: "Box",
+      action: deployApp,
+    },
+    {
+      type: "Preview",
+      name: "App",
+      description: "",
+      icon: "Play",
+      action: () => window.open(`/${$store.appId}`),
+    },
+    {
+      type: "Preview",
+      name: "Published App",
+      icon: "Play",
+      action: () => window.open(`/app${$store.url}`),
+    },
+    {
+      type: "Support",
+      name: "Raise Github Discussion",
+      icon: "Help",
+      action: () =>
+        window.open(`https://github.com/Budibase/budibase/discussions/new`),
+    },
+    {
+      type: "Support",
+      name: "Raise A Bug",
+      icon: "Bug",
+      action: () =>
+        window.open(
+          `https://github.com/Budibase/budibase/issues/new?assignees=&labels=bug&template=bug_report.md&title=`
+        ),
+    },
+    ...$datasources?.list.map(datasource => ({
+      type: "Datasource",
+      name: `${datasource.name}`,
+      icon: "Data",
+      action: () => $goto(`./data/datasource/${datasource._id}`),
+    })),
+    ...$tables?.list.map(table => ({
+      type: "Table",
+      name: table.name,
+      icon: "Table",
+      action: () => $goto(`./data/table/${table._id}`),
+    })),
+    ...$views?.list.map(view => ({
+      type: "View",
+      name: view.name,
+      icon: "Remove",
+      action: () => $goto(`./data/view/${view.name}`),
+    })),
+    ...$queries?.list.map(query => ({
+      type: "Query",
+      name: query.name,
+      icon: "SQLQuery",
+      action: () => $goto(`./data/query/${query._id}`),
+    })),
+    ...$sortedScreens.map(screen => ({
+      type: "Screen",
+      name: screen.routing.route,
+      icon: "WebPage",
+      action: () => $goto(`./design/${screen._id}/components`),
+    })),
+    ...$automationStore?.automations.map(automation => ({
+      type: "Automation",
+      name: automation.name,
+      icon: "ShareAndroid",
+      action: () => $goto(`./automate/${automation._id}`),
+    })),
+    ...Constants.Themes.map(theme => ({
+      type: "Change Builder Theme",
+      name: theme.name,
+      icon: "ColorPalette",
+      action: () =>
+        themeStore.update(state => {
+          state.theme = theme.class
+          return state
+        }),
+    })),
+  ]
+
+  let search
+  let selected = null
+
+  $: enrichedCommands = commands.map(cmd => ({
+    ...cmd,
+    searchValue: `${cmd.type} ${cmd.name}`.toLowerCase(),
+  }))
+  $: results = filterResults(enrichedCommands, search)
+  $: categories = groupResults(results)
+
+  const filterResults = (commands, search) => {
+    if (!search) {
+      selected = null
+      return commands
+    }
+    selected = 0
+    search = search.toLowerCase()
+    return commands
+      .filter(cmd => cmd.searchValue.includes(search))
+      .map((cmd, idx) => ({
+        ...cmd,
+        idx,
+      }))
+  }
+
+  const groupResults = results => {
+    let categories = {}
+    results?.forEach(result => {
+      if (!categories[result.type]) {
+        categories[result.type] = []
+      }
+      categories[result.type].push(result)
+    })
+    return Object.entries(categories)
+  }
+
+  const onKeyDown = e => {
+    if (e.key === "ArrowDown") {
+      e.preventDefault()
+      if (selected === null) {
+        selected = 0
+        return
+      }
+      if (selected < results.length - 1) {
+        selected += 1
+      }
+    } else if (e.key === "ArrowUp") {
+      e.preventDefault()
+      if (selected === null) {
+        selected = results.length - 1
+        return
+      }
+      if (selected > 0) {
+        selected -= 1
+      }
+    } else if (e.key === "Enter") {
+      if (selected == null) {
+        return
+      }
+      runAction(results[selected])
+    } else if (e.key === "Escape") {
+      modalContext.hide()
+    }
+  }
+
+  async function deployApp() {
+    try {
+      await API.deployAppChanges()
+      notifications.success("Application published successfully")
+    } catch (error) {
+      notifications.error("Error publishing app")
+    }
+  }
+
+  const runAction = command => {
+    if (!command) {
+      return
+    }
+    command.action()
+    modalContext.hide()
+  }
+</script>
+
+<svelte:window on:keydown={onKeyDown} />
+<ModalContent
+  size="L"
+  showCancelButton={false}
+  showConfirmButton={false}
+  showCloseIcon={false}
+>
+  <div class="content">
+    <div class="title">
+      <Icon size="XL" name="Search" />
+      <Input bind:value={search} quiet placeholder="Search for command" />
+    </div>
+    <div class="commands">
+      {#each categories as [name, results], catIdx}
+        <div class="category">
+          <Detail>{name}</Detail>
+          <div class="options">
+            {#each results as command, cmdIdx}
+              <div
+                class="command"
+                on:click={() => runAction(command)}
+                class:selected={command.idx === selected}
+              >
+                <Icon size="M" name={command.icon} />
+                <strong>{command.type}:&nbsp;</strong>
+                <div class="name">
+                  {command.name}
+                </div>
+              </div>
+            {/each}
+          </div>
+        </div>
+      {/each}
+    </div>
+  </div>
+</ModalContent>
+
+<style>
+  .content {
+    margin: -40px;
+    overflow: hidden;
+  }
+  .title {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+    padding: var(--spacing-xl) var(--spacing-xl) var(--spacing-l)
+      var(--spacing-xl);
+    border-bottom: var(--border-dark);
+    gap: var(--spacing-m);
+    border-bottom-width: 2px;
+  }
+  .title :global(.spectrum-Textfield-input) {
+    border-bottom: none;
+    font-size: 20px;
+  }
+
+  .commands {
+    height: 378px;
+    overflow: scroll;
+  }
+
+  .category {
+    padding: var(--spacing-m) var(--spacing-xl);
+    border-bottom: var(--border-light);
+  }
+  .category:last-of-type {
+    border-bottom: none;
+  }
+  .category :global(.spectrum-Detail) {
+    color: var(--spectrum-global-color-gray-600);
+  }
+  .options {
+    padding-top: var(--spacing-m);
+    margin: 0 calc(-1 * var(--spacing-xl));
+  }
+
+  .command {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+    padding: var(--spacing-s) var(--spacing-xl);
+    cursor: pointer;
+    overflow: hidden;
+    transition: color 130ms ease-out, background-color 130ms ease-out;
+  }
+  .command:hover,
+  .selected {
+    color: var(--spectrum-global-color-gray-900);
+    background-color: var(--spectrum-global-color-gray-300);
+  }
+  .command strong {
+    margin-left: var(--spacing-m);
+  }
+  .name {
+    overflow: hidden;
+    text-overflow: ellipsis;
+    white-space: nowrap;
+  }
+
+  footer {
+    display: flex;
+    justify-content: center;
+  }
+</style>

packages/builder/src/components/deploy/DeployModal.svelte

@@ -5,12 +5,12 @@
     notifications,
     ModalContent,
     Layout,
+    ProgressCircle,
+    CopyInput,
   } from "@budibase/bbui"
   import { API } from "api"
   import analytics, { Events, EventSource } from "analytics"
   import { store } from "builderStore"
-  import { ProgressCircle } from "@budibase/bbui"
-  import CopyInput from "components/common/inputs/CopyInput.svelte"
   import TourWrap from "../portal/onboarding/TourWrap.svelte"
   import { TOUR_STEP_KEYS } from "../portal/onboarding/tours.js"
 

packages/builder/src/components/deploy/VersionModal.svelte

@@ -24,7 +24,10 @@
   let updateModal
 
   $: appId = $store.appId
-  $: updateAvailable = clientPackage.version !== $store.version
+  $: updateAvailable =
+    clientPackage.version &&
+    $store.version &&
+    clientPackage.version !== $store.version
   $: revertAvailable = $store.revertableVersion != null
 
   const refreshAppPackage = async () => {

packages/builder/src/components/design/Panel.svelte

@@ -14,10 +14,11 @@
   export let borderRight = false
 
   let wide = false
+  $: customHeaderContent = $$slots["panel-header-content"]
 </script>
 
 <div class="panel" class:wide class:borderLeft class:borderRight>
-  <div class="header">
+  <div class="header" class:custom={customHeaderContent}>
     {#if showBackButton}
       <Icon name="ArrowLeft" hoverable on:click={onClickBackButton} />
     {/if}
@@ -43,6 +44,13 @@
       <Icon name="Close" hoverable on:click={onClickCloseButton} />
     {/if}
   </div>
+
+  {#if customHeaderContent}
+    <span class="custom-content-wrap">
+      <slot name="panel-header-content" />
+    </span>
+  {/if}
+
   <div class="body">
     <slot />
   </div>
@@ -116,4 +124,10 @@
     justify-content: flex-start;
     align-items: stretch;
   }
+  .header.custom {
+    border: none;
+  }
+  .custom-content-wrap {
+    border-bottom: var(--border-light);
+  }
 </style>

packages/builder/src/components/portal/onboarding/TourPopover.svelte

@@ -15,20 +15,12 @@
   $: tourKey = $store.tourKey
   $: tourStepKey = $store.tourStepKey
 
-  const initTour = targetKey => {
+  const updateTourStep = (targetStepKey, tourKey) => {
-    if (!targetKey) {
+    if (!tourKey) {
       return
     }
-    tourSteps = [...TOURS[targetKey]]
-    tourStepIdx = 0
-    tourStep = { ...tourSteps[tourStepIdx] }
-  }
-
-  $: initTour(tourKey)
-
-  const updateTourStep = targetStepKey => {
     if (!tourSteps?.length) {
-      return
+      tourSteps = [...TOURS[tourKey]]
     }
     tourStepIdx = getCurrentStepIdx(tourSteps, targetStepKey)
     lastStep = tourStepIdx + 1 == tourSteps.length
@@ -36,7 +28,7 @@
     tourStep.onLoad()
   }
 
-  $: updateTourStep(tourStepKey)
+  $: updateTourStep(tourStepKey, tourKey)
 
   const showPopover = (tourStep, tourNodes, popover) => {
     if (!tourStep) {

packages/builder/src/components/portal/onboarding/TourWrap.svelte

@@ -8,20 +8,28 @@
 
   let currentTourStep
   let ready = false
+  let registered = false
   let handler
 
-  onMount(() => {
+  const registerTourNode = (tourKey, stepKey) => {
-    if (!$store.tourKey) return
+    if (ready && !registered && tourKey) {
-
+      currentTourStep = TOURS[tourKey].find(step => step.id === stepKey)
-    currentTourStep = TOURS[$store.tourKey].find(
+      if (!currentTourStep) {
-      step => step.id === tourStepKey
+        return
-    )
+      }
-    if (!currentTourStep) return
-
       const elem = document.querySelector(currentTourStep.query)
-    handler = tourHandler(elem, tourStepKey)
+      handler = tourHandler(elem, stepKey)
+      registered = true
+    }
+  }
+
+  $: tourKeyWatch = $store.tourKey
+  $: registerTourNode(tourKeyWatch, tourStepKey, ready)
+
+  onMount(() => {
     ready = true
   })
+
   onDestroy(() => {
     if (handler) {
       handler.destroy()

packages/builder/src/components/settings/APIKeyModal.svelte

@@ -1,9 +1,7 @@
 <script>
-  import { ModalContent } from "@budibase/bbui"
+  import { ModalContent, Body, notifications, CopyInput } from "@budibase/bbui"
-  import { Body, notifications } from "@budibase/bbui"
   import { auth } from "stores/portal"
   import { onMount } from "svelte"
-  import CopyInput from "components/common/inputs/CopyInput.svelte"
 
   let apiKey = null
 

packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte

@@ -7,6 +7,7 @@
     clickOutside,
     notifications,
     ActionButton,
+    CopyInput,
   } from "@budibase/bbui"
   import { store } from "builderStore"
   import { groups, licensing, apps, users } from "stores/portal"
@@ -17,7 +18,6 @@
   import RoleSelect from "components/common/RoleSelect.svelte"
   import { Constants, Utils } from "@budibase/frontend-core"
   import { emailValidator } from "helpers/validation"
-  import CopyInput from "components/common/inputs/CopyInput.svelte"
   import { roles } from "stores/backend"
 
   let query = null
@@ -346,8 +346,15 @@
 
   onMount(() => {
     rendered = true
+    searchFocus = true
   })
 
+  function handleKeyDown(evt) {
+    if (evt.key === "Enter" && queryIsEmail && !inviting) {
+      onInviteUser()
+    }
+  }
+
   const userTitle = user => {
     if (user.admin?.global) {
       return "Admin"
@@ -370,6 +377,8 @@
   }
 </script>
 
+<svelte:window on:keydown={handleKeyDown} />
+
 <div
   id="builder-side-panel-container"
   class:open={$store.builderSidePanel}
@@ -403,6 +412,7 @@
         autocomplete="off"
         disabled={inviting}
         value={query}
+        autofocus
         on:input={e => {
           query = e.target.value.trim()
         }}

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -10,6 +10,7 @@
     Tabs,
     Tab,
     Heading,
+    Modal,
     notifications,
   } from "@budibase/bbui"
 
@@ -18,6 +19,7 @@
   import { isActive, goto, layout, redirect } from "@roxi/routify"
   import { capitalise } from "helpers"
   import { onMount, onDestroy } from "svelte"
+  import CommandPalette from "components/commandPalette/CommandPalette.svelte"
   import TourWrap from "components/portal/onboarding/TourWrap.svelte"
   import TourPopover from "components/portal/onboarding/TourPopover.svelte"
   import BuilderSidePanel from "./_components/BuilderSidePanel.svelte"
@@ -25,12 +27,9 @@
 
   export let application
 
-  // Get Package and set store
   let promise = getPackage()
-  // let betaAccess = false
-
-  // Sync once when you load the app
   let hasSynced = false
+  let commandPaletteModal
 
   $: selected = capitalise(
     $layout.children.find(layout => $isActive(layout.path))?.title ?? "data"
@@ -50,7 +49,6 @@
       $redirect("../../")
     }
   }
-
   // Handles navigation between frontend, backend, automation.
   // This remembers your last place on each of the sections
   // e.g. if one of your screens is selected on front end, then
@@ -67,6 +65,14 @@
     })
   }
 
+  // Event handler for the command palette
+  const handleKeyDown = e => {
+    if (e.key === "k" && (e.ctrlKey || e.metaKey)) {
+      e.preventDefault()
+      commandPaletteModal.toggle()
+    }
+  }
+
   const initTour = async () => {
     // Check if onboarding is enabled.
     if (isEnabled(TENANT_FEATURE_FLAGS.ONBOARDING_TOUR)) {
@@ -120,17 +126,13 @@
   })
 </script>
 
-{#await promise}
+<TourPopover />
-  <!-- This should probably be some kind of loading state? -->
-  <div class="loading" />
-{:then _}
-  <TourPopover />
 
-  {#if $store.builderSidePanel}
+{#if $store.builderSidePanel}
   <BuilderSidePanel />
-  {/if}
+{/if}
 
-  <div class="root">
+<div class="root">
   <div class="top-nav">
     <div class="topleftnav">
       <ActionMenu>
@@ -146,8 +148,7 @@
           Overview
         </MenuItem>
         <MenuItem
-            on:click={() =>
+          on:click={() => $goto(`../../portal/overview/${application}/access`)}
-              $goto(`../../portal/overview/${application}/access`)}
         >
           Access
         </MenuItem>
@@ -158,8 +159,7 @@
           Automation history
         </MenuItem>
         <MenuItem
-            on:click={() =>
+          on:click={() => $goto(`../../portal/overview/${application}/backups`)}
-              $goto(`../../portal/overview/${application}/backups`)}
         >
           Backups
         </MenuItem>
@@ -171,13 +171,12 @@
           Name and URL
         </MenuItem>
         <MenuItem
-            on:click={() =>
+          on:click={() => $goto(`../../portal/overview/${application}/version`)}
-              $goto(`../../portal/overview/${application}/version`)}
         >
           Version
         </MenuItem>
       </ActionMenu>
-        <Heading size="XS">{$store.name || "App"}</Heading>
+      <Heading size="XS">{$store.name}</Heading>
     </div>
     <div class="topcenternav">
       <Tabs {selected} size="M">
@@ -198,11 +197,20 @@
       <AppActions {application} />
     </div>
   </div>
+  {#await promise}
+    <!-- This should probably be some kind of loading state? -->
+    <div class="loading" />
+  {:then _}
     <slot />
-  </div>
+  {:catch error}
-{:catch error}
     <p>Something went wrong: {error.message}</p>
-{/await}
+  {/await}
+</div>
+
+<svelte:window on:keydown={handleKeyDown} />
+<Modal bind:this={commandPaletteModal}>
+  <CommandPalette />
+</Modal>
 
 <style>
   .loading {

packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/_components/settings/ComponentSettingsPanel.svelte

@@ -10,6 +10,8 @@
     getBindableProperties,
     getComponentBindableProperties,
   } from "builderStore/dataBinding"
+  import { ActionButton } from "@budibase/bbui"
+  import { capitalise } from "helpers"
 
   $: componentInstance = $selectedComponent
   $: componentDefinition = store.actions.components.getDefinition(
@@ -25,11 +27,34 @@
   )
   $: isScreen = $selectedComponent?._id === $selectedScreen?.props._id
   $: title = isScreen ? "Screen" : $selectedComponent?._instanceName
+
+  let section = "settings"
+  const tabs = ["settings", "styles", "conditions"]
+
+  $: id = $selectedComponent?._id
+  $: id, (section = tabs[0])
 </script>
 
 {#if $selectedComponent}
   {#key $selectedComponent._id}
     <Panel {title} icon={componentDefinition?.icon} borderLeft>
+      <span slot="panel-header-content">
+        <div class="settings-tabs">
+          {#each tabs as tab}
+            <ActionButton
+              size="M"
+              quiet
+              selected={section === tab}
+              on:click={() => {
+                section = tab
+              }}
+            >
+              {capitalise(tab)}
+            </ActionButton>
+          {/each}
+        </div>
+      </span>
+      {#if section == "settings"}
         {#if componentDefinition?.info}
           <ComponentInfoSection {componentDefinition} />
         {/if}
@@ -40,17 +65,31 @@
           {componentBindings}
           {isScreen}
         />
+      {/if}
+      {#if section == "styles"}
         <DesignSection {componentInstance} {componentDefinition} {bindings} />
         <CustomStylesSection
           {componentInstance}
           {componentDefinition}
           {bindings}
         />
+      {/if}
+      {#if section == "conditions"}
         <ConditionalUISection
           {componentInstance}
           {componentDefinition}
           {bindings}
         />
+      {/if}
     </Panel>
   {/key}
 {/if}
+
+<style>
+  .settings-tabs {
+    display: flex;
+    gap: var(--spacing-s);
+    padding: 0 var(--spacing-l);
+    padding-bottom: var(--spacing-l);
+  }
+</style>

packages/builder/src/pages/builder/portal/_components/UpgradeButton.svelte

@@ -1,11 +1,11 @@
 <script>
   import { Button } from "@budibase/bbui"
   import { goto } from "@roxi/routify"
-  import { auth, admin } from "stores/portal"
+  import { auth, admin, licensing } from "stores/portal"
   import { isEnabled, TENANT_FEATURE_FLAGS } from "helpers/featureFlags"
 </script>
 
-{#if isEnabled(TENANT_FEATURE_FLAGS.LICENSING)}
+{#if isEnabled(TENANT_FEATURE_FLAGS.LICENSING) && !$licensing.isEnterprisePlan}
   {#if $admin.cloud && $auth?.user?.accountPortalAccess}
     <Button
       cta

packages/builder/src/stores/portal/licensing.js

@@ -12,6 +12,7 @@ export const createLicensingStore = () => {
     // the top level license
     license: undefined,
     isFreePlan: true,
+    isEnterprisePlan: true,
     // features
     groupsEnabled: false,
     backupsEnabled: false,
@@ -53,7 +54,9 @@ export const createLicensingStore = () => {
     },
     setLicense: () => {
       const license = get(auth).user.license
-      const isFreePlan = license?.plan.type === Constants.PlanType.FREE
+      const planType = license?.plan.type
+      const isEnterprisePlan = planType === Constants.PlanType.ENTERPRISE
+      const isFreePlan = planType === Constants.PlanType.FREE
       const groupsEnabled = license.features.includes(
         Constants.Features.USER_GROUPS
       )
@@ -74,6 +77,7 @@ export const createLicensingStore = () => {
         return {
           ...state,
           license,
+          isEnterprisePlan,
           isFreePlan,
           groupsEnabled,
           backupsEnabled,

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "2.4.20",
+  "version": "2.4.26",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "dist/index.js",
   "bin": {
@@ -29,9 +29,9 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "^2.4.20",
+    "@budibase/backend-core": "^2.4.26",
-    "@budibase/string-templates": "^2.4.20",
+    "@budibase/string-templates": "^2.4.26",
-    "@budibase/types": "^2.4.20",
+    "@budibase/types": "^2.4.26",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",

packages/cli/src/hosting/makeFiles.ts

@@ -13,6 +13,7 @@ export const ENV_PATH = path.resolve("./.env")
 
 function getSecrets(opts = { single: false }) {
   const secrets = [
+    "API_ENCRYPTION_KEY",
     "JWT_SECRET",
     "MINIO_ACCESS_KEY",
     "MINIO_SECRET_KEY",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "2.4.20",
+  "version": "2.4.26",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,11 +19,11 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^2.4.20",
+    "@budibase/bbui": "^2.4.26",
-    "@budibase/frontend-core": "^2.4.20",
+    "@budibase/frontend-core": "^2.4.26",
-    "@budibase/shared-core": "^2.4.20",
+    "@budibase/shared-core": "^2.4.26",
-    "@budibase/string-templates": "^2.4.20",
+    "@budibase/string-templates": "^2.4.26",
-    "@budibase/types": "^2.4.20",
+    "@budibase/types": "^2.4.26",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/client/yarn.lock

@@ -23,6 +23,11 @@
     chalk "^2.0.0"
     js-tokens "^4.0.0"
 
+"@budibase/types@2.4.8-alpha.4":
+  version "2.4.8-alpha.4"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.4.8-alpha.4.tgz#4e6dec50eef381994432ef4d08587a9a7156dd84"
+  integrity sha512-aiHHOvsDLHQ2OFmLgaSUttQwSuaPBqF1lbyyCkEJIbbl/qo9EPNZGl+AkB7wo12U5HdqWhr9OpFL12EqkcD4GA==
+
 "@jridgewell/gen-mapping@^0.3.0":
   version "0.3.2"
   resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz#c1aedc61e853f2bb9f5dfe6d4442d3b565b253b9"

packages/frontend-core/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "2.4.20",
+  "version": "2.4.26",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "^2.4.20",
+    "@budibase/bbui": "^2.4.26",
-    "@budibase/shared-core": "^2.4.20",
+    "@budibase/shared-core": "^2.4.26",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/sdk",
-  "version": "2.4.20",
+  "version": "2.4.26",
   "description": "Budibase Public API SDK",
   "author": "Budibase",
   "license": "MPL-2.0",

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.4.20",
+  "version": "2.4.26",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -43,12 +43,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "^2.4.20",
+    "@budibase/backend-core": "^2.4.26",
-    "@budibase/client": "^2.4.20",
+    "@budibase/client": "^2.4.26",
-    "@budibase/pro": "2.4.20",
+    "@budibase/pro": "2.4.26",
-    "@budibase/shared-core": "^2.4.20",
+    "@budibase/shared-core": "^2.4.26",
-    "@budibase/string-templates": "^2.4.20",
+    "@budibase/string-templates": "^2.4.26",
-    "@budibase/types": "^2.4.20",
+    "@budibase/types": "^2.4.26",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",

packages/server/specs/openapi.json

@@ -115,6 +115,15 @@
           ]
         }
       },
+      "deploymentOutput": {
+        "value": {
+          "data": {
+            "_id": "ef12381f934b4f129675cdbb76eff3c2",
+            "status": "SUCCESS",
+            "appUrl": "/app-url"
+          }
+        }
+      },
       "inputRow": {
         "value": {
           "_id": "ro_ta_5b1649e42a5b41dea4ef7742a36a7a70_e6dc7e38cf1343b2b56760265201cda4",
@@ -413,6 +422,9 @@
             }
           ]
         }
+      },
+      "metrics": {
+        "value": "# HELP budibase_os_uptime Time in seconds that the host operating system has been up.\n# TYPE budibase_os_uptime counter\nbudibase_os_uptime 54958\n# HELP budibase_os_free_mem Bytes of memory free for usage on the host operating system.\n# TYPE budibase_os_free_mem gauge\nbudibase_os_free_mem 804507648\n# HELP budibase_os_total_mem Total bytes of memory on the host operating system.\n# TYPE budibase_os_total_mem gauge\nbudibase_os_total_mem 16742404096\n# HELP budibase_os_used_mem Total bytes of memory in use on the host operating system.\n# TYPE budibase_os_used_mem gauge\nbudibase_os_used_mem 15937896448\n# HELP budibase_os_load1 Host operating system load average.\n# TYPE budibase_os_load1 gauge\nbudibase_os_load1 1.91\n# HELP budibase_os_load5 Host operating system load average.\n# TYPE budibase_os_load5 gauge\nbudibase_os_load5 1.75\n# HELP budibase_os_load15 Host operating system load average.\n# TYPE budibase_os_load15 gauge\nbudibase_os_load15 1.56\n# HELP budibase_tenant_user_count The number of users created.\n# TYPE budibase_tenant_user_count gauge\nbudibase_tenant_user_count 1\n# HELP budibase_tenant_app_count The number of apps created by a user.\n# TYPE budibase_tenant_app_count gauge\nbudibase_tenant_app_count 2\n# HELP budibase_tenant_production_app_count The number of apps a user has published.\n# TYPE budibase_tenant_production_app_count gauge\nbudibase_tenant_production_app_count 1\n# HELP budibase_tenant_dev_app_count The number of apps a user has unpublished in development.\n# TYPE budibase_tenant_dev_app_count gauge\nbudibase_tenant_dev_app_count 1\n# HELP budibase_tenant_db_count The number of couchdb databases including global tables such as _users.\n# TYPE budibase_tenant_db_count gauge\nbudibase_tenant_db_count 3\n# HELP budibase_quota_usage_apps The number of apps created.\n# TYPE budibase_quota_usage_apps gauge\nbudibase_quota_usage_apps 1\n# HELP budibase_quota_limit_apps The limit on the number of apps that can be created.\n# TYPE budibase_quota_limit_apps gauge\nbudibase_quota_limit_apps 9007199254740991\n# HELP budibase_quota_usage_rows The number of database rows used from the quota.\n# TYPE budibase_quota_usage_rows gauge\nbudibase_quota_usage_rows 0\n# HELP budibase_quota_limit_rows The limit on the number of rows that can be created.\n# TYPE budibase_quota_limit_rows gauge\nbudibase_quota_limit_rows 9007199254740991\n# HELP budibase_quota_usage_plugins The number of plugins in use.\n# TYPE budibase_quota_usage_plugins gauge\nbudibase_quota_usage_plugins 0\n# HELP budibase_quota_limit_plugins The limit on the number of plugins that can be created.\n# TYPE budibase_quota_limit_plugins gauge\nbudibase_quota_limit_plugins 9007199254740991\n# HELP budibase_quota_usage_user_groups The number of user groups created.\n# TYPE budibase_quota_usage_user_groups gauge\nbudibase_quota_usage_user_groups 0\n# HELP budibase_quota_limit_user_groups The limit on the number of user groups that can be created.\n# TYPE budibase_quota_limit_user_groups gauge\nbudibase_quota_limit_user_groups 9007199254740991\n# HELP budibase_quota_usage_queries The number of queries used in the current month.\n# TYPE budibase_quota_usage_queries gauge\nbudibase_quota_usage_queries 0\n# HELP budibase_quota_limit_queries The limit on the number of queries for the current month.\n# TYPE budibase_quota_limit_queries gauge\nbudibase_quota_limit_queries 9007199254740991\n# HELP budibase_quota_usage_automations The number of automations used in the current month.\n# TYPE budibase_quota_usage_automations gauge\nbudibase_quota_usage_automations 0\n# HELP budibase_quota_limit_automations The limit on the number of automations that can be created.\n# TYPE budibase_quota_limit_automations gauge\nbudibase_quota_limit_automations 9007199254740991\n"
       }
     },
     "securitySchemes": {
@@ -2054,6 +2066,33 @@
         }
       }
     },
+    "/metrics": {
+      "get": {
+        "operationId": "metricsGet",
+        "summary": "Retrieve Budibase tenant metrics",
+        "description": "Output metrics in OpenMetrics format compatible with Prometheus",
+        "tags": [
+          "metrics"
+        ],
+        "responses": {
+          "200": {
+            "description": "Returns tenant metrics.",
+            "content": {
+              "text/plain": {
+                "schema": {
+                  "type": "string"
+                },
+                "examples": {
+                  "metrics": {
+                    "$ref": "#/components/examples/metrics"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    },
     "/queries/{queryId}": {
       "post": {
         "operationId": "queryExecute",

packages/server/specs/openapi.yaml

@@ -85,6 +85,12 @@ components:
             updatedAt: 2022-02-22T13:00:54.035Z
             createdAt: 2022-02-11T18:02:26.961Z
             status: development
+    deploymentOutput:
+      value:
+        data:
+          _id: ef12381f934b4f129675cdbb76eff3c2
+          status: SUCCESS
+          appUrl: /app-url
     inputRow:
       value:
         _id: ro_ta_5b1649e42a5b41dea4ef7742a36a7a70_e6dc7e38cf1343b2b56760265201cda4
@@ -290,6 +296,152 @@ components:
               name: Admin
               permissionId: admin
               inherits: POWER
+    metrics:
+      value: >
+        # HELP budibase_os_uptime Time in seconds that the host operating system
+        has been up.
+
+        # TYPE budibase_os_uptime counter
+
+        budibase_os_uptime 54958
+
+        # HELP budibase_os_free_mem Bytes of memory free for usage on the host operating system.
+
+        # TYPE budibase_os_free_mem gauge
+
+        budibase_os_free_mem 804507648
+
+        # HELP budibase_os_total_mem Total bytes of memory on the host operating system.
+
+        # TYPE budibase_os_total_mem gauge
+
+        budibase_os_total_mem 16742404096
+
+        # HELP budibase_os_used_mem Total bytes of memory in use on the host operating system.
+
+        # TYPE budibase_os_used_mem gauge
+
+        budibase_os_used_mem 15937896448
+
+        # HELP budibase_os_load1 Host operating system load average.
+
+        # TYPE budibase_os_load1 gauge
+
+        budibase_os_load1 1.91
+
+        # HELP budibase_os_load5 Host operating system load average.
+
+        # TYPE budibase_os_load5 gauge
+
+        budibase_os_load5 1.75
+
+        # HELP budibase_os_load15 Host operating system load average.
+
+        # TYPE budibase_os_load15 gauge
+
+        budibase_os_load15 1.56
+
+        # HELP budibase_tenant_user_count The number of users created.
+
+        # TYPE budibase_tenant_user_count gauge
+
+        budibase_tenant_user_count 1
+
+        # HELP budibase_tenant_app_count The number of apps created by a user.
+
+        # TYPE budibase_tenant_app_count gauge
+
+        budibase_tenant_app_count 2
+
+        # HELP budibase_tenant_production_app_count The number of apps a user has published.
+
+        # TYPE budibase_tenant_production_app_count gauge
+
+        budibase_tenant_production_app_count 1
+
+        # HELP budibase_tenant_dev_app_count The number of apps a user has unpublished in development.
+
+        # TYPE budibase_tenant_dev_app_count gauge
+
+        budibase_tenant_dev_app_count 1
+
+        # HELP budibase_tenant_db_count The number of couchdb databases including global tables such as _users.
+
+        # TYPE budibase_tenant_db_count gauge
+
+        budibase_tenant_db_count 3
+
+        # HELP budibase_quota_usage_apps The number of apps created.
+
+        # TYPE budibase_quota_usage_apps gauge
+
+        budibase_quota_usage_apps 1
+
+        # HELP budibase_quota_limit_apps The limit on the number of apps that can be created.
+
+        # TYPE budibase_quota_limit_apps gauge
+
+        budibase_quota_limit_apps 9007199254740991
+
+        # HELP budibase_quota_usage_rows The number of database rows used from the quota.
+
+        # TYPE budibase_quota_usage_rows gauge
+
+        budibase_quota_usage_rows 0
+
+        # HELP budibase_quota_limit_rows The limit on the number of rows that can be created.
+
+        # TYPE budibase_quota_limit_rows gauge
+
+        budibase_quota_limit_rows 9007199254740991
+
+        # HELP budibase_quota_usage_plugins The number of plugins in use.
+
+        # TYPE budibase_quota_usage_plugins gauge
+
+        budibase_quota_usage_plugins 0
+
+        # HELP budibase_quota_limit_plugins The limit on the number of plugins that can be created.
+
+        # TYPE budibase_quota_limit_plugins gauge
+
+        budibase_quota_limit_plugins 9007199254740991
+
+        # HELP budibase_quota_usage_user_groups The number of user groups created.
+
+        # TYPE budibase_quota_usage_user_groups gauge
+
+        budibase_quota_usage_user_groups 0
+
+        # HELP budibase_quota_limit_user_groups The limit on the number of user groups that can be created.
+
+        # TYPE budibase_quota_limit_user_groups gauge
+
+        budibase_quota_limit_user_groups 9007199254740991
+
+        # HELP budibase_quota_usage_queries The number of queries used in the current month.
+
+        # TYPE budibase_quota_usage_queries gauge
+
+        budibase_quota_usage_queries 0
+
+        # HELP budibase_quota_limit_queries The limit on the number of queries for the current month.
+
+        # TYPE budibase_quota_limit_queries gauge
+
+        budibase_quota_limit_queries 9007199254740991
+
+        # HELP budibase_quota_usage_automations The number of automations used in the current month.
+
+        # TYPE budibase_quota_usage_automations gauge
+
+        budibase_quota_usage_automations 0
+
+        # HELP budibase_quota_limit_automations The limit on the number of automations that can be created.
+
+        # TYPE budibase_quota_limit_automations gauge
+
+        budibase_quota_limit_automations 9007199254740991
   securitySchemes:
     ApiKeyAuth:
       type: apiKey
@@ -1531,6 +1683,23 @@ paths:
               examples:
                 applications:
                   $ref: "#/components/examples/applications"
+  /metrics:
+    get:
+      operationId: metricsGet
+      summary: Retrieve Budibase tenant metrics
+      description: Output metrics in OpenMetrics format compatible with Prometheus
+      tags:
+        - metrics
+      responses:
+        "200":
+          description: Returns tenant metrics.
+          content:
+            text/plain:
+              schema:
+                type: string
+              examples:
+                metrics:
+                  $ref: "#/components/examples/metrics"
   "/queries/{queryId}":
     post:
       operationId: queryExecute

packages/server/specs/resources/application.ts

@@ -15,6 +15,12 @@ const application = {
   lockedBy: userResource.getExamples().user.value.user,
 }
 
+const deployment = {
+  _id: "ef12381f934b4f129675cdbb76eff3c2",
+  status: "SUCCESS",
+  appUrl: "/app-url",
+}
+
 const base = {
   name: {
     description: "The name of the app.",
@@ -108,6 +114,11 @@ export default new Resource()
         data: [application],
       },
     },
+    deploymentOutput: {
+      value: {
+        data: deployment,
+      },
+    },
   })
   .setSchemas({
     application: applicationSchema,

packages/server/specs/resources/index.ts

@@ -3,6 +3,7 @@ import row from "./row"
 import table from "./table"
 import query from "./query"
 import user from "./user"
+import metrics from "./metrics"
 import misc from "./misc"
 
 export const examples = {
@@ -12,6 +13,7 @@ export const examples = {
   ...query.getExamples(),
   ...user.getExamples(),
   ...misc.getExamples(),
+  ...metrics.getExamples(),
 }
 
 export const schemas = {

packages/server/specs/resources/metrics.ts

@@ -0,0 +1,81 @@
+import Resource from "./utils/Resource"
+
+const metricsResponse =
+  "# HELP budibase_os_uptime Time in seconds that the host operating system has been up.\n" +
+  "# TYPE budibase_os_uptime counter\n" +
+  "budibase_os_uptime 54958\n" +
+  "# HELP budibase_os_free_mem Bytes of memory free for usage on the host operating system.\n" +
+  "# TYPE budibase_os_free_mem gauge\n" +
+  "budibase_os_free_mem 804507648\n" +
+  "# HELP budibase_os_total_mem Total bytes of memory on the host operating system.\n" +
+  "# TYPE budibase_os_total_mem gauge\n" +
+  "budibase_os_total_mem 16742404096\n" +
+  "# HELP budibase_os_used_mem Total bytes of memory in use on the host operating system.\n" +
+  "# TYPE budibase_os_used_mem gauge\n" +
+  "budibase_os_used_mem 15937896448\n" +
+  "# HELP budibase_os_load1 Host operating system load average.\n" +
+  "# TYPE budibase_os_load1 gauge\n" +
+  "budibase_os_load1 1.91\n" +
+  "# HELP budibase_os_load5 Host operating system load average.\n" +
+  "# TYPE budibase_os_load5 gauge\n" +
+  "budibase_os_load5 1.75\n" +
+  "# HELP budibase_os_load15 Host operating system load average.\n" +
+  "# TYPE budibase_os_load15 gauge\n" +
+  "budibase_os_load15 1.56\n" +
+  "# HELP budibase_tenant_user_count The number of users created.\n" +
+  "# TYPE budibase_tenant_user_count gauge\n" +
+  "budibase_tenant_user_count 1\n" +
+  "# HELP budibase_tenant_app_count The number of apps created by a user.\n" +
+  "# TYPE budibase_tenant_app_count gauge\n" +
+  "budibase_tenant_app_count 2\n" +
+  "# HELP budibase_tenant_production_app_count The number of apps a user has published.\n" +
+  "# TYPE budibase_tenant_production_app_count gauge\n" +
+  "budibase_tenant_production_app_count 1\n" +
+  "# HELP budibase_tenant_dev_app_count The number of apps a user has unpublished in development.\n" +
+  "# TYPE budibase_tenant_dev_app_count gauge\n" +
+  "budibase_tenant_dev_app_count 1\n" +
+  "# HELP budibase_tenant_db_count The number of couchdb databases including global tables such as _users.\n" +
+  "# TYPE budibase_tenant_db_count gauge\n" +
+  "budibase_tenant_db_count 3\n" +
+  "# HELP budibase_quota_usage_apps The number of apps created.\n" +
+  "# TYPE budibase_quota_usage_apps gauge\n" +
+  "budibase_quota_usage_apps 1\n" +
+  "# HELP budibase_quota_limit_apps The limit on the number of apps that can be created.\n" +
+  "# TYPE budibase_quota_limit_apps gauge\n" +
+  "budibase_quota_limit_apps 9007199254740991\n" +
+  "# HELP budibase_quota_usage_rows The number of database rows used from the quota.\n" +
+  "# TYPE budibase_quota_usage_rows gauge\n" +
+  "budibase_quota_usage_rows 0\n" +
+  "# HELP budibase_quota_limit_rows The limit on the number of rows that can be created.\n" +
+  "# TYPE budibase_quota_limit_rows gauge\n" +
+  "budibase_quota_limit_rows 9007199254740991\n" +
+  "# HELP budibase_quota_usage_plugins The number of plugins in use.\n" +
+  "# TYPE budibase_quota_usage_plugins gauge\n" +
+  "budibase_quota_usage_plugins 0\n" +
+  "# HELP budibase_quota_limit_plugins The limit on the number of plugins that can be created.\n" +
+  "# TYPE budibase_quota_limit_plugins gauge\n" +
+  "budibase_quota_limit_plugins 9007199254740991\n" +
+  "# HELP budibase_quota_usage_user_groups The number of user groups created.\n" +
+  "# TYPE budibase_quota_usage_user_groups gauge\n" +
+  "budibase_quota_usage_user_groups 0\n" +
+  "# HELP budibase_quota_limit_user_groups The limit on the number of user groups that can be created.\n" +
+  "# TYPE budibase_quota_limit_user_groups gauge\n" +
+  "budibase_quota_limit_user_groups 9007199254740991\n" +
+  "# HELP budibase_quota_usage_queries The number of queries used in the current month.\n" +
+  "# TYPE budibase_quota_usage_queries gauge\n" +
+  "budibase_quota_usage_queries 0\n" +
+  "# HELP budibase_quota_limit_queries The limit on the number of queries for the current month.\n" +
+  "# TYPE budibase_quota_limit_queries gauge\n" +
+  "budibase_quota_limit_queries 9007199254740991\n" +
+  "# HELP budibase_quota_usage_automations The number of automations used in the current month.\n" +
+  "# TYPE budibase_quota_usage_automations gauge\n" +
+  "budibase_quota_usage_automations 0\n" +
+  "# HELP budibase_quota_limit_automations The limit on the number of automations that can be created.\n" +
+  "# TYPE budibase_quota_limit_automations gauge\n" +
+  "budibase_quota_limit_automations 9007199254740991\n"
+
+export default new Resource().setExamples({
+  metrics: {
+    value: metricsResponse,
+  },
+})

packages/server/src/api/controllers/public/metrics.ts

@@ -0,0 +1,251 @@
+import { Ctx } from "@budibase/types"
+import { users as userCore, db as dbCore } from "@budibase/backend-core"
+import { quotas, licensing } from "@budibase/pro"
+
+import os from "os"
+
+export async function fetch(ctx: Ctx) {
+  // *** OPERATING SYSTEM ***
+  const freeMem = os.freemem()
+  const totalMem = os.totalmem()
+  const usedMem = totalMem - freeMem
+  const uptime = os.uptime()
+
+  // *** APPS ***
+  const allDatabases = await dbCore.getAllDbs()
+  const devAppIDs = await dbCore.getDevAppIDs()
+  const prodAppIDs = await dbCore.getProdAppIDs()
+  const allAppIds = await dbCore.getAllApps({ idsOnly: true })
+
+  // *** USERS ***
+  const usersObject = await userCore.getAllUserIds()
+
+  // *** QUOTAS ***
+  const usage = await quotas.getQuotaUsage()
+  const license = await licensing.cache.getCachedLicense()
+  const appsQuotaUsage = usage.usageQuota.apps
+  const rowsQuotaUsage = usage.usageQuota.rows
+  const pluginsQuotaUsage = usage.usageQuota.plugins
+  const userGroupsQuotaUsage = usage.usageQuota.userGroups
+  const queryQuotaUsage = usage.monthly.current.queries
+  const automationsQuotaUsage = usage.monthly.current.automations
+  const appsQuotaLimit = license.quotas.usage.static.apps.value
+  const rowsQuotaLimit = license.quotas.usage.static.rows.value
+  const userGroupsQuotaLimit = license.quotas.usage.static.userGroups.value
+  const pluginsQuotaLimit = license.quotas.usage.static.plugins.value
+  const queryQuotaLimit = license.quotas.usage.monthly.queries.value
+  const automationsQuotaLimit = license.quotas.usage.monthly.automations.value
+
+  // *** BUILD THE OUTPUT STRING ***
+  var outputString = ""
+
+  // **** budibase_os_uptime ****
+  outputString += convertToOpenMetrics(
+    "budibase_os_uptime",
+    "Time in seconds that the host operating system has been up",
+    "counter",
+    uptime
+  )
+
+  // **** budibase_os_free_mem ****
+  outputString += convertToOpenMetrics(
+    "budibase_os_free_mem",
+    "Bytes of memory free for usage on the host operating system",
+    "gauge",
+    freeMem
+  )
+
+  // **** budibase_os_total_mem ****
+  outputString += convertToOpenMetrics(
+    "budibase_os_total_mem",
+    "Total bytes of memory on the host operating system",
+    "gauge",
+    totalMem
+  )
+
+  // **** budibase_os_used_mem ****
+  outputString += convertToOpenMetrics(
+    "budibase_os_used_mem",
+    "Total bytes of memory in use on the host operating system",
+    "gauge",
+    usedMem
+  )
+
+  // **** budibase_os_load1 ****
+  outputString += convertToOpenMetrics(
+    "budibase_os_load1",
+    "Host operating system load average",
+    "gauge",
+    os.loadavg()[0]
+  )
+
+  // **** budibase_os_load5 ****
+  outputString += convertToOpenMetrics(
+    "budibase_os_load5",
+    "Host operating system load average",
+    "gauge",
+    os.loadavg()[1]
+  )
+  // **** budibase_os_load15 ****
+  outputString += convertToOpenMetrics(
+    "budibase_os_load15",
+    "Host operating system load average",
+    "gauge",
+    os.loadavg()[2]
+  )
+
+  // **** budibase_tenant_user_count ****
+  outputString += convertToOpenMetrics(
+    "budibase_tenant_user_count",
+    "The number of users created",
+    "gauge",
+    usersObject.length
+  )
+
+  // **** budibase_tenant_app_count ****
+  outputString += convertToOpenMetrics(
+    "budibase_tenant_app_count",
+    "The number of apps created by a user",
+    "gauge",
+    allAppIds.length
+  )
+
+  // **** budibase_tenant_production_app_count ****
+  outputString += convertToOpenMetrics(
+    "budibase_tenant_production_app_count",
+    "The number of apps a user has published",
+    "gauge",
+    prodAppIDs.length
+  )
+
+  // **** budibase_tenant_dev_app_count ****
+  outputString += convertToOpenMetrics(
+    "budibase_tenant_dev_app_count",
+    "The number of apps a user has unpublished in development",
+    "gauge",
+    devAppIDs.length
+  )
+
+  // **** budibase_tenant_db_count ****
+  outputString += convertToOpenMetrics(
+    "budibase_tenant_db_count",
+    "The number of couchdb databases including global tables such as _users",
+    "gauge",
+    allDatabases.length
+  )
+
+  // **** budibase_quota_usage_apps ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_usage_apps",
+    "The number of apps created",
+    "gauge",
+    appsQuotaUsage
+  )
+
+  // **** budibase_quota_limit_apps ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_limit_apps",
+    "The limit on the number of apps that can be created",
+    "gauge",
+    appsQuotaLimit == -1 ? Number.MAX_SAFE_INTEGER : appsQuotaLimit
+  )
+
+  // **** budibase_quota_usage_rows ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_usage_rows",
+    "The number of database rows used from the quota",
+    "gauge",
+    rowsQuotaUsage
+  )
+
+  // **** budibase_quota_limit_rows ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_limit_rows",
+    "The limit on the number of rows that can be created",
+    "gauge",
+    rowsQuotaLimit == -1 ? Number.MAX_SAFE_INTEGER : rowsQuotaLimit
+  )
+
+  // **** budibase_quota_usage_plugins ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_usage_plugins",
+    "The number of plugins in use",
+    "gauge",
+    pluginsQuotaUsage
+  )
+
+  // **** budibase_quota_limit_plugins ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_limit_plugins",
+    "The limit on the number of plugins that can be created",
+    "gauge",
+    pluginsQuotaLimit == -1 ? Number.MAX_SAFE_INTEGER : pluginsQuotaLimit
+  )
+
+  // **** budibase_quota_usage_user_groups ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_usage_user_groups",
+    "The number of user groups created",
+    "gauge",
+    userGroupsQuotaUsage
+  )
+
+  // **** budibase_quota_limit_user_groups ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_limit_user_groups",
+    "The limit on the number of user groups that can be created",
+    "gauge",
+    userGroupsQuotaLimit == -1 ? Number.MAX_SAFE_INTEGER : userGroupsQuotaLimit
+  )
+
+  // **** budibase_quota_usage_queries ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_usage_queries",
+    "The number of queries used in the current month",
+    "gauge",
+    queryQuotaUsage
+  )
+
+  // **** budibase_quota_limit_queries ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_limit_queries",
+    "The limit on the number of queries for the current month",
+    "gauge",
+    queryQuotaLimit == -1 ? Number.MAX_SAFE_INTEGER : queryQuotaLimit
+  )
+
+  // **** budibase_quota_usage_automations ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_usage_automations",
+    "The number of automations used in the current month",
+    "gauge",
+    automationsQuotaUsage
+  )
+
+  // **** budibase_quota_limit_automations ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_limit_automations",
+    "The limit on the number of automations that can be created",
+    "gauge",
+    automationsQuotaLimit == -1
+      ? Number.MAX_SAFE_INTEGER
+      : automationsQuotaLimit
+  )
+  ctx.body = outputString
+  ctx.set("Content-Type", "text/plain")
+}
+
+export function convertToOpenMetrics(
+  metricName: string,
+  metricHelp: string,
+  metricType: string,
+  metricValue: number
+) {
+  return `# HELP ${metricName} ${metricHelp}.
+# TYPE ${metricName} ${metricType}
+${metricName} ${metricValue}\n`
+}
+
+export default {
+  fetch,
+}

packages/server/src/api/index.ts

@@ -29,13 +29,6 @@ router
       br: false,
     })
   )
-  .use(async (ctx, next) => {
-    ctx.config = {
-      jwtSecret: env.JWT_SECRET,
-      useAppRootPath: true,
-    }
-    await next()
-  })
   // re-direct before any middlewares occur
   .redirect("/", "/builder")
   .use(

packages/server/src/api/routes/public/index.ts

@@ -1,4 +1,5 @@
 import appEndpoints from "./applications"
+import metricEndpoints from "./metrics"
 import queryEndpoints from "./queries"
 import tableEndpoints from "./tables"
 import rowEndpoints from "./rows"
@@ -12,7 +13,7 @@ import env from "../../../environment"
 // below imports don't have declaration files
 const Router = require("@koa/router")
 const { RateLimit, Stores } = require("koa2-ratelimit")
-import { redis, permissions } from "@budibase/backend-core"
+import { middleware, redis, permissions } from "@budibase/backend-core"
 const { PermissionType, PermissionLevel } = permissions
 
 const PREFIX = "/api/public/v1"
@@ -91,6 +92,13 @@ function addToRouter(endpoints: any) {
   }
 }
 
+function applyAdminRoutes(endpoints: any) {
+  addMiddleware(endpoints.read, middleware.builderOrAdmin)
+  addMiddleware(endpoints.write, middleware.builderOrAdmin)
+  addToRouter(endpoints.read)
+  addToRouter(endpoints.write)
+}
+
 function applyRoutes(
   endpoints: any,
   permType: string,
@@ -119,6 +127,7 @@ function applyRoutes(
   addToRouter(endpoints.write)
 }
 
+applyAdminRoutes(metricEndpoints)
 applyRoutes(appEndpoints, PermissionType.APP, "appId")
 applyRoutes(tableEndpoints, PermissionType.TABLE, "tableId")
 applyRoutes(userEndpoints, PermissionType.USER, "userId")

packages/server/src/api/routes/public/metrics.ts

@@ -0,0 +1,28 @@
+import controller from "../../controllers/public/metrics"
+import Endpoint from "./utils/Endpoint"
+
+const read = []
+
+/**
+ * @openapi
+ * /metrics:
+ *   get:
+ *     operationId: metricsGet
+ *     summary: Retrieve Budibase tenant metrics
+ *     description: Output metrics in OpenMetrics format compatible with Prometheus
+ *     tags:
+ *       - metrics
+ *     responses:
+ *       200:
+ *         description: Returns tenant metrics.
+ *         content:
+ *           text/plain:
+ *             schema:
+ *               type: string
+ *             examples:
+ *               metrics:
+ *                 $ref: '#/components/examples/metrics'
+ */
+read.push(new Endpoint("get", "/metrics", controller.fetch))
+
+export default { read }

packages/server/src/api/routes/public/tests/metrics.spec.js

@@ -0,0 +1,34 @@
+const setup = require("../../tests/utilities")
+
+jest.setTimeout(30000)
+
+describe("/metrics", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+
+  afterAll(setup.afterAll)
+
+  // For some reason this cannot be a beforeAll or the test "should be able to update the user" fail
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  describe("get", () => {
+    it("returns a list of metrics", async () => {
+      const res = await request
+        .get(`/api/public/v1/metrics`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /text\/plain/)
+        .expect(200)
+      expect(res.text).toContain("budibase_tenant_user_count")
+    })
+
+    it("endpoint should not be publicly exposed", async () => {
+      await request
+      .get(`/api/public/v1/metrics`)
+      .set(config.publicHeaders())
+      .expect(403)
+    })
+  })
+
+})

packages/server/src/definitions/openapi.ts

@@ -22,6 +22,10 @@ export interface paths {
     /** Based on application properties (currently only name) search for applications. */
     post: operations["appSearch"];
   };
+  "/metrics": {
+    /** Output metrics in OpenMetrics format compatible with Prometheus */
+    get: operations["metricsGet"];
+  };
   "/queries/{queryId}": {
     /** Queries which have been created within a Budibase app can be executed using this, */
     post: operations["queryExecute"];
@@ -844,6 +848,17 @@ export interface operations {
       };
     };
   };
+  /** Output metrics in OpenMetrics format compatible with Prometheus */
+  metricsGet: {
+    responses: {
+      /** Returns tenant metrics. */
+      200: {
+        content: {
+          "text/plain": string;
+        };
+      };
+    };
+  };
   /** Queries which have been created within a Budibase app can be executed using this, */
   queryExecute: {
     parameters: {

packages/server/src/environment.ts

@@ -39,7 +39,6 @@ let inThread = false
 const environment = {
   // important - prefer app port to generic port
   PORT: process.env.APP_PORT || process.env.PORT,
-  JWT_SECRET: process.env.JWT_SECRET,
   COUCH_DB_URL: process.env.COUCH_DB_URL,
   MINIO_URL: process.env.MINIO_URL,
   WORKER_URL: process.env.WORKER_URL,
@@ -48,7 +47,6 @@ const environment = {
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   REDIS_URL: process.env.REDIS_URL,
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
-  INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   HTTP_MIGRATIONS: process.env.HTTP_MIGRATIONS,
   API_REQ_LIMIT_PER_SEC: process.env.API_REQ_LIMIT_PER_SEC,
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,

packages/server/src/integrations/s3.ts

@@ -107,6 +107,7 @@ const SCHEMA: Integration = {
     readCsv: {
       displayName: "Read CSV",
       type: QueryType.FIELDS,
+      readable: true,
       fields: {
         bucket: {
           type: DatasourceFieldType.STRING,

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -205,7 +205,6 @@ class TestConfiguration {
     request.appId = appId
     // fake cookies, we don't need them
     request.cookies = { set: () => {}, get: () => {} }
-    request.config = { jwtSecret: env.JWT_SECRET }
     request.user = { appId, tenantId: this.getTenantId() }
     request.query = {}
     request.request = {
@@ -332,8 +331,8 @@ class TestConfiguration {
         roleId: roleId,
         appId,
       }
-      const authToken = auth.jwt.sign(authObj, env.JWT_SECRET)
+      const authToken = auth.jwt.sign(authObj, coreEnv.JWT_SECRET)
-      const appToken = auth.jwt.sign(app, env.JWT_SECRET)
+      const appToken = auth.jwt.sign(app, coreEnv.JWT_SECRET)
 
       // returning necessary request headers
       await cache.user.invalidateUser(userId)
@@ -361,8 +360,8 @@ class TestConfiguration {
       roleId: roles.BUILTIN_ROLE_IDS.ADMIN,
       appId: this.appId,
     }
-    const authToken = auth.jwt.sign(authObj, env.JWT_SECRET)
+    const authToken = auth.jwt.sign(authObj, coreEnv.JWT_SECRET)
-    const appToken = auth.jwt.sign(app, env.JWT_SECRET)
+    const appToken = auth.jwt.sign(app, coreEnv.JWT_SECRET)
     const headers: any = {
       Accept: "application/json",
       Cookie: [

packages/server/src/utilities/workerRequests.ts

@@ -6,6 +6,7 @@ import {
   constants,
   tenancy,
   logging,
+  env as coreEnv,
 } from "@budibase/backend-core"
 import { updateAppRole } from "./global"
 import { BBContext, User } from "@budibase/types"
@@ -15,7 +16,7 @@ export function request(ctx?: BBContext, request?: any) {
     request.headers = {}
   }
   if (!ctx) {
-    request.headers[constants.Header.API_KEY] = env.INTERNAL_API_KEY
+    request.headers[constants.Header.API_KEY] = coreEnv.INTERNAL_API_KEY
     if (tenancy.isTenantIdSet()) {
       request.headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
     }

packages/server/yarn.lock

@@ -1278,14 +1278,14 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@2.4.20":
+"@budibase/backend-core@2.4.26":
-  version "2.4.20"
+  version "2.4.26"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.4.20.tgz#cd48ad052458bc2e2a5a04a91538988a56c37bc3"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.4.26.tgz#ae9679f20e86ce1706d6d549aed78a342365a4b4"
-  integrity sha512-5gxLmE1mmqgY70CA55FA7hBAyWp8tLr0gzWvkBCb7Eakbb8f1Z1gkEhG9c/XTA+6x73XuUp8RL+fWIazmpS6AQ==
+  integrity sha512-9QYJbAT9WPiOckBIR6a/CoqqbUiP9vlmc/Iy5TR5Yj2wy1JnWsf09ReTuL3CsHmh+8bCJlUHZZC4m6PUMg7+ow==
   dependencies:
     "@budibase/nano" "10.1.2"
     "@budibase/pouchdb-replication-stream" "1.2.10"
-    "@budibase/types" "^2.4.20"
+    "@budibase/types" "^2.4.26"
     "@shopify/jest-koa-mocks" "5.0.1"
     "@techpass/passport-openidconnect" "0.3.2"
     aws-cloudfront-sign "2.2.0"
@@ -1417,14 +1417,14 @@
     pouchdb-promise "^6.0.4"
     through2 "^2.0.0"
 
-"@budibase/pro@2.4.20":
+"@budibase/pro@2.4.26":
-  version "2.4.20"
+  version "2.4.26"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.4.20.tgz#b4dc91c9c38471d9655173b52ac17308db74eb55"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.4.26.tgz#37ca2b94f5dfc28ee4ff0ffa088e29112de5b66f"
-  integrity sha512-0NjnvXjSEDzYT6L76uhlmN3Ty1F3ajhnLGO0JI2UndkdiGgefYhHPZiKF5d0wUk9kwwxxw5fbV0moyG182xmYw==
+  integrity sha512-PXpsj5DFnUaSlp3AHZRZa/N4CD02HPpvVFv35/FUGkeGwGJ5AihhmzxlD54U9Q9X3Ln8miejYTFoWvEnV5Ei8w==
   dependencies:
-    "@budibase/backend-core" "2.4.20"
+    "@budibase/backend-core" "2.4.26"
     "@budibase/string-templates" "2.3.20"
-    "@budibase/types" "2.4.20"
+    "@budibase/types" "2.4.26"
     "@koa/router" "8.0.8"
     bull "4.10.1"
     joi "17.6.0"
@@ -1463,10 +1463,10 @@
     lodash "^4.17.20"
     vm2 "^3.9.4"
 
-"@budibase/types@2.4.20", "@budibase/types@^2.4.20":
+"@budibase/types@2.4.26", "@budibase/types@^2.4.26":
-  version "2.4.20"
+  version "2.4.26"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.4.20.tgz#ff35fe91936a6254c7802c79b7e57d2cff98ca91"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.4.26.tgz#c4efd9286e736feee56d623c21a9f6fd7c922b94"
-  integrity sha512-xUedzq4Hc1mQ9nhXZ7X+SU9oBHjiz5w9F6QitUmdIaVaad79tF88a9a/sLtkT/poXbdWcNBLnDg7ILwR/SMmtQ==
+  integrity sha512-q2QfDXJAopmHNq6Y25udmVJoEtnoskZEtaMy5d7/hX4jePJX3QnBd9sjgnAoOeSC3NOuXDjmvcRGtqXz6ao/Ag==
 
 "@bull-board/api@3.7.0":
   version "3.7.0"

packages/shared-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/shared-core",
-  "version": "2.4.20",
+  "version": "2.4.26",
   "description": "Shared data utils",
   "main": "dist/cjs/src/index.js",
   "types": "dist/mjs/src/index.d.ts",
@@ -20,7 +20,7 @@
     "dev:builder": "yarn prebuild && concurrently \"tsc -p tsconfig.build.json --watch\" \"tsc -p tsconfig-cjs.build.json --watch\""
   },
   "dependencies": {
-    "@budibase/types": "2.4.5-alpha.0"
+    "@budibase/types": "^2.4.26"
   },
   "devDependencies": {
     "concurrently": "^7.6.0",

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "2.4.20",
+  "version": "2.4.26",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/types",
-  "version": "2.4.20",
+  "version": "2.4.26",
   "description": "Budibase types",
   "main": "dist/cjs/index.js",
   "types": "dist/mjs/index.d.ts",

packages/types/src/sdk/locks.ts

@@ -13,6 +13,7 @@ export enum LockName {
   TRIGGER_QUOTA = "trigger_quota",
   SYNC_ACCOUNT_LICENSE = "sync_account_license",
   UPDATE_TENANTS_DOC = "update_tenants_doc",
+  PERSIST_WRITETHROUGH = "persist_writethrough",
 }
 
 export interface LockOptions {
@@ -29,9 +30,9 @@ export interface LockOptions {
    */
   ttl: number
   /**
-   * The suffix to add to the lock name for additional uniqueness
+   * The individual resource to lock. This is useful for locking around very specific identifiers, e.g. a document that is prone to conflicts
    */
-  nameSuffix?: string
+  resource?: string
   /**
    * This is a system-wide lock - don't use tenancy in lock key
    */

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "2.4.20",
+  "version": "2.4.26",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -36,10 +36,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "^2.4.20",
+    "@budibase/backend-core": "^2.4.26",
-    "@budibase/pro": "2.4.20",
+    "@budibase/pro": "2.4.26",
-    "@budibase/string-templates": "^2.4.20",
+    "@budibase/string-templates": "^2.4.26",
-    "@budibase/types": "^2.4.20",
+    "@budibase/types": "^2.4.26",
     "@koa/router": "8.0.8",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "0.3.2",

packages/worker/src/api/controllers/global/auth.ts

@@ -204,13 +204,16 @@ export const googleCallback = async (ctx: any, next: any) => {
 
   return passport.authenticate(
     strategy,
-    { successRedirect: "/", failureRedirect: "/error" },
+    {
+      successRedirect: env.PASSPORT_GOOGLEAUTH_SUCCESS_REDIRECT,
+      failureRedirect: env.PASSPORT_GOOGLEAUTH_FAILURE_REDIRECT,
+    },
     async (err: any, user: SSOUser, info: any) => {
       await passportCallback(ctx, user, err, info)
       await context.identity.doInUserContext(user, ctx, async () => {
         await events.auth.login("google-internal", user.email)
       })
-      ctx.redirect("/")
+      ctx.redirect(env.PASSPORT_GOOGLEAUTH_SUCCESS_REDIRECT)
     }
   )(ctx, next)
 }
@@ -269,13 +272,16 @@ export const oidcCallback = async (ctx: any, next: any) => {
 
   return passport.authenticate(
     strategy,
-    { successRedirect: "/", failureRedirect: "/error" },
+    {
+      successRedirect: env.PASSPORT_OIDCAUTH_SUCCESS_REDIRECT,
+      failureRedirect: env.PASSPORT_OIDCAUTH_FAILURE_REDIRECT,
+    },
     async (err: any, user: SSOUser, info: any) => {
       await passportCallback(ctx, user, err, info)
       await context.identity.doInUserContext(user, ctx, async () => {
         await events.auth.login("oidc", user.email)
       })
-      ctx.redirect("/")
+      ctx.redirect(env.PASSPORT_OIDCAUTH_SUCCESS_REDIRECT)
     }
   )(ctx, next)
 }

packages/worker/src/environment.ts

@@ -30,10 +30,8 @@ const environment = {
   // auth
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
-  JWT_SECRET: process.env.JWT_SECRET,
   SALT_ROUNDS: process.env.SALT_ROUNDS,
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
-  INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
   // urls
   MINIO_URL: process.env.MINIO_URL,
@@ -68,6 +66,15 @@ const environment = {
    * Mock the email service in use - links to ethereal hosted emails are logged instead.
    */
   ENABLE_EMAIL_TEST_MODE: process.env.ENABLE_EMAIL_TEST_MODE,
+  PASSPORT_GOOGLEAUTH_SUCCESS_REDIRECT:
+    process.env.PASSPORT_GOOGLEAUTH_SUCCESS_REDIRECT || "/",
+  PASSPORT_GOOGLEAUTH_FAILURE_REDIRECT:
+    process.env.PASSPORT_GOOGLEAUTH_FAILURE_REDIRECT || "/error",
+  PASSPORT_OIDCAUTH_SUCCESS_REDIRECT:
+    process.env.PASSPORT_OIDCAUTH_SUCCESS_REDIRECT || "/",
+  PASSPORT_OIDCAUTH_FAILURE_REDIRECT:
+    process.env.PASSPORT_OIDCAUTH_FAILURE_REDIRECT || "/error",
+
   _set(key: any, value: any) {
     process.env[key] = value
     // @ts-ignore

packages/worker/src/middleware/cloudRestricted.ts

@@ -1,5 +1,5 @@
 import env from "../environment"
-import { constants } from "@budibase/backend-core"
+import { constants, utils } from "@budibase/backend-core"
 import { BBContext } from "@budibase/types"
 
 /**
@@ -9,7 +9,15 @@ import { BBContext } from "@budibase/types"
 export default async (ctx: BBContext, next: any) => {
   if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
     const apiKey = ctx.request.headers[constants.Header.API_KEY]
-    if (apiKey !== env.INTERNAL_API_KEY) {
+    if (!apiKey) {
+      ctx.throw(403, "Unauthorized")
+    }
+
+    if (Array.isArray(apiKey)) {
+      ctx.throw(403, "Unauthorized")
+    }
+
+    if (!utils.isValidInternalAPIKey(apiKey)) {
       ctx.throw(403, "Unauthorized")
     }
   }

packages/worker/src/sdk/auth/auth.ts

@@ -5,10 +5,10 @@ import {
   sessions,
   events,
   HTTPError,
+  env as coreEnv,
 } from "@budibase/backend-core"
 import { PlatformLogoutOpts, User } from "@budibase/types"
 import jwt from "jsonwebtoken"
-import env from "../../environment"
 import * as userSdk from "../users"
 import * as emails from "../../utilities/email"
 import * as redis from "../../utilities/redis"
@@ -26,7 +26,7 @@ export async function loginUser(user: User) {
       sessionId,
       tenantId,
     },
-    env.JWT_SECRET!
+    coreEnv.JWT_SECRET!
   )
   return token
 }

packages/worker/src/tests/TestConfiguration.ts

@@ -74,7 +74,6 @@ class TestConfiguration {
     const request: any = {}
     // fake cookies, we don't need them
     request.cookies = { set: () => {}, get: () => {} }
-    request.config = { jwtSecret: env.JWT_SECRET }
     request.user = { tenantId: this.getTenantId() }
     request.query = {}
     request.request = {
@@ -180,7 +179,7 @@ class TestConfiguration {
       sessionId: "sessionid",
       tenantId: user.tenantId,
     }
-    const authCookie = auth.jwt.sign(authToken, env.JWT_SECRET)
+    const authCookie = auth.jwt.sign(authToken, coreEnv.JWT_SECRET)
     return {
       Accept: "application/json",
       ...this.cookieHeader([`${constants.Cookie.Auth}=${authCookie}`]),
@@ -197,7 +196,7 @@ class TestConfiguration {
   }
 
   internalAPIHeaders() {
-    return { [constants.Header.API_KEY]: env.INTERNAL_API_KEY }
+    return { [constants.Header.API_KEY]: coreEnv.INTERNAL_API_KEY }
   }
 
   adminOnlyResponse = () => {
@@ -277,7 +276,7 @@ class TestConfiguration {
   // CONFIGS - OIDC
 
   getOIDConfigCookie(configId: string) {
-    const token = auth.jwt.sign(configId, env.JWT_SECRET)
+    const token = auth.jwt.sign(configId, coreEnv.JWT_SECRET)
     return this.cookieHeader([[`${constants.Cookie.OIDC_CONFIG}=${token}`]])
   }
 

packages/worker/src/utilities/appService.ts

@@ -1,5 +1,10 @@
 import fetch from "node-fetch"
-import { constants, tenancy, logging } from "@budibase/backend-core"
+import {
+  constants,
+  tenancy,
+  logging,
+  env as coreEnv,
+} from "@budibase/backend-core"
 import { checkSlashesInUrl } from "../utilities"
 import env from "../environment"
 import { SyncUserRequest, User } from "@budibase/types"
@@ -9,7 +14,7 @@ async function makeAppRequest(url: string, method: string, body: any) {
     return
   }
   const request: any = { headers: {} }
-  request.headers[constants.Header.API_KEY] = env.INTERNAL_API_KEY
+  request.headers[constants.Header.API_KEY] = coreEnv.INTERNAL_API_KEY
   if (tenancy.isTenantIdSet()) {
     request.headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
   }

packages/worker/yarn.lock

@@ -475,14 +475,14 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@2.4.20":
+"@budibase/backend-core@2.4.26":
-  version "2.4.20"
+  version "2.4.26"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.4.20.tgz#cd48ad052458bc2e2a5a04a91538988a56c37bc3"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.4.26.tgz#ae9679f20e86ce1706d6d549aed78a342365a4b4"
-  integrity sha512-5gxLmE1mmqgY70CA55FA7hBAyWp8tLr0gzWvkBCb7Eakbb8f1Z1gkEhG9c/XTA+6x73XuUp8RL+fWIazmpS6AQ==
+  integrity sha512-9QYJbAT9WPiOckBIR6a/CoqqbUiP9vlmc/Iy5TR5Yj2wy1JnWsf09ReTuL3CsHmh+8bCJlUHZZC4m6PUMg7+ow==
   dependencies:
     "@budibase/nano" "10.1.2"
     "@budibase/pouchdb-replication-stream" "1.2.10"
-    "@budibase/types" "^2.4.20"
+    "@budibase/types" "^2.4.26"
     "@shopify/jest-koa-mocks" "5.0.1"
     "@techpass/passport-openidconnect" "0.3.2"
     aws-cloudfront-sign "2.2.0"
@@ -564,14 +564,14 @@
     pouchdb-promise "^6.0.4"
     through2 "^2.0.0"
 
-"@budibase/pro@2.4.20":
+"@budibase/pro@2.4.26":
-  version "2.4.20"
+  version "2.4.26"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.4.20.tgz#b4dc91c9c38471d9655173b52ac17308db74eb55"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.4.26.tgz#37ca2b94f5dfc28ee4ff0ffa088e29112de5b66f"
-  integrity sha512-0NjnvXjSEDzYT6L76uhlmN3Ty1F3ajhnLGO0JI2UndkdiGgefYhHPZiKF5d0wUk9kwwxxw5fbV0moyG182xmYw==
+  integrity sha512-PXpsj5DFnUaSlp3AHZRZa/N4CD02HPpvVFv35/FUGkeGwGJ5AihhmzxlD54U9Q9X3Ln8miejYTFoWvEnV5Ei8w==
   dependencies:
-    "@budibase/backend-core" "2.4.20"
+    "@budibase/backend-core" "2.4.26"
     "@budibase/string-templates" "2.3.20"
-    "@budibase/types" "2.4.20"
+    "@budibase/types" "2.4.26"
     "@koa/router" "8.0.8"
     bull "4.10.1"
     joi "17.6.0"
@@ -592,10 +592,10 @@
     lodash "^4.17.20"
     vm2 "^3.9.4"
 
-"@budibase/types@2.4.20", "@budibase/types@^2.4.20":
+"@budibase/types@2.4.26", "@budibase/types@^2.4.26":
-  version "2.4.20"
+  version "2.4.26"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.4.20.tgz#ff35fe91936a6254c7802c79b7e57d2cff98ca91"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.4.26.tgz#c4efd9286e736feee56d623c21a9f6fd7c922b94"
-  integrity sha512-xUedzq4Hc1mQ9nhXZ7X+SU9oBHjiz5w9F6QitUmdIaVaad79tF88a9a/sLtkT/poXbdWcNBLnDg7ILwR/SMmtQ==
+  integrity sha512-q2QfDXJAopmHNq6Y25udmVJoEtnoskZEtaMy5d7/hX4jePJX3QnBd9sjgnAoOeSC3NOuXDjmvcRGtqXz6ao/Ag==
 
 "@cspotcode/source-map-support@^0.8.0":
   version "0.8.1"

qa-core/yarn.lock

@@ -716,9 +716,9 @@
     "@hapi/hoek" "^9.0.0"
 
 "@sideway/formula@^3.0.0":
-  version "3.0.0"
+  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/@sideway/formula/-/formula-3.0.0.tgz#fe158aee32e6bd5de85044be615bc08478a0a13c"
+  resolved "https://registry.yarnpkg.com/@sideway/formula/-/formula-3.0.1.tgz#80fcbcbaf7ce031e0ef2dd29b1bfc7c3f583611f"
-  integrity sha512-vHe7wZ4NOXVfkoRb8T5otiENVlT7a3IAiw7H5M2+GO+9CDgcVUUsX1zalAztCmwyOr2RUTGJdgB+ZvSVqmdHmg==
+  integrity sha512-/poHZJJVjx3L+zVD6g9KgHfYnb443oi7wLu/XKojDviHy6HOEOA6z1Trk5aR1dGcmPenJEgb2sK2I80LeS3MIg==
 
 "@sideway/pinpoint@^2.0.0":
   version "2.0.0"