Merge branch 'develop' into api-tets-public-api-key-generation

.github/workflows/budibase_ci.yml

@@ -10,7 +10,7 @@ on:
  pull_request:
     branches:
       - master
-      - develop 
+      - develop
  workflow_dispatch:
 
 env:
@@ -64,6 +64,20 @@ jobs:
           name: codecov-umbrella
           verbose: true
 
+  test-pro:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - name: Install Pro
+        run: yarn install:pro $BRANCH $BASE_BRANCH
+      - run: yarn
+      - run: yarn bootstrap
+      - run: yarn test:pro
+
   integration-test:
     runs-on: ubuntu-latest
     services:

.github/workflows/release-develop.yml

@@ -68,83 +68,6 @@ jobs:
           DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
 
-  deploy-to-release-env:
-    needs: [release-images]
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Get the current budibase release version
-        id: version
-        run: |
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
-
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: eu-west-1
-
-      - name: Pull values.yaml from budibase-infra
-        run: | 
-          curl -H "Authorization: token ${{ secrets.GH_PERSONAL_TOKEN }}" \
-          -H 'Accept: application/vnd.github.v3.raw' \
-          -o values.release.yaml \
-          -L https://api.github.com/repos/budibase/budibase-infra/contents/kubernetes/budibase-release/values.yaml
-          wc -l values.release.yaml
-
-      - name: Deploy to Release Environment
-        uses: glopezep/helm@v1.7.1
-        with:
-          release: budibase-release
-          namespace: budibase
-          chart: charts/budibase
-          token: ${{ github.token }}
-          helm: helm3
-          values: |
-            globals: 
-              appVersion: develop
-            ingress:
-              enabled: true
-              nginx: true
-          value-files: >-
-            [
-              "values.release.yaml"
-            ]
-        env:
-          KUBECONFIG_FILE: '${{ secrets.RELEASE_KUBECONFIG }}'
-      
-      - name: Re roll app-service
-        uses: actions-hub/kubectl@master
-        env:
-          KUBE_CONFIG: ${{ secrets.RELEASE_KUBECONFIG_BASE64 }}
-        with:
-          args: rollout restart deployment app-service -n budibase
-
-      - name: Re roll proxy-service
-        uses: actions-hub/kubectl@master
-        env:
-          KUBE_CONFIG: ${{ secrets.RELEASE_KUBECONFIG_BASE64 }}
-        with:
-          args: rollout restart deployment proxy-service -n budibase
-
-      - name: Re roll worker-service
-        uses: actions-hub/kubectl@master
-        env:
-          KUBE_CONFIG: ${{ secrets.RELEASE_KUBECONFIG_BASE64 }}
-        with:
-          args: rollout restart deployment worker-service -n budibase
-
-      - name: Discord Webhook Action
-        uses: tsickert/discord-webhook@v4.0.0
-        with:
-          webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
-          content: "Release Env Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Budibase Release Env."
-          embed-title: ${{ env.RELEASE_VERSION }}
-
   release-helm-chart:
     needs: [release-images]
     runs-on: ubuntu-latest

.tool-versions

@@ -1,2 +1,2 @@
 nodejs 14.19.3
-python 3.11.1
+python 3.10.0

hosting/docker-compose.test.yaml

@@ -8,8 +8,8 @@ services:
     # Last version that supports the "fs" backend
     image: minio/minio:RELEASE.2022-10-24T18-35-07Z
     ports:
-      - 9000
-      - 9001
+      - "9000"
+      - "9001"
     environment:
       MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
       MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
@@ -28,9 +28,9 @@ services:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}
     ports:
-      - 5984
-      - 4369
-      - 9100
+      - "5984"
+      - "4369"
+      - "9100"
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:5984/_up"]
       interval: 30s
@@ -42,6 +42,6 @@ services:
     image: redis
     command: redis-server --requirepass ${REDIS_PASSWORD}
     ports:
-      - 6379
+      - "6379"
     healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
+      test: ["CMD", "redis-cli", "ping"]

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.3.18-alpha.7",
+  "version": "2.3.18-alpha.29",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -13,7 +13,7 @@
     "js-yaml": "^4.1.0",
     "kill-port": "^1.6.1",
     "lerna": "3.14.1",
-    "madge": "^5.0.1",
+    "madge": "^6.0.0",
     "prettier": "^2.3.1",
     "prettier-plugin-svelte": "^2.3.0",
     "rimraf": "^3.0.2",
@@ -44,7 +44,7 @@
     "dev": "yarn run kill-all && lerna link && lerna run --parallel dev:builder --concurrency 1",
     "dev:noserver": "yarn run kill-builder && lerna link && lerna run dev:stack:up && lerna run --parallel dev:builder --concurrency 1 --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
     "dev:server": "yarn run kill-server && lerna run --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server",
-    "test": "lerna run test && yarn test:pro",
+    "test": "lerna run test",
     "test:pro": "bash scripts/pro/test.sh",
     "lint:eslint": "eslint packages && eslint qa-core",
     "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"",
@@ -84,4 +84,4 @@
     "install:pro": "bash scripts/pro/install.sh",
     "dep:clean": "yarn clean && yarn bootstrap"
   }
-}
+}

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.3.18-alpha.7",
+  "version": "2.3.18-alpha.29",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -24,7 +24,7 @@
   "dependencies": {
     "@budibase/nano": "10.1.1",
     "@budibase/pouchdb-replication-stream": "1.2.10",
-    "@budibase/types": "2.3.18-alpha.7",
+    "@budibase/types": "2.3.18-alpha.29",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",

packages/backend-core/src/auth/auth.ts

@@ -2,25 +2,34 @@ const _passport = require("koa-passport")
 const LocalStrategy = require("passport-local").Strategy
 const JwtStrategy = require("passport-jwt").Strategy
 import { getGlobalDB } from "../context"
-const refresh = require("passport-oauth2-refresh")
-import { Config, Cookie } from "../constants"
-import { getScopedConfig } from "../db"
+import { Cookie } from "../constants"
 import { getSessionsForUser, invalidateSessions } from "../security/sessions"
 import {
+  authenticated,
+  csrf,
+  google,
   jwt as jwtPassport,
   local,
-  authenticated,
-  tenancy,
-  csrf,
   oidc,
-  google,
+  tenancy,
 } from "../middleware"
+import * as userCache from "../cache/user"
 import { invalidateUser } from "../cache/user"
-import { PlatformLogoutOpts, User } from "@budibase/types"
+import {
+  ConfigType,
+  GoogleInnerConfig,
+  OIDCInnerConfig,
+  PlatformLogoutOpts,
+  SSOProviderType,
+  User,
+} from "@budibase/types"
 import { logAlert } from "../logging"
 import * as events from "../events"
-import * as userCache from "../cache/user"
+import * as configs from "../configs"
 import { clearCookie, getCookie } from "../utils"
+import { ssoSaveUserNoOp } from "../middleware/passport/sso/sso"
+
+const refresh = require("passport-oauth2-refresh")
 export {
   auditLog,
   authError,
@@ -33,7 +42,6 @@ export {
   google,
   oidc,
 } from "../middleware"
-import { ssoSaveUserNoOp } from "../middleware/passport/sso/sso"
 export const buildAuthMiddleware = authenticated
 export const buildTenancyMiddleware = tenancy
 export const buildCsrfMiddleware = csrf
@@ -63,11 +71,10 @@ _passport.deserializeUser(async (user: User, done: any) => {
 })
 
 async function refreshOIDCAccessToken(
-  db: any,
-  chosenConfig: any,
+  chosenConfig: OIDCInnerConfig,
   refreshToken: string
-) {
-  const callbackUrl = await oidc.getCallbackUrl(db, chosenConfig)
+): Promise<RefreshResponse> {
+  const callbackUrl = await oidc.getCallbackUrl()
   let enrichedConfig: any
   let strategy: any
 
@@ -90,7 +97,7 @@ async function refreshOIDCAccessToken(
 
   return new Promise(resolve => {
     refresh.requestNewAccessToken(
-      Config.OIDC,
+      ConfigType.OIDC,
       refreshToken,
       (err: any, accessToken: string, refreshToken: any, params: any) => {
         resolve({ err, accessToken, refreshToken, params })
@@ -100,11 +107,10 @@ async function refreshOIDCAccessToken(
 }
 
 async function refreshGoogleAccessToken(
-  db: any,
-  config: any,
+  config: GoogleInnerConfig,
   refreshToken: any
-) {
-  let callbackUrl = await google.getCallbackUrl(db, config)
+): Promise<RefreshResponse> {
+  let callbackUrl = await google.getCallbackUrl(config)
 
   let strategy
   try {
@@ -124,7 +130,7 @@ async function refreshGoogleAccessToken(
 
   return new Promise(resolve => {
     refresh.requestNewAccessToken(
-      Config.GOOGLE,
+      ConfigType.GOOGLE,
       refreshToken,
       (err: any, accessToken: string, refreshToken: string, params: any) => {
         resolve({ err, accessToken, refreshToken, params })
@@ -133,41 +139,37 @@ async function refreshGoogleAccessToken(
   })
 }
 
+interface RefreshResponse {
+  err?: {
+    data?: string
+  }
+  accessToken?: string
+  refreshToken?: string
+  params?: any
+}
+
 export async function refreshOAuthToken(
   refreshToken: string,
-  configType: string,
-  configId: string
-) {
-  const db = getGlobalDB()
-
-  const config = await getScopedConfig(db, {
-    type: configType,
-    group: {},
-  })
-
-  let chosenConfig = {}
-  let refreshResponse
-  if (configType === Config.OIDC) {
-    // configId - retrieved from cookie.
-    chosenConfig = config.configs.filter((c: any) => c.uuid === configId)[0]
-    if (!chosenConfig) {
-      throw new Error("Invalid OIDC configuration")
-    }
-    refreshResponse = await refreshOIDCAccessToken(
-      db,
-      chosenConfig,
-      refreshToken
-    )
-  } else {
-    chosenConfig = config
-    refreshResponse = await refreshGoogleAccessToken(
-      db,
-      chosenConfig,
-      refreshToken
-    )
+  providerType: SSOProviderType,
+  configId?: string
+): Promise<RefreshResponse> {
+  switch (providerType) {
+    case SSOProviderType.OIDC:
+      if (!configId) {
+        return { err: { data: "OIDC config id not provided" } }
+      }
+      const oidcConfig = await configs.getOIDCConfigById(configId)
+      if (!oidcConfig) {
+        return { err: { data: "OIDC configuration not found" } }
+      }
+      return refreshOIDCAccessToken(oidcConfig, refreshToken)
+    case SSOProviderType.GOOGLE:
+      let googleConfig = await configs.getGoogleConfig()
+      if (!googleConfig) {
+        return { err: { data: "Google configuration not found" } }
+      }
+      return refreshGoogleAccessToken(googleConfig, refreshToken)
   }
-
-  return refreshResponse
 }
 
 // TODO: Refactor to use user save function instead to prevent the need for
@@ -225,6 +227,6 @@ export async function platformLogout(opts: PlatformLogoutOpts) {
 
   const sessionIds = sessions.map(({ sessionId }) => sessionId)
   await invalidateSessions(userId, { sessionIds, reason: "logout" })
-  await events.auth.logout()
+  await events.auth.logout(ctx.user?.email)
   await userCache.invalidateUser(userId)
 }

packages/backend-core/src/cache/appMetadata.ts

@@ -1,6 +1,6 @@
 import { getAppClient } from "../redis/init"
 import { doWithDB, DocumentType } from "../db"
-import { Database } from "@budibase/types"
+import { Database, App } from "@budibase/types"
 
 const AppState = {
   INVALID: "invalid",
@@ -65,7 +65,7 @@ export async function getAppMetadata(appId: string) {
   if (isInvalid(metadata)) {
     throw { status: 404, message: "No app metadata found" }
   }
-  return metadata
+  return metadata as App
 }
 
 /**

packages/backend-core/src/configs/configs.ts

@@ -0,0 +1,242 @@
+import {
+  Config,
+  ConfigType,
+  GoogleConfig,
+  GoogleInnerConfig,
+  OIDCConfig,
+  OIDCInnerConfig,
+  SettingsConfig,
+  SettingsInnerConfig,
+  SMTPConfig,
+  SMTPInnerConfig,
+} from "@budibase/types"
+import { DocumentType, SEPARATOR } from "../constants"
+import { CacheKey, TTL, withCache } from "../cache"
+import * as context from "../context"
+import env from "../environment"
+import environment from "../environment"
+
+// UTILS
+
+/**
+ * Generates a new configuration ID.
+ * @returns {string} The new configuration ID which the config doc can be stored under.
+ */
+export function generateConfigID(type: ConfigType) {
+  return `${DocumentType.CONFIG}${SEPARATOR}${type}`
+}
+
+export async function getConfig<T extends Config>(
+  type: ConfigType
+): Promise<T | undefined> {
+  const db = context.getGlobalDB()
+  try {
+    // await to catch error
+    const config = (await db.get(generateConfigID(type))) as T
+    return config
+  } catch (e: any) {
+    if (e.status === 404) {
+      return
+    }
+    throw e
+  }
+}
+
+export async function save(config: Config) {
+  const db = context.getGlobalDB()
+  return db.put(config)
+}
+
+// SETTINGS
+
+export async function getSettingsConfigDoc(): Promise<SettingsConfig> {
+  let config = await getConfig<SettingsConfig>(ConfigType.SETTINGS)
+
+  if (!config) {
+    config = {
+      _id: generateConfigID(ConfigType.SETTINGS),
+      type: ConfigType.SETTINGS,
+      config: {},
+    }
+  }
+
+  // overridden fields
+  config.config.platformUrl = await getPlatformUrl({
+    tenantAware: true,
+    config: config.config,
+  })
+  config.config.analyticsEnabled = await analyticsEnabled({
+    config: config.config,
+  })
+
+  return config
+}
+
+export async function getSettingsConfig(): Promise<SettingsInnerConfig> {
+  return (await getSettingsConfigDoc()).config
+}
+
+export async function getPlatformUrl(
+  opts: { tenantAware: boolean; config?: SettingsInnerConfig } = {
+    tenantAware: true,
+  }
+) {
+  let platformUrl = env.PLATFORM_URL || "http://localhost:10000"
+
+  if (!env.SELF_HOSTED && env.MULTI_TENANCY && opts.tenantAware) {
+    // cloud and multi tenant - add the tenant to the default platform url
+    const tenantId = context.getTenantId()
+    if (!platformUrl.includes("localhost:")) {
+      platformUrl = platformUrl.replace("://", `://${tenantId}.`)
+    }
+  } else if (env.SELF_HOSTED) {
+    const config = opts?.config
+      ? opts.config
+      : // direct to db to prevent infinite loop
+        (await getConfig<SettingsConfig>(ConfigType.SETTINGS))?.config
+    if (config?.platformUrl) {
+      platformUrl = config.platformUrl
+    }
+  }
+
+  return platformUrl
+}
+
+export const analyticsEnabled = async (opts?: {
+  config?: SettingsInnerConfig
+}) => {
+  // cloud - always use the environment variable
+  if (!env.SELF_HOSTED) {
+    return !!env.ENABLE_ANALYTICS
+  }
+
+  // self host - prefer the settings doc
+  // use cache as events have high throughput
+  const enabledInDB = await withCache(
+    CacheKey.ANALYTICS_ENABLED,
+    TTL.ONE_DAY,
+    async () => {
+      const config = opts?.config
+        ? opts.config
+        : // direct to db to prevent infinite loop
+          (await getConfig<SettingsConfig>(ConfigType.SETTINGS))?.config
+
+      // need to do explicit checks in case the field is not set
+      if (config?.analyticsEnabled === false) {
+        return false
+      } else if (config?.analyticsEnabled === true) {
+        return true
+      }
+    }
+  )
+
+  if (enabledInDB !== undefined) {
+    return enabledInDB
+  }
+
+  // fallback to the environment variable
+  // explicitly check for 0 or false here, undefined or otherwise is treated as true
+  const envEnabled: any = env.ENABLE_ANALYTICS
+  if (envEnabled === 0 || envEnabled === false) {
+    return false
+  } else {
+    return true
+  }
+}
+
+// GOOGLE
+
+async function getGoogleConfigDoc(): Promise<GoogleConfig | undefined> {
+  return await getConfig<GoogleConfig>(ConfigType.GOOGLE)
+}
+
+export async function getGoogleConfig(): Promise<
+  GoogleInnerConfig | undefined
+> {
+  const config = await getGoogleConfigDoc()
+  return config?.config
+}
+
+export async function getGoogleDatasourceConfig(): Promise<
+  GoogleInnerConfig | undefined
+> {
+  if (!env.SELF_HOSTED) {
+    // always use the env vars in cloud
+    return getDefaultGoogleConfig()
+  }
+
+  // prefer the config in self-host
+  let config = await getGoogleConfig()
+
+  // fallback to env vars
+  if (!config || !config.activated) {
+    config = getDefaultGoogleConfig()
+  }
+
+  return config
+}
+
+export function getDefaultGoogleConfig(): GoogleInnerConfig | undefined {
+  if (environment.GOOGLE_CLIENT_ID && environment.GOOGLE_CLIENT_SECRET) {
+    return {
+      clientID: environment.GOOGLE_CLIENT_ID!,
+      clientSecret: environment.GOOGLE_CLIENT_SECRET!,
+      activated: true,
+    }
+  }
+}
+
+// OIDC
+
+async function getOIDCConfigDoc(): Promise<OIDCConfig | undefined> {
+  return getConfig<OIDCConfig>(ConfigType.OIDC)
+}
+
+export async function getOIDCConfig(): Promise<OIDCInnerConfig | undefined> {
+  const config = (await getOIDCConfigDoc())?.config
+  // default to the 0th config
+  return config?.configs && config.configs[0]
+}
+
+/**
+ * @param configId The config id of the inner config to retrieve
+ */
+export async function getOIDCConfigById(
+  configId: string
+): Promise<OIDCInnerConfig | undefined> {
+  const config = (await getConfig<OIDCConfig>(ConfigType.OIDC))?.config
+  return config && config.configs.filter((c: any) => c.uuid === configId)[0]
+}
+
+// SMTP
+
+export async function getSMTPConfigDoc(): Promise<SMTPConfig | undefined> {
+  return getConfig<SMTPConfig>(ConfigType.SMTP)
+}
+
+export async function getSMTPConfig(
+  isAutomation?: boolean
+): Promise<SMTPInnerConfig | undefined> {
+  const config = await getSMTPConfigDoc()
+  if (config) {
+    return config.config
+  }
+
+  // always allow fallback in self host
+  // in cloud don't allow for automations
+  const allowFallback = env.SELF_HOSTED || !isAutomation
+
+  // Use an SMTP fallback configuration from env variables
+  if (env.SMTP_FALLBACK_ENABLED && allowFallback) {
+    return {
+      port: env.SMTP_PORT,
+      host: env.SMTP_HOST!,
+      secure: false,
+      from: env.SMTP_FROM_ADDRESS!,
+      auth: {
+        user: env.SMTP_USER!,
+        pass: env.SMTP_PASSWORD!,
+      },
+    }
+  }
+}

packages/backend-core/src/configs/index.ts

@@ -0,0 +1 @@
+export * from "./configs"

packages/backend-core/src/configs/tests/configs.spec.ts

@@ -0,0 +1,116 @@
+import { DBTestConfiguration, generator, testEnv } from "../../../tests"
+import { ConfigType } from "@budibase/types"
+import env from "../../environment"
+import * as configs from "../configs"
+
+const DEFAULT_URL = "http://localhost:10000"
+const ENV_URL = "http://env.com"
+
+describe("configs", () => {
+  const config = new DBTestConfiguration()
+
+  const setDbPlatformUrl = async (dbUrl: string) => {
+    const settingsConfig = {
+      _id: configs.generateConfigID(ConfigType.SETTINGS),
+      type: ConfigType.SETTINGS,
+      config: {
+        platformUrl: dbUrl,
+      },
+    }
+    await configs.save(settingsConfig)
+  }
+
+  beforeEach(async () => {
+    config.newTenant()
+  })
+
+  describe("getPlatformUrl", () => {
+    describe("self host", () => {
+      beforeEach(async () => {
+        testEnv.selfHosted()
+      })
+
+      it("gets the default url", async () => {
+        await config.doInTenant(async () => {
+          const url = await configs.getPlatformUrl()
+          expect(url).toBe(DEFAULT_URL)
+        })
+      })
+
+      it("gets the platform url from the environment", async () => {
+        await config.doInTenant(async () => {
+          env._set("PLATFORM_URL", ENV_URL)
+          const url = await configs.getPlatformUrl()
+          expect(url).toBe(ENV_URL)
+        })
+      })
+
+      it("gets the platform url from the database", async () => {
+        await config.doInTenant(async () => {
+          const dbUrl = generator.url()
+          await setDbPlatformUrl(dbUrl)
+          const url = await configs.getPlatformUrl()
+          expect(url).toBe(dbUrl)
+        })
+      })
+    })
+
+    describe("cloud", () => {
+      function getTenantAwareUrl() {
+        return `http://${config.tenantId}.env.com`
+      }
+
+      beforeEach(async () => {
+        testEnv.cloudHosted()
+        testEnv.multiTenant()
+
+        env._set("PLATFORM_URL", ENV_URL)
+      })
+
+      it("gets the platform url from the environment without tenancy", async () => {
+        await config.doInTenant(async () => {
+          const url = await configs.getPlatformUrl({ tenantAware: false })
+          expect(url).toBe(ENV_URL)
+        })
+      })
+
+      it("gets the platform url from the environment with tenancy", async () => {
+        await config.doInTenant(async () => {
+          const url = await configs.getPlatformUrl()
+          expect(url).toBe(getTenantAwareUrl())
+        })
+      })
+
+      it("never gets the platform url from the database", async () => {
+        await config.doInTenant(async () => {
+          await setDbPlatformUrl(generator.url())
+          const url = await configs.getPlatformUrl()
+          expect(url).toBe(getTenantAwareUrl())
+        })
+      })
+    })
+  })
+
+  describe("getSettingsConfig", () => {
+    beforeAll(async () => {
+      testEnv.selfHosted()
+      env._set("PLATFORM_URL", "")
+    })
+
+    it("returns the platform url with an existing config", async () => {
+      await config.doInTenant(async () => {
+        const dbUrl = generator.url()
+        await setDbPlatformUrl(dbUrl)
+        const config = await configs.getSettingsConfig()
+        expect(config.platformUrl).toBe(dbUrl)
+      })
+    })
+
+    it("returns the platform url without an existing config", async () => {
+      await config.doInTenant(async () => {
+        const config = await configs.getSettingsConfig()
+        expect(config.platformUrl).toBe(DEFAULT_URL)
+      })
+    })
+  })
+})

packages/backend-core/src/constants/db.ts

@@ -68,6 +68,7 @@ export enum DocumentType {
   MEM_VIEW = "view",
   USER_FLAG = "flag",
   AUTOMATION_METADATA = "meta_au",
+  AUDIT_LOG = "al",
 }
 
 export const StaticDatabases = {
@@ -88,6 +89,9 @@ export const StaticDatabases = {
       install: "install",
     },
   },
+  AUDIT_LOGS: {
+    name: "audit-logs",
+  },
 }
 
 export const APP_PREFIX = DocumentType.APP + SEPARATOR

packages/backend-core/src/constants/misc.ts

@@ -41,5 +41,6 @@ export enum Config {
   OIDC_LOGOS = "logos_oidc",
 }
 
+export const MIN_VALID_DATE = new Date(-2147483647000)
 export const MAX_VALID_DATE = new Date(2147483647000)
 export const DEFAULT_TENANT_ID = "default"

packages/backend-core/src/context/Context.ts

@@ -1,5 +1,5 @@
 import { AsyncLocalStorage } from "async_hooks"
-import { ContextMap } from "./mainContext"
+import { ContextMap } from "./types"
 
 export default class Context {
   static storage = new AsyncLocalStorage<ContextMap>()

packages/backend-core/src/context/identity.ts

@@ -5,6 +5,8 @@ import {
   isCloudAccount,
   Account,
   AccountUserContext,
+  UserContext,
+  Ctx,
 } from "@budibase/types"
 import * as context from "."
 
@@ -16,15 +18,22 @@ export function doInIdentityContext(identity: IdentityContext, task: any) {
   return context.doInIdentityContext(identity, task)
 }
 
-export function doInUserContext(user: User, task: any) {
-  const userContext: any = {
+// used in server/worker
+export function doInUserContext(user: User, ctx: Ctx, task: any) {
+  const userContext: UserContext = {
     ...user,
     _id: user._id as string,
     type: IdentityType.USER,
+    hostInfo: {
+      ipAddress: ctx.request.ip,
+      // filled in by koa-useragent package
+      userAgent: ctx.userAgent._agent.source,
+    },
   }
   return doInIdentityContext(userContext, task)
 }
 
+// used in account portal
 export function doInAccountContext(account: Account, task: any) {
   const _id = getAccountUserId(account)
   const tenantId = account.tenantId

packages/backend-core/src/context/mainContext.ts

@@ -11,13 +11,7 @@ import {
   DEFAULT_TENANT_ID,
 } from "../constants"
 import { Database, IdentityContext } from "@budibase/types"
-
-export type ContextMap = {
-  tenantId?: string
-  appId?: string
-  identity?: IdentityContext
-  environmentVariables?: Record<string, string>
-}
+import { ContextMap } from "./types"
 
 let TEST_APP_ID: string | null = null
 
@@ -30,14 +24,23 @@ export function getGlobalDBName(tenantId?: string) {
   return baseGlobalDBName(tenantId)
 }
 
-export function baseGlobalDBName(tenantId: string | undefined | null) {
-  let dbName
-  if (!tenantId || tenantId === DEFAULT_TENANT_ID) {
-    dbName = StaticDatabases.GLOBAL.name
-  } else {
-    dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}`
+export function getAuditLogDBName(tenantId?: string) {
+  if (!tenantId) {
+    tenantId = getTenantId()
+  }
+  if (tenantId === DEFAULT_TENANT_ID) {
+    return StaticDatabases.AUDIT_LOGS.name
+  } else {
+    return `${tenantId}${SEPARATOR}${StaticDatabases.AUDIT_LOGS.name}`
+  }
+}
+
+export function baseGlobalDBName(tenantId: string | undefined | null) {
+  if (!tenantId || tenantId === DEFAULT_TENANT_ID) {
+    return StaticDatabases.GLOBAL.name
+  } else {
+    return `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}`
   }
-  return dbName
 }
 
 export function isMultiTenant() {
@@ -228,6 +231,13 @@ export function getGlobalDB(): Database {
   return getDB(baseGlobalDBName(context?.tenantId))
 }
 
+export function getAuditLogsDB(): Database {
+  if (!getTenantId()) {
+    throw new Error("No tenant ID found - cannot open audit log DB")
+  }
+  return getDB(getAuditLogDBName())
+}
+
 /**
  * Gets the app database based on whatever the request
  * contained, dev or prod.

packages/backend-core/src/context/types.ts

@@ -0,0 +1,9 @@
+import { IdentityContext } from "@budibase/types"
+
+// keep this out of Budibase types, don't want to expose context info
+export type ContextMap = {
+  tenantId?: string
+  appId?: string
+  identity?: IdentityContext
+  environmentVariables?: Record<string, string>
+}

packages/backend-core/src/db/index.ts

@@ -7,3 +7,4 @@ export { default as Replication } from "./Replication"
 // exports to support old export structure
 export * from "../constants/db"
 export { getGlobalDBName, baseGlobalDBName } from "../context"
+export * from "./lucene"

packages/backend-core/src/db/lucene.ts

@@ -0,0 +1,624 @@
+import fetch from "node-fetch"
+import { getCouchInfo } from "./couch"
+import { SearchFilters, Row } from "@budibase/types"
+
+const QUERY_START_REGEX = /\d[0-9]*:/g
+
+interface SearchResponse<T> {
+  rows: T[] | any[]
+  bookmark: string
+}
+
+interface PaginatedSearchResponse<T> extends SearchResponse<T> {
+  hasNextPage: boolean
+}
+
+export type SearchParams<T> = {
+  tableId?: string
+  sort?: string
+  sortOrder?: string
+  sortType?: string
+  limit?: number
+  bookmark?: string
+  version?: string
+  indexer?: () => Promise<any>
+  disableEscaping?: boolean
+  rows?: T | Row[]
+}
+
+export function removeKeyNumbering(key: any): string {
+  if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) {
+    const parts = key.split(":")
+    // remove the number
+    parts.shift()
+    return parts.join(":")
+  } else {
+    return key
+  }
+}
+
+/**
+ * Class to build lucene query URLs.
+ * Optionally takes a base lucene query object.
+ */
+export class QueryBuilder<T> {
+  dbName: string
+  index: string
+  query: SearchFilters
+  limit: number
+  sort?: string
+  bookmark?: string
+  sortOrder: string
+  sortType: string
+  includeDocs: boolean
+  version?: string
+  indexBuilder?: () => Promise<any>
+  noEscaping = false
+
+  constructor(dbName: string, index: string, base?: SearchFilters) {
+    this.dbName = dbName
+    this.index = index
+    this.query = {
+      allOr: false,
+      string: {},
+      fuzzy: {},
+      range: {},
+      equal: {},
+      notEqual: {},
+      empty: {},
+      notEmpty: {},
+      oneOf: {},
+      contains: {},
+      notContains: {},
+      containsAny: {},
+      ...base,
+    }
+    this.limit = 50
+    this.sortOrder = "ascending"
+    this.sortType = "string"
+    this.includeDocs = true
+  }
+
+  disableEscaping() {
+    this.noEscaping = true
+    return this
+  }
+
+  setIndexBuilder(builderFn: () => Promise<any>) {
+    this.indexBuilder = builderFn
+    return this
+  }
+
+  setVersion(version?: string) {
+    if (version != null) {
+      this.version = version
+    }
+    return this
+  }
+
+  setTable(tableId: string) {
+    this.query.equal!.tableId = tableId
+    return this
+  }
+
+  setLimit(limit?: number) {
+    if (limit != null) {
+      this.limit = limit
+    }
+    return this
+  }
+
+  setSort(sort?: string) {
+    if (sort != null) {
+      this.sort = sort
+    }
+    return this
+  }
+
+  setSortOrder(sortOrder?: string) {
+    if (sortOrder != null) {
+      this.sortOrder = sortOrder
+    }
+    return this
+  }
+
+  setSortType(sortType?: string) {
+    if (sortType != null) {
+      this.sortType = sortType
+    }
+    return this
+  }
+
+  setBookmark(bookmark?: string) {
+    if (bookmark != null) {
+      this.bookmark = bookmark
+    }
+    return this
+  }
+
+  excludeDocs() {
+    this.includeDocs = false
+    return this
+  }
+
+  addString(key: string, partial: string) {
+    this.query.string![key] = partial
+    return this
+  }
+
+  addFuzzy(key: string, fuzzy: string) {
+    this.query.fuzzy![key] = fuzzy
+    return this
+  }
+
+  addRange(key: string, low: string | number, high: string | number) {
+    this.query.range![key] = {
+      low,
+      high,
+    }
+    return this
+  }
+
+  addEqual(key: string, value: any) {
+    this.query.equal![key] = value
+    return this
+  }
+
+  addNotEqual(key: string, value: any) {
+    this.query.notEqual![key] = value
+    return this
+  }
+
+  addEmpty(key: string, value: any) {
+    this.query.empty![key] = value
+    return this
+  }
+
+  addNotEmpty(key: string, value: any) {
+    this.query.notEmpty![key] = value
+    return this
+  }
+
+  addOneOf(key: string, value: any) {
+    this.query.oneOf![key] = value
+    return this
+  }
+
+  addContains(key: string, value: any) {
+    this.query.contains![key] = value
+    return this
+  }
+
+  addNotContains(key: string, value: any) {
+    this.query.notContains![key] = value
+    return this
+  }
+
+  addContainsAny(key: string, value: any) {
+    this.query.containsAny![key] = value
+    return this
+  }
+
+  handleSpaces(input: string) {
+    if (this.noEscaping) {
+      return input
+    } else {
+      return input.replace(/ /g, "_")
+    }
+  }
+
+  /**
+   * Preprocesses a value before going into a lucene search.
+   * Transforms strings to lowercase and wraps strings and bools in quotes.
+   * @param value The value to process
+   * @param options The preprocess options
+   * @returns {string|*}
+   */
+  preprocess(value: any, { escape, lowercase, wrap, type }: any = {}) {
+    const hasVersion = !!this.version
+    // Determine if type needs wrapped
+    const originalType = typeof value
+    // Convert to lowercase
+    if (value && lowercase) {
+      value = value.toLowerCase ? value.toLowerCase() : value
+    }
+    // Escape characters
+    if (!this.noEscaping && escape && originalType === "string") {
+      value = `${value}`.replace(/[ #+\-&|!(){}\]^"~*?:\\]/g, "\\$&")
+    }
+
+    // Wrap in quotes
+    if (originalType === "string" && !isNaN(value) && !type) {
+      value = `"${value}"`
+    } else if (hasVersion && wrap) {
+      value = originalType === "number" ? value : `"${value}"`
+    }
+    return value
+  }
+
+  buildSearchQuery() {
+    const builder = this
+    let allOr = this.query && this.query.allOr
+    let query = allOr ? "" : "*:*"
+    const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true }
+    let tableId
+    if (this.query.equal!.tableId) {
+      tableId = this.query.equal!.tableId
+      delete this.query.equal!.tableId
+    }
+
+    const equal = (key: string, value: any) => {
+      // 0 evaluates to false, which means we would return all rows if we don't check it
+      if (!value && value !== 0) {
+        return null
+      }
+      return `${key}:${builder.preprocess(value, allPreProcessingOpts)}`
+    }
+
+    const contains = (key: string, value: any, mode = "AND") => {
+      if (Array.isArray(value) && value.length === 0) {
+        return null
+      }
+      if (!Array.isArray(value)) {
+        return `${key}:${value}`
+      }
+      let statement = `${builder.preprocess(value[0], { escape: true })}`
+      for (let i = 1; i < value.length; i++) {
+        statement += ` ${mode} ${builder.preprocess(value[i], {
+          escape: true,
+        })}`
+      }
+      return `${key}:(${statement})`
+    }
+
+    const notContains = (key: string, value: any) => {
+      // @ts-ignore
+      const allPrefix = allOr === "" ? "*:* AND" : ""
+      return allPrefix + "NOT " + contains(key, value)
+    }
+
+    const containsAny = (key: string, value: any) => {
+      return contains(key, value, "OR")
+    }
+
+    const oneOf = (key: string, value: any) => {
+      if (!Array.isArray(value)) {
+        if (typeof value === "string") {
+          value = value.split(",")
+        } else {
+          return ""
+        }
+      }
+      let orStatement = `${builder.preprocess(value[0], allPreProcessingOpts)}`
+      for (let i = 1; i < value.length; i++) {
+        orStatement += ` OR ${builder.preprocess(
+          value[i],
+          allPreProcessingOpts
+        )}`
+      }
+      return `${key}:(${orStatement})`
+    }
+
+    function build(structure: any, queryFn: any) {
+      for (let [key, value] of Object.entries(structure)) {
+        // check for new format - remove numbering if needed
+        key = removeKeyNumbering(key)
+        key = builder.preprocess(builder.handleSpaces(key), {
+          escape: true,
+        })
+        const expression = queryFn(key, value)
+        if (expression == null) {
+          continue
+        }
+        if (query.length > 0) {
+          query += ` ${allOr ? "OR" : "AND"} `
+        }
+        query += expression
+      }
+    }
+
+    // Construct the actual lucene search query string from JSON structure
+    if (this.query.string) {
+      build(this.query.string, (key: string, value: any) => {
+        if (!value) {
+          return null
+        }
+        value = builder.preprocess(value, {
+          escape: true,
+          lowercase: true,
+          type: "string",
+        })
+        return `${key}:${value}*`
+      })
+    }
+    if (this.query.range) {
+      build(this.query.range, (key: string, value: any) => {
+        if (!value) {
+          return null
+        }
+        if (value.low == null || value.low === "") {
+          return null
+        }
+        if (value.high == null || value.high === "") {
+          return null
+        }
+        const low = builder.preprocess(value.low, allPreProcessingOpts)
+        const high = builder.preprocess(value.high, allPreProcessingOpts)
+        return `${key}:[${low} TO ${high}]`
+      })
+    }
+    if (this.query.fuzzy) {
+      build(this.query.fuzzy, (key: string, value: any) => {
+        if (!value) {
+          return null
+        }
+        value = builder.preprocess(value, {
+          escape: true,
+          lowercase: true,
+          type: "fuzzy",
+        })
+        return `${key}:${value}~`
+      })
+    }
+    if (this.query.equal) {
+      build(this.query.equal, equal)
+    }
+    if (this.query.notEqual) {
+      build(this.query.notEqual, (key: string, value: any) => {
+        if (!value) {
+          return null
+        }
+        return `!${key}:${builder.preprocess(value, allPreProcessingOpts)}`
+      })
+    }
+    if (this.query.empty) {
+      build(this.query.empty, (key: string) => `!${key}:["" TO *]`)
+    }
+    if (this.query.notEmpty) {
+      build(this.query.notEmpty, (key: string) => `${key}:["" TO *]`)
+    }
+    if (this.query.oneOf) {
+      build(this.query.oneOf, oneOf)
+    }
+    if (this.query.contains) {
+      build(this.query.contains, contains)
+    }
+    if (this.query.notContains) {
+      build(this.query.notContains, notContains)
+    }
+    if (this.query.containsAny) {
+      build(this.query.containsAny, containsAny)
+    }
+    // make sure table ID is always added as an AND
+    if (tableId) {
+      query = `(${query})`
+      allOr = false
+      build({ tableId }, equal)
+    }
+    return query
+  }
+
+  buildSearchBody() {
+    let body: any = {
+      q: this.buildSearchQuery(),
+      limit: Math.min(this.limit, 200),
+      include_docs: this.includeDocs,
+    }
+    if (this.bookmark) {
+      body.bookmark = this.bookmark
+    }
+    if (this.sort) {
+      const order = this.sortOrder === "descending" ? "-" : ""
+      const type = `<${this.sortType}>`
+      body.sort = `${order}${this.handleSpaces(this.sort)}${type}`
+    }
+    return body
+  }
+
+  async run() {
+    const { url, cookie } = getCouchInfo()
+    const fullPath = `${url}/${this.dbName}/_design/database/_search/${this.index}`
+    const body = this.buildSearchBody()
+    try {
+      return await runQuery<T>(fullPath, body, cookie)
+    } catch (err: any) {
+      if (err.status === 404 && this.indexBuilder) {
+        await this.indexBuilder()
+        return await runQuery<T>(fullPath, body, cookie)
+      } else {
+        throw err
+      }
+    }
+  }
+}
+
+/**
+ * Executes a lucene search query.
+ * @param url The query URL
+ * @param body The request body defining search criteria
+ * @param cookie The auth cookie for CouchDB
+ * @returns {Promise<{rows: []}>}
+ */
+async function runQuery<T>(
+  url: string,
+  body: any,
+  cookie: string
+): Promise<SearchResponse<T>> {
+  const response = await fetch(url, {
+    body: JSON.stringify(body),
+    method: "POST",
+    headers: {
+      Authorization: cookie,
+    },
+  })
+
+  if (response.status === 404) {
+    throw response
+  }
+  const json = await response.json()
+
+  let output: any = {
+    rows: [],
+  }
+  if (json.rows != null && json.rows.length > 0) {
+    output.rows = json.rows.map((row: any) => row.doc)
+  }
+  if (json.bookmark) {
+    output.bookmark = json.bookmark
+  }
+  return output
+}
+
+/**
+ * Gets round the fixed limit of 200 results from a query by fetching as many
+ * pages as required and concatenating the results. This recursively operates
+ * until enough results have been found.
+ * @param dbName {string} Which database to run a lucene query on
+ * @param index {string} Which search index to utilise
+ * @param query {object} The JSON query structure
+ * @param params {object} The search params including:
+ *   tableId {string} The table ID to search
+ *   sort {string} The sort column
+ *   sortOrder {string} The sort order ("ascending" or "descending")
+ *   sortType {string} Whether to treat sortable values as strings or
+ *     numbers. ("string" or "number")
+ *   limit {number} The number of results to fetch
+ *   bookmark {string|null} Current bookmark in the recursive search
+ *   rows {array|null} Current results in the recursive search
+ * @returns {Promise<*[]|*>}
+ */
+async function recursiveSearch<T>(
+  dbName: string,
+  index: string,
+  query: any,
+  params: any
+): Promise<any> {
+  const bookmark = params.bookmark
+  const rows = params.rows || []
+  if (rows.length >= params.limit) {
+    return rows
+  }
+  let pageSize = 200
+  if (rows.length > params.limit - 200) {
+    pageSize = params.limit - rows.length
+  }
+  const page = await new QueryBuilder<T>(dbName, index, query)
+    .setVersion(params.version)
+    .setTable(params.tableId)
+    .setBookmark(bookmark)
+    .setLimit(pageSize)
+    .setSort(params.sort)
+    .setSortOrder(params.sortOrder)
+    .setSortType(params.sortType)
+    .run()
+  if (!page.rows.length) {
+    return rows
+  }
+  if (page.rows.length < 200) {
+    return [...rows, ...page.rows]
+  }
+  const newParams = {
+    ...params,
+    bookmark: page.bookmark,
+    rows: [...rows, ...page.rows],
+  }
+  return await recursiveSearch(dbName, index, query, newParams)
+}
+
+/**
+ * Performs a paginated search. A bookmark will be returned to allow the next
+ * page to be fetched. There is a max limit off 200 results per page in a
+ * paginated search.
+ * @param dbName {string} Which database to run a lucene query on
+ * @param index {string} Which search index to utilise
+ * @param query {object} The JSON query structure
+ * @param params {object} The search params including:
+ *   tableId {string} The table ID to search
+ *   sort {string} The sort column
+ *   sortOrder {string} The sort order ("ascending" or "descending")
+ *   sortType {string} Whether to treat sortable values as strings or
+ *     numbers. ("string" or "number")
+ *   limit {number} The desired page size
+ *   bookmark {string} The bookmark to resume from
+ * @returns {Promise<{hasNextPage: boolean, rows: *[]}>}
+ */
+export async function paginatedSearch<T>(
+  dbName: string,
+  index: string,
+  query: SearchFilters,
+  params: SearchParams<T>
+) {
+  let limit = params.limit
+  if (limit == null || isNaN(limit) || limit < 0) {
+    limit = 50
+  }
+  limit = Math.min(limit, 200)
+  const search = new QueryBuilder<T>(dbName, index, query)
+  if (params.version) {
+    search.setVersion(params.version)
+  }
+  if (params.tableId) {
+    search.setTable(params.tableId)
+  }
+  if (params.sort) {
+    search
+      .setSort(params.sort)
+      .setSortOrder(params.sortOrder)
+      .setSortType(params.sortType)
+  }
+  if (params.indexer) {
+    search.setIndexBuilder(params.indexer)
+  }
+  if (params.disableEscaping) {
+    search.disableEscaping()
+  }
+  const searchResults = await search
+    .setBookmark(params.bookmark)
+    .setLimit(limit)
+    .run()
+
+  // Try fetching 1 row in the next page to see if another page of results
+  // exists or not
+  search.setBookmark(searchResults.bookmark).setLimit(1)
+  if (params.tableId) {
+    search.setTable(params.tableId)
+  }
+  const nextResults = await search.run()
+
+  return {
+    ...searchResults,
+    hasNextPage: nextResults.rows && nextResults.rows.length > 0,
+  }
+}
+
+/**
+ * Performs a full search, fetching multiple pages if required to return the
+ * desired amount of results. There is a limit of 1000 results to avoid
+ * heavy performance hits, and to avoid client components breaking from
+ * handling too much data.
+ * @param dbName {string} Which database to run a lucene query on
+ * @param index {string} Which search index to utilise
+ * @param query {object} The JSON query structure
+ * @param params {object} The search params including:
+ *   tableId {string} The table ID to search
+ *   sort {string} The sort column
+ *   sortOrder {string} The sort order ("ascending" or "descending")
+ *   sortType {string} Whether to treat sortable values as strings or
+ *     numbers. ("string" or "number")
+ *   limit {number} The desired number of results
+ * @returns {Promise<{rows: *}>}
+ */
+export async function fullSearch<T>(
+  dbName: string,
+  index: string,
+  query: SearchFilters,
+  params: SearchParams<T>
+) {
+  let limit = params.limit
+  if (limit == null || isNaN(limit) || limit < 0) {
+    limit = 1000
+  }
+  params.limit = Math.min(limit, 1000)
+  const rows = await recursiveSearch<T>(dbName, index, query, params)
+  return { rows }
+}

packages/backend-core/src/db/tests/lucene.spec.ts

@@ -0,0 +1,161 @@
+import { newid } from "../../newid"
+import { getDB } from "../db"
+import { Database } from "@budibase/types"
+import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene"
+
+const INDEX_NAME = "main"
+
+const index = `function(doc) {
+  let props = ["property", "number"]
+  for (let key of props) {
+    if (doc[key]) {
+      index(key, doc[key])
+    }
+  }
+}`
+
+describe("lucene", () => {
+  let db: Database, dbName: string
+
+  beforeAll(async () => {
+    dbName = `db-${newid()}`
+    // create the DB for testing
+    db = getDB(dbName)
+    await db.put({ _id: newid(), property: "word" })
+    await db.put({ _id: newid(), property: "word2" })
+    await db.put({ _id: newid(), property: "word3", number: 1 })
+  })
+
+  it("should be able to create a lucene index", async () => {
+    const response = await db.put({
+      _id: "_design/database",
+      indexes: {
+        [INDEX_NAME]: {
+          index: index,
+          analyzer: "standard",
+        },
+      },
+    })
+    expect(response.ok).toBe(true)
+  })
+
+  describe("query builder", () => {
+    it("should be able to perform a basic query", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.setSort("property")
+      builder.setSortOrder("desc")
+      builder.setSortType("string")
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(3)
+    })
+
+    it("should handle limits", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.setLimit(1)
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(1)
+    })
+
+    it("should be able to perform a string search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addString("property", "wo")
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(3)
+    })
+
+    it("should be able to perform a range search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addRange("number", 0, 1)
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(1)
+    })
+
+    it("should be able to perform an equal search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addEqual("property", "word2")
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(1)
+    })
+
+    it("should be able to perform a not equal search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addNotEqual("property", "word2")
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(2)
+    })
+
+    it("should be able to perform an empty search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addEmpty("number", true)
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(2)
+    })
+
+    it("should be able to perform a not empty search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addNotEmpty("number", true)
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(1)
+    })
+
+    it("should be able to perform a one of search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addOneOf("property", ["word", "word2"])
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(2)
+    })
+
+    it("should be able to perform a contains search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addContains("property", ["word"])
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(1)
+    })
+
+    it("should be able to perform a not contains search", async () => {
+      const builder = new QueryBuilder(dbName, INDEX_NAME)
+      builder.addNotContains("property", ["word2"])
+      const resp = await builder.run()
+      expect(resp.rows.length).toBe(2)
+    })
+  })
+
+  describe("paginated search", () => {
+    it("should be able to perform a paginated search", async () => {
+      const page = await paginatedSearch(
+        dbName,
+        INDEX_NAME,
+        {
+          string: {
+            property: "wo",
+          },
+        },
+        {
+          limit: 1,
+          sort: "property",
+          sortType: "string",
+          sortOrder: "desc",
+        }
+      )
+      expect(page.rows.length).toBe(1)
+      expect(page.hasNextPage).toBe(true)
+      expect(page.bookmark).toBeDefined()
+    })
+  })
+
+  describe("full search", () => {
+    it("should be able to perform a full search", async () => {
+      const page = await fullSearch(
+        dbName,
+        INDEX_NAME,
+        {
+          string: {
+            property: "wo",
+          },
+        },
+        {}
+      )
+      expect(page.rows.length).toBe(3)
+    })
+  })
+})

packages/backend-core/src/db/tests/utils.spec.ts

@@ -1,19 +1,13 @@
-import { generator, DBTestConfiguration, testEnv } from "../../../tests"
 import {
   getDevelopmentAppID,
   getProdAppID,
   isDevAppID,
   isProdAppID,
 } from "../conversions"
-import { generateAppID, getPlatformUrl, getScopedConfig } from "../utils"
-import * as context from "../../context"
-import { Config } from "../../constants"
-import env from "../../environment"
+import { generateAppID } from "../utils"
 
 describe("utils", () => {
-  const config = new DBTestConfiguration()
-
-  describe("app ID manipulation", () => {
+  describe("generateAppID", () => {
     function getID() {
       const appId = generateAppID()
       const split = appId.split("_")
@@ -66,127 +60,4 @@ describe("utils", () => {
       expect(isProdAppID(devAppId)).toEqual(false)
     })
   })
-
-  const DEFAULT_URL = "http://localhost:10000"
-  const ENV_URL = "http://env.com"
-
-  const setDbPlatformUrl = async (dbUrl: string) => {
-    const db = context.getGlobalDB()
-    await db.put({
-      _id: "config_settings",
-      type: Config.SETTINGS,
-      config: {
-        platformUrl: dbUrl,
-      },
-    })
-  }
-
-  const clearSettingsConfig = async () => {
-    await config.doInTenant(async () => {
-      const db = context.getGlobalDB()
-      try {
-        const config = await db.get("config_settings")
-        await db.remove("config_settings", config._rev)
-      } catch (e: any) {
-        if (e.status !== 404) {
-          throw e
-        }
-      }
-    })
-  }
-
-  describe("getPlatformUrl", () => {
-    describe("self host", () => {
-      beforeEach(async () => {
-        testEnv.selfHosted()
-        await clearSettingsConfig()
-      })
-
-      it("gets the default url", async () => {
-        await config.doInTenant(async () => {
-          const url = await getPlatformUrl()
-          expect(url).toBe(DEFAULT_URL)
-        })
-      })
-
-      it("gets the platform url from the environment", async () => {
-        await config.doInTenant(async () => {
-          env._set("PLATFORM_URL", ENV_URL)
-          const url = await getPlatformUrl()
-          expect(url).toBe(ENV_URL)
-        })
-      })
-
-      it("gets the platform url from the database", async () => {
-        await config.doInTenant(async () => {
-          const dbUrl = generator.url()
-          await setDbPlatformUrl(dbUrl)
-          const url = await getPlatformUrl()
-          expect(url).toBe(dbUrl)
-        })
-      })
-    })
-
-    describe("cloud", () => {
-      const TENANT_AWARE_URL = `http://${config.tenantId}.env.com`
-
-      beforeEach(async () => {
-        testEnv.cloudHosted()
-        testEnv.multiTenant()
-
-        env._set("PLATFORM_URL", ENV_URL)
-        await clearSettingsConfig()
-      })
-
-      it("gets the platform url from the environment without tenancy", async () => {
-        await config.doInTenant(async () => {
-          const url = await getPlatformUrl({ tenantAware: false })
-          expect(url).toBe(ENV_URL)
-        })
-      })
-
-      it("gets the platform url from the environment with tenancy", async () => {
-        await config.doInTenant(async () => {
-          const url = await getPlatformUrl()
-          expect(url).toBe(TENANT_AWARE_URL)
-        })
-      })
-
-      it("never gets the platform url from the database", async () => {
-        await config.doInTenant(async () => {
-          await setDbPlatformUrl(generator.url())
-          const url = await getPlatformUrl()
-          expect(url).toBe(TENANT_AWARE_URL)
-        })
-      })
-    })
-  })
-
-  describe("getScopedConfig", () => {
-    describe("settings config", () => {
-      beforeEach(async () => {
-        env._set("SELF_HOSTED", 1)
-        env._set("PLATFORM_URL", "")
-        await clearSettingsConfig()
-      })
-
-      it("returns the platform url with an existing config", async () => {
-        await config.doInTenant(async () => {
-          const dbUrl = generator.url()
-          await setDbPlatformUrl(dbUrl)
-          const db = context.getGlobalDB()
-          const config = await getScopedConfig(db, { type: Config.SETTINGS })
-          expect(config.platformUrl).toBe(dbUrl)
-        })
-      })
-
-      it("returns the platform url without an existing config", async () => {
-        await config.doInTenant(async () => {
-          const db = context.getGlobalDB()
-          const config = await getScopedConfig(db, { type: Config.SETTINGS })
-          expect(config.platformUrl).toBe(DEFAULT_URL)
-        })
-      })
-    })
-  })
 })

packages/backend-core/src/db/utils.ts

@@ -9,12 +9,11 @@ import {
   InternalTable,
   APP_PREFIX,
 } from "../constants"
-import { getTenantId, getGlobalDB, getGlobalDBName } from "../context"
+import { getTenantId, getGlobalDBName } from "../context"
 import { doWithDB, directCouchAllDbs } from "./db"
 import { getAppMetadata } from "../cache/appMetadata"
 import { isDevApp, isDevAppID, getProdAppID } from "./conversions"
-import * as events from "../events"
-import { App, Database, ConfigType, isSettingsConfig } from "@budibase/types"
+import { App, Database } from "@budibase/types"
 
 /**
  * Generates a new app ID.
@@ -366,6 +365,16 @@ export async function getAllApps({
   }
 }
 
+export async function getAppsByIDs(appIds: string[]) {
+  const settled = await Promise.allSettled(
+    appIds.map(appId => getAppMetadata(appId))
+  )
+  // have to list the apps which exist, some may have been deleted
+  return settled
+    .filter(promise => promise.status === "fulfilled")
+    .map(promise => (promise as PromiseFulfilledResult<App>).value)
+}
+
 /**
  * Utility function for getAllApps but filters to production apps only.
  */
@@ -382,6 +391,16 @@ export async function getDevAppIDs() {
   return apps.filter((id: any) => isDevAppID(id))
 }
 
+export function isSameAppID(
+  appId1: string | undefined,
+  appId2: string | undefined
+) {
+  if (appId1 == undefined || appId2 == undefined) {
+    return false
+  }
+  return getProdAppID(appId1) === getProdAppID(appId2)
+}
+
 export async function dbExists(dbName: any) {
   return doWithDB(
     dbName,
@@ -392,32 +411,6 @@ export async function dbExists(dbName: any) {
   )
 }
 
-/**
- * Generates a new configuration ID.
- * @returns {string} The new configuration ID which the config doc can be stored under.
- */
-export const generateConfigID = ({ type, workspace, user }: any) => {
-  const scope = [type, workspace, user].filter(Boolean).join(SEPARATOR)
-
-  return `${DocumentType.CONFIG}${SEPARATOR}${scope}`
-}
-
-/**
- * Gets parameters for retrieving configurations.
- */
-export const getConfigParams = (
-  { type, workspace, user }: any,
-  otherProps = {}
-) => {
-  const scope = [type, workspace, user].filter(Boolean).join(SEPARATOR)
-
-  return {
-    ...otherProps,
-    startkey: `${DocumentType.CONFIG}${SEPARATOR}${scope}`,
-    endkey: `${DocumentType.CONFIG}${SEPARATOR}${scope}${UNICODE_MAX}`,
-  }
-}
-
 /**
  * Generates a new dev info document ID - this is scoped to a user.
  * @returns {string} The new dev info ID which info for dev (like api key) can be stored under.
@@ -441,109 +434,6 @@ export const getPluginParams = (pluginId?: string | null, otherProps = {}) => {
   return getDocParams(DocumentType.PLUGIN, pluginId, otherProps)
 }
 
-/**
- * Returns the most granular configuration document from the DB based on the type, workspace and userID passed.
- * @param {Object} db - db instance to query
- * @param {Object} scopes - the type, workspace and userID scopes of the configuration.
- * @returns The most granular configuration document based on the scope.
- */
-export const getScopedFullConfig = async function (
-  db: any,
-  { type, user, workspace }: any
-) {
-  const response = await db.allDocs(
-    getConfigParams(
-      { type, user, workspace },
-      {
-        include_docs: true,
-      }
-    )
-  )
-
-  function determineScore(row: any) {
-    const config = row.doc
-
-    // Config is specific to a user and a workspace
-    if (config._id.includes(generateConfigID({ type, user, workspace }))) {
-      return 4
-    } else if (config._id.includes(generateConfigID({ type, user }))) {
-      // Config is specific to a user only
-      return 3
-    } else if (config._id.includes(generateConfigID({ type, workspace }))) {
-      // Config is specific to a workspace only
-      return 2
-    } else if (config._id.includes(generateConfigID({ type }))) {
-      // Config is specific to a type only
-      return 1
-    }
-    return 0
-  }
-
-  // Find the config with the most granular scope based on context
-  let scopedConfig = response.rows.sort(
-    (a: any, b: any) => determineScore(a) - determineScore(b)
-  )[0]
-
-  // custom logic for settings doc
-  if (type === ConfigType.SETTINGS) {
-    if (!scopedConfig || !scopedConfig.doc) {
-      // defaults
-      scopedConfig = {
-        doc: {
-          _id: generateConfigID({ type, user, workspace }),
-          type: ConfigType.SETTINGS,
-          config: {
-            platformUrl: await getPlatformUrl({ tenantAware: true }),
-            analyticsEnabled: await events.analytics.enabled(),
-          },
-        },
-      }
-    }
-
-    // will always be true - use assertion function to get type access
-    if (isSettingsConfig(scopedConfig.doc)) {
-      // overrides affected by environment
-      scopedConfig.doc.config.platformUrl = await getPlatformUrl({
-        tenantAware: true,
-      })
-      scopedConfig.doc.config.analyticsEnabled =
-        await events.analytics.enabled()
-    }
-  }
-
-  return scopedConfig && scopedConfig.doc
-}
-
-export const getPlatformUrl = async (opts = { tenantAware: true }) => {
-  let platformUrl = env.PLATFORM_URL || "http://localhost:10000"
-
-  if (!env.SELF_HOSTED && env.MULTI_TENANCY && opts.tenantAware) {
-    // cloud and multi tenant - add the tenant to the default platform url
-    const tenantId = getTenantId()
-    if (!platformUrl.includes("localhost:")) {
-      platformUrl = platformUrl.replace("://", `://${tenantId}.`)
-    }
-  } else if (env.SELF_HOSTED) {
-    const db = getGlobalDB()
-    // get the doc directly instead of with getScopedConfig to prevent loop
-    let settings
-    try {
-      settings = await db.get(generateConfigID({ type: ConfigType.SETTINGS }))
-    } catch (e: any) {
-      if (e.status !== 404) {
-        throw e
-      }
-    }
-
-    // self hosted - check for platform url override
-    if (settings && settings.config && settings.config.platformUrl) {
-      platformUrl = settings.config.platformUrl
-    }
-  }
-
-  return platformUrl
-}
-
 export function pagination(
   data: any[],
   pageSize: number,
@@ -577,8 +467,3 @@ export function pagination(
     nextPage,
   }
 }
-
-export async function getScopedConfig(db: any, params: any) {
-  const configDoc = await getScopedFullConfig(db, params)
-  return configDoc && configDoc.config ? configDoc.config : configDoc
-}

packages/backend-core/src/environment.ts

@@ -28,6 +28,8 @@ const DefaultBucketName = {
   PLUGINS: "plugins",
 }
 
+const selfHosted = !!parseInt(process.env.SELF_HOSTED || "")
+
 const environment = {
   isTest,
   isJest,
@@ -58,7 +60,7 @@ const environment = {
     process.env.ACCOUNT_PORTAL_URL || "https://account.budibase.app",
   ACCOUNT_PORTAL_API_KEY: process.env.ACCOUNT_PORTAL_API_KEY || "",
   DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
-  SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED || ""),
+  SELF_HOSTED: selfHosted,
   COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
   PLATFORM_URL: process.env.PLATFORM_URL || "",
   POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,
@@ -84,6 +86,22 @@ const environment = {
   DEPLOYMENT_ENVIRONMENT:
     process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose",
   ENABLE_4XX_HTTP_LOGGING: process.env.ENABLE_4XX_HTTP_LOGGING || true,
+  ENABLE_AUDIT_LOG_IP_ADDR: process.env.ENABLE_AUDIT_LOG_IP_ADDR,
+  // smtp
+  SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,
+  SMTP_USER: process.env.SMTP_USER,
+  SMTP_PASSWORD: process.env.SMTP_PASSWORD,
+  SMTP_HOST: process.env.SMTP_HOST,
+  SMTP_PORT: parseInt(process.env.SMTP_PORT || ""),
+  SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS,
+  /**
+   * Enable to allow an admin user to login using a password.
+   * This can be useful to prevent lockout when configuring SSO.
+   * However, this should be turned OFF by default for security purposes.
+   */
+  ENABLE_SSO_MAINTENANCE_MODE: selfHosted
+    ? process.env.ENABLE_SSO_MAINTENANCE_MODE
+    : false,
   _set(key: any, value: any) {
     process.env[key] = value
     // @ts-ignore

packages/backend-core/src/events/analytics.ts

@@ -1,55 +1,6 @@
-import env from "../environment"
-import * as context from "../context"
-import * as dbUtils from "../db/utils"
-import { Config } from "../constants"
-import { withCache, TTL, CacheKey } from "../cache"
+import * as configs from "../configs"
 
+// wrapper utility function
 export const enabled = async () => {
-  // cloud - always use the environment variable
-  if (!env.SELF_HOSTED) {
-    return !!env.ENABLE_ANALYTICS
-  }
-
-  // self host - prefer the settings doc
-  // use cache as events have high throughput
-  const enabledInDB = await withCache(
-    CacheKey.ANALYTICS_ENABLED,
-    TTL.ONE_DAY,
-    async () => {
-      const settings = await getSettingsDoc()
-
-      // need to do explicit checks in case the field is not set
-      if (settings?.config?.analyticsEnabled === false) {
-        return false
-      } else if (settings?.config?.analyticsEnabled === true) {
-        return true
-      }
-    }
-  )
-
-  if (enabledInDB !== undefined) {
-    return enabledInDB
-  }
-
-  // fallback to the environment variable
-  // explicitly check for 0 or false here, undefined or otherwise is treated as true
-  const envEnabled: any = env.ENABLE_ANALYTICS
-  if (envEnabled === 0 || envEnabled === false) {
-    return false
-  } else {
-    return true
-  }
-}
-
-const getSettingsDoc = async () => {
-  const db = context.getGlobalDB()
-  let settings
-  try {
-    settings = await db.get(dbUtils.generateConfigID({ type: Config.SETTINGS }))
-  } catch (e: any) {
-    if (e.status !== 404) {
-      throw e
-    }
-  }
-  return settings
+  return configs.analyticsEnabled()
 }

packages/backend-core/src/events/events.ts

@@ -1,4 +1,4 @@
-import { Event } from "@budibase/types"
+import { Event, AuditedEventFriendlyName } from "@budibase/types"
 import { processors } from "./processors"
 import identification from "./identification"
 import * as backfill from "./backfill"

packages/backend-core/src/events/identification.ts

@@ -10,7 +10,6 @@ import {
   isCloudAccount,
   isSSOAccount,
   TenantGroup,
-  SettingsConfig,
   CloudAccount,
   UserIdentity,
   InstallationGroup,
@@ -19,10 +18,9 @@ import {
   isSSOUser,
 } from "@budibase/types"
 import { processors } from "./processors"
-import * as dbUtils from "../db/utils"
-import { Config } from "../constants"
 import { newid } from "../utils"
 import * as installation from "../installation"
+import * as configs from "../configs"
 import { withCache, TTL, CacheKey } from "../cache/generic"
 
 const pkg = require("../../package.json")
@@ -89,6 +87,7 @@ const getCurrentIdentity = async (): Promise<Identity> => {
       installationId,
       tenantId,
       environment,
+      hostInfo: userContext.hostInfo,
     }
   } else {
     throw new Error("Unknown identity type")
@@ -270,9 +269,7 @@ const getUniqueTenantId = async (tenantId: string): Promise<string> => {
   return context.doInTenant(tenantId, () => {
     return withCache(CacheKey.UNIQUE_TENANT_ID, TTL.ONE_DAY, async () => {
       const db = context.getGlobalDB()
-      const config: SettingsConfig = await dbUtils.getScopedFullConfig(db, {
-        type: Config.SETTINGS,
-      })
+      const config = await configs.getSettingsConfigDoc()
 
       let uniqueTenantId: string
       if (config.config.uniqueTenantId) {

packages/backend-core/src/events/processors/AuditLogsProcessor.ts

@@ -0,0 +1,93 @@
+import {
+  Event,
+  Identity,
+  Group,
+  IdentityType,
+  AuditLogQueueEvent,
+  AuditLogFn,
+  HostInfo,
+} from "@budibase/types"
+import { EventProcessor } from "./types"
+import { getAppId, doInTenant, getTenantId } from "../../context"
+import BullQueue from "bull"
+import { createQueue, JobQueue } from "../../queue"
+import { isAudited } from "../../utils"
+import env from "../../environment"
+
+export default class AuditLogsProcessor implements EventProcessor {
+  static auditLogsEnabled = false
+  static auditLogQueue: BullQueue.Queue<AuditLogQueueEvent>
+
+  // can't use constructor as need to return promise
+  static init(fn: AuditLogFn) {
+    AuditLogsProcessor.auditLogsEnabled = true
+    const writeAuditLogs = fn
+    AuditLogsProcessor.auditLogQueue = createQueue<AuditLogQueueEvent>(
+      JobQueue.AUDIT_LOG
+    )
+    return AuditLogsProcessor.auditLogQueue.process(async job => {
+      return doInTenant(job.data.tenantId, async () => {
+        let properties = job.data.properties
+        if (properties.audited) {
+          properties = {
+            ...properties,
+            ...properties.audited,
+          }
+          delete properties.audited
+        }
+
+        // this feature is disabled by default due to privacy requirements
+        // in some countries - available as env var in-case it is desired
+        // in self host deployments
+        let hostInfo: HostInfo | undefined = {}
+        if (env.ENABLE_AUDIT_LOG_IP_ADDR) {
+          hostInfo = job.data.opts.hostInfo
+        }
+
+        await writeAuditLogs(job.data.event, properties, {
+          userId: job.data.opts.userId,
+          timestamp: job.data.opts.timestamp,
+          appId: job.data.opts.appId,
+          hostInfo,
+        })
+      })
+    })
+  }
+
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string
+  ): Promise<void> {
+    if (AuditLogsProcessor.auditLogsEnabled && isAudited(event)) {
+      // only audit log actual events, don't include backfills
+      const userId =
+        identity.type === IdentityType.USER ? identity.id : undefined
+      // add to the event queue, rather than just writing immediately
+      await AuditLogsProcessor.auditLogQueue.add({
+        event,
+        properties,
+        opts: {
+          userId,
+          timestamp,
+          appId: getAppId(),
+          hostInfo: identity.hostInfo,
+        },
+        tenantId: getTenantId(),
+      })
+    }
+  }
+
+  async identify(identity: Identity, timestamp?: string | number) {
+    // no-op
+  }
+
+  async identifyGroup(group: Group, timestamp?: string | number) {
+    // no-op
+  }
+
+  shutdown(): void {
+    AuditLogsProcessor.auditLogQueue?.close()
+  }
+}

packages/backend-core/src/events/processors/index.ts

@@ -1,8 +1,19 @@
 import AnalyticsProcessor from "./AnalyticsProcessor"
 import LoggingProcessor from "./LoggingProcessor"
+import AuditLogsProcessor from "./AuditLogsProcessor"
 import Processors from "./Processors"
+import { AuditLogFn } from "@budibase/types"
 
 export const analyticsProcessor = new AnalyticsProcessor()
 const loggingProcessor = new LoggingProcessor()
+const auditLogsProcessor = new AuditLogsProcessor()
 
-export const processors = new Processors([analyticsProcessor, loggingProcessor])
+export function init(auditingFn: AuditLogFn) {
+  return AuditLogsProcessor.init(auditingFn)
+}
+
+export const processors = new Processors([
+  analyticsProcessor,
+  loggingProcessor,
+  auditLogsProcessor,
+])

packages/backend-core/src/events/processors/posthog/PosthogProcessor.ts

@@ -47,6 +47,8 @@ export default class PosthogProcessor implements EventProcessor {
       return
     }
 
+    properties = this.clearPIIProperties(properties)
+
     properties.version = pkg.version
     properties.service = env.SERVICE
     properties.environment = identity.environment
@@ -79,6 +81,16 @@ export default class PosthogProcessor implements EventProcessor {
     this.posthog.capture(payload)
   }
 
+  clearPIIProperties(properties: any) {
+    if (properties.email) {
+      delete properties.email
+    }
+    if (properties.audited) {
+      delete properties.audited
+    }
+    return properties
+  }
+
   async identify(identity: Identity, timestamp?: string | number) {
     const payload: any = { distinctId: identity.id, properties: identity }
     if (timestamp) {

packages/backend-core/src/events/processors/posthog/tests/PosthogProcessor.spec.ts

@@ -49,6 +49,25 @@ describe("PosthogProcessor", () => {
       expect(processor.posthog.capture).toHaveBeenCalledTimes(0)
     })
 
+    it("removes audited information", async () => {
+      const processor = new PosthogProcessor("test")
+
+      const identity = newIdentity()
+      const properties = {
+        email: "test",
+        audited: {
+          name: "test",
+        },
+      }
+
+      await processor.processEvent(Event.USER_CREATED, identity, properties)
+      expect(processor.posthog.capture).toHaveBeenCalled()
+      // @ts-ignore
+      const call = processor.posthog.capture.mock.calls[0][0]
+      expect(call.properties.audited).toBeUndefined()
+      expect(call.properties.email).toBeUndefined()
+    })
+
     describe("rate limiting", () => {
       it("sends daily event once in same day", async () => {
         const processor = new PosthogProcessor("test")

packages/backend-core/src/events/publishers/app.ts

@@ -19,6 +19,9 @@ const created = async (app: App, timestamp?: string | number) => {
   const properties: AppCreatedEvent = {
     appId: app.appId,
     version: app.version,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_CREATED, properties, timestamp)
 }
@@ -27,6 +30,9 @@ async function updated(app: App) {
   const properties: AppUpdatedEvent = {
     appId: app.appId,
     version: app.version,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_UPDATED, properties)
 }
@@ -34,6 +40,9 @@ async function updated(app: App) {
 async function deleted(app: App) {
   const properties: AppDeletedEvent = {
     appId: app.appId,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_DELETED, properties)
 }
@@ -41,6 +50,9 @@ async function deleted(app: App) {
 async function published(app: App, timestamp?: string | number) {
   const properties: AppPublishedEvent = {
     appId: app.appId,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_PUBLISHED, properties, timestamp)
 }
@@ -48,6 +60,9 @@ async function published(app: App, timestamp?: string | number) {
 async function unpublished(app: App) {
   const properties: AppUnpublishedEvent = {
     appId: app.appId,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_UNPUBLISHED, properties)
 }
@@ -55,6 +70,9 @@ async function unpublished(app: App) {
 async function fileImported(app: App) {
   const properties: AppFileImportedEvent = {
     appId: app.appId,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_FILE_IMPORTED, properties)
 }
@@ -63,6 +81,9 @@ async function templateImported(app: App, templateKey: string) {
   const properties: AppTemplateImportedEvent = {
     appId: app.appId,
     templateKey,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_TEMPLATE_IMPORTED, properties)
 }
@@ -76,6 +97,9 @@ async function versionUpdated(
     appId: app.appId,
     currentVersion,
     updatedToVersion,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_VERSION_UPDATED, properties)
 }
@@ -89,6 +113,9 @@ async function versionReverted(
     appId: app.appId,
     currentVersion,
     revertedToVersion,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_VERSION_REVERTED, properties)
 }
@@ -96,6 +123,9 @@ async function versionReverted(
 async function reverted(app: App) {
   const properties: AppRevertedEvent = {
     appId: app.appId,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_REVERTED, properties)
 }
@@ -103,6 +133,9 @@ async function reverted(app: App) {
 async function exported(app: App) {
   const properties: AppExportedEvent = {
     appId: app.appId,
+    audited: {
+      name: app.name,
+    },
   }
   await publishEvent(Event.APP_EXPORTED, properties)
 }

packages/backend-core/src/events/publishers/auditLog.ts

@@ -0,0 +1,26 @@
+import {
+  Event,
+  AuditLogSearchParams,
+  AuditLogFilteredEvent,
+  AuditLogDownloadedEvent,
+} from "@budibase/types"
+import { publishEvent } from "../events"
+
+async function filtered(search: AuditLogSearchParams) {
+  const properties: AuditLogFilteredEvent = {
+    filters: search,
+  }
+  await publishEvent(Event.AUDIT_LOGS_FILTERED, properties)
+}
+
+async function downloaded(search: AuditLogSearchParams) {
+  const properties: AuditLogDownloadedEvent = {
+    filters: search,
+  }
+  await publishEvent(Event.AUDIT_LOGS_DOWNLOADED, properties)
+}
+
+export default {
+  filtered,
+  downloaded,
+}

packages/backend-core/src/events/publishers/auth.ts

@@ -12,19 +12,25 @@ import {
 } from "@budibase/types"
 import { identification } from ".."
 
-async function login(source: LoginSource) {
+async function login(source: LoginSource, email: string) {
   const identity = await identification.getCurrentIdentity()
   const properties: LoginEvent = {
     userId: identity.id,
     source,
+    audited: {
+      email,
+    },
   }
   await publishEvent(Event.AUTH_LOGIN, properties)
 }
 
-async function logout() {
+async function logout(email?: string) {
   const identity = await identification.getCurrentIdentity()
   const properties: LogoutEvent = {
     userId: identity.id,
+    audited: {
+      email,
+    },
   }
   await publishEvent(Event.AUTH_LOGOUT, properties)
 }

packages/backend-core/src/events/publishers/automation.ts

@@ -18,6 +18,9 @@ async function created(automation: Automation, timestamp?: string | number) {
     automationId: automation._id as string,
     triggerId: automation.definition?.trigger?.id,
     triggerType: automation.definition?.trigger?.stepId,
+    audited: {
+      name: automation.name,
+    },
   }
   await publishEvent(Event.AUTOMATION_CREATED, properties, timestamp)
 }
@@ -38,6 +41,9 @@ async function deleted(automation: Automation) {
     automationId: automation._id as string,
     triggerId: automation.definition?.trigger?.id,
     triggerType: automation.definition?.trigger?.stepId,
+    audited: {
+      name: automation.name,
+    },
   }
   await publishEvent(Event.AUTOMATION_DELETED, properties)
 }
@@ -71,6 +77,9 @@ async function stepCreated(
     triggerType: automation.definition?.trigger?.stepId,
     stepId: step.id!,
     stepType: step.stepId,
+    audited: {
+      name: automation.name,
+    },
   }
   await publishEvent(Event.AUTOMATION_STEP_CREATED, properties, timestamp)
 }
@@ -83,6 +92,9 @@ async function stepDeleted(automation: Automation, step: AutomationStep) {
     triggerType: automation.definition?.trigger?.stepId,
     stepId: step.id!,
     stepType: step.stepId,
+    audited: {
+      name: automation.name,
+    },
   }
   await publishEvent(Event.AUTOMATION_STEP_DELETED, properties)
 }

packages/backend-core/src/events/publishers/backup.ts

@@ -13,6 +13,7 @@ async function appBackupRestored(backup: AppBackup) {
     appId: backup.appId,
     restoreId: backup._id!,
     backupCreatedAt: backup.timestamp,
+    name: backup.name as string,
   }
 
   await publishEvent(Event.APP_BACKUP_RESTORED, properties)
@@ -22,13 +23,15 @@ async function appBackupTriggered(
   appId: string,
   backupId: string,
   type: AppBackupType,
-  trigger: AppBackupTrigger
+  trigger: AppBackupTrigger,
+  name: string
 ) {
   const properties: AppBackupTriggeredEvent = {
     appId: appId,
     backupId,
     type,
     trigger,
+    name,
   }
   await publishEvent(Event.APP_BACKUP_TRIGGERED, properties)
 }

packages/backend-core/src/events/publishers/group.ts

@@ -8,12 +8,16 @@ import {
   GroupUsersAddedEvent,
   GroupUsersDeletedEvent,
   GroupAddedOnboardingEvent,
+  GroupPermissionsEditedEvent,
   UserGroupRoles,
 } from "@budibase/types"
 
 async function created(group: UserGroup, timestamp?: number) {
   const properties: GroupCreatedEvent = {
     groupId: group._id as string,
+    audited: {
+      name: group.name,
+    },
   }
   await publishEvent(Event.USER_GROUP_CREATED, properties, timestamp)
 }
@@ -21,6 +25,9 @@ async function created(group: UserGroup, timestamp?: number) {
 async function updated(group: UserGroup) {
   const properties: GroupUpdatedEvent = {
     groupId: group._id as string,
+    audited: {
+      name: group.name,
+    },
   }
   await publishEvent(Event.USER_GROUP_UPDATED, properties)
 }
@@ -28,6 +35,9 @@ async function updated(group: UserGroup) {
 async function deleted(group: UserGroup) {
   const properties: GroupDeletedEvent = {
     groupId: group._id as string,
+    audited: {
+      name: group.name,
+    },
   }
   await publishEvent(Event.USER_GROUP_DELETED, properties)
 }
@@ -36,6 +46,9 @@ async function usersAdded(count: number, group: UserGroup) {
   const properties: GroupUsersAddedEvent = {
     count,
     groupId: group._id as string,
+    audited: {
+      name: group.name,
+    },
   }
   await publishEvent(Event.USER_GROUP_USERS_ADDED, properties)
 }
@@ -44,6 +57,9 @@ async function usersDeleted(count: number, group: UserGroup) {
   const properties: GroupUsersDeletedEvent = {
     count,
     groupId: group._id as string,
+    audited: {
+      name: group.name,
+    },
   }
   await publishEvent(Event.USER_GROUP_USERS_REMOVED, properties)
 }
@@ -56,9 +72,13 @@ async function createdOnboarding(groupId: string) {
   await publishEvent(Event.USER_GROUP_ONBOARDING, properties)
 }
 
-async function permissionsEdited(roles: UserGroupRoles) {
-  const properties: UserGroupRoles = {
-    ...roles,
+async function permissionsEdited(group: UserGroup) {
+  const properties: GroupPermissionsEditedEvent = {
+    permissions: group.roles!,
+    groupId: group._id as string,
+    audited: {
+      name: group.name,
+    },
   }
   await publishEvent(Event.USER_GROUP_PERMISSIONS_EDITED, properties)
 }

packages/backend-core/src/events/publishers/index.ts

@@ -21,3 +21,4 @@ export { default as group } from "./group"
 export { default as plugin } from "./plugin"
 export { default as backup } from "./backup"
 export { default as environmentVariable } from "./environmentVariable"
+export { default as auditLog } from "./auditLog"

packages/backend-core/src/events/publishers/screen.ts

@@ -11,6 +11,9 @@ async function created(screen: Screen, timestamp?: string | number) {
     layoutId: screen.layoutId,
     screenId: screen._id as string,
     roleId: screen.routing.roleId,
+    audited: {
+      name: screen.routing?.route,
+    },
   }
   await publishEvent(Event.SCREEN_CREATED, properties, timestamp)
 }
@@ -20,6 +23,9 @@ async function deleted(screen: Screen) {
     layoutId: screen.layoutId,
     screenId: screen._id as string,
     roleId: screen.routing.roleId,
+    audited: {
+      name: screen.routing?.route,
+    },
   }
   await publishEvent(Event.SCREEN_DELETED, properties)
 }

packages/backend-core/src/events/publishers/table.ts

@@ -13,6 +13,9 @@ import {
 async function created(table: Table, timestamp?: string | number) {
   const properties: TableCreatedEvent = {
     tableId: table._id as string,
+    audited: {
+      name: table.name,
+    },
   }
   await publishEvent(Event.TABLE_CREATED, properties, timestamp)
 }
@@ -20,6 +23,9 @@ async function created(table: Table, timestamp?: string | number) {
 async function updated(table: Table) {
   const properties: TableUpdatedEvent = {
     tableId: table._id as string,
+    audited: {
+      name: table.name,
+    },
   }
   await publishEvent(Event.TABLE_UPDATED, properties)
 }
@@ -27,6 +33,9 @@ async function updated(table: Table) {
 async function deleted(table: Table) {
   const properties: TableDeletedEvent = {
     tableId: table._id as string,
+    audited: {
+      name: table.name,
+    },
   }
   await publishEvent(Event.TABLE_DELETED, properties)
 }
@@ -35,6 +44,9 @@ async function exported(table: Table, format: TableExportFormat) {
   const properties: TableExportedEvent = {
     tableId: table._id as string,
     format,
+    audited: {
+      name: table.name,
+    },
   }
   await publishEvent(Event.TABLE_EXPORTED, properties)
 }
@@ -42,6 +54,9 @@ async function exported(table: Table, format: TableExportFormat) {
 async function imported(table: Table) {
   const properties: TableImportedEvent = {
     tableId: table._id as string,
+    audited: {
+      name: table.name,
+    },
   }
   await publishEvent(Event.TABLE_IMPORTED, properties)
 }

packages/backend-core/src/events/publishers/user.ts

@@ -19,6 +19,9 @@ import {
 async function created(user: User, timestamp?: number) {
   const properties: UserCreatedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_CREATED, properties, timestamp)
 }
@@ -26,6 +29,9 @@ async function created(user: User, timestamp?: number) {
 async function updated(user: User) {
   const properties: UserUpdatedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_UPDATED, properties)
 }
@@ -33,6 +39,9 @@ async function updated(user: User) {
 async function deleted(user: User) {
   const properties: UserDeletedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_DELETED, properties)
 }
@@ -40,6 +49,9 @@ async function deleted(user: User) {
 export async function onboardingComplete(user: User) {
   const properties: UserOnboardingEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_ONBOARDING_COMPLETE, properties)
 }
@@ -49,6 +61,9 @@ export async function onboardingComplete(user: User) {
 async function permissionAdminAssigned(user: User, timestamp?: number) {
   const properties: UserPermissionAssignedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(
     Event.USER_PERMISSION_ADMIN_ASSIGNED,
@@ -60,6 +75,9 @@ async function permissionAdminAssigned(user: User, timestamp?: number) {
 async function permissionAdminRemoved(user: User) {
   const properties: UserPermissionRemovedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_PERMISSION_ADMIN_REMOVED, properties)
 }
@@ -67,6 +85,9 @@ async function permissionAdminRemoved(user: User) {
 async function permissionBuilderAssigned(user: User, timestamp?: number) {
   const properties: UserPermissionAssignedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(
     Event.USER_PERMISSION_BUILDER_ASSIGNED,
@@ -78,20 +99,30 @@ async function permissionBuilderAssigned(user: User, timestamp?: number) {
 async function permissionBuilderRemoved(user: User) {
   const properties: UserPermissionRemovedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_PERMISSION_BUILDER_REMOVED, properties)
 }
 
 // INVITE
 
-async function invited() {
-  const properties: UserInvitedEvent = {}
+async function invited(email: string) {
+  const properties: UserInvitedEvent = {
+    audited: {
+      email,
+    },
+  }
   await publishEvent(Event.USER_INVITED, properties)
 }
 
 async function inviteAccepted(user: User) {
   const properties: UserInviteAcceptedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_INVITED_ACCEPTED, properties)
 }
@@ -101,6 +132,9 @@ async function inviteAccepted(user: User) {
 async function passwordForceReset(user: User) {
   const properties: UserPasswordForceResetEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_PASSWORD_FORCE_RESET, properties)
 }
@@ -108,6 +142,9 @@ async function passwordForceReset(user: User) {
 async function passwordUpdated(user: User) {
   const properties: UserPasswordUpdatedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_PASSWORD_UPDATED, properties)
 }
@@ -115,6 +152,9 @@ async function passwordUpdated(user: User) {
 async function passwordResetRequested(user: User) {
   const properties: UserPasswordResetRequestedEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_PASSWORD_RESET_REQUESTED, properties)
 }
@@ -122,6 +162,9 @@ async function passwordResetRequested(user: User) {
 async function passwordReset(user: User) {
   const properties: UserPasswordResetEvent = {
     userId: user._id as string,
+    audited: {
+      email: user.email,
+    },
   }
   await publishEvent(Event.USER_PASSWORD_RESET, properties)
 }

packages/backend-core/src/index.ts

@@ -1,3 +1,4 @@
+export * as configs from "./configs"
 export * as events from "./events"
 export * as migrations from "./migrations"
 export * as users from "./users"
@@ -20,11 +21,11 @@ export * as context from "./context"
 export * as cache from "./cache"
 export * as objectStore from "./objectStore"
 export * as redis from "./redis"
-export * as locks from "./redis/redlock"
+export * as locks from "./redis/redlockImpl"
 export * as utils from "./utils"
 export * as errors from "./errors"
 export { default as env } from "./environment"
-
+export { SearchParams } from "./db"
 // Add context to tenancy for backwards compatibility
 // only do this for external usages to prevent internal
 // circular dependencies

packages/backend-core/src/middleware/authenticated.ts

@@ -8,7 +8,7 @@ import { getGlobalDB, doInTenant } from "../context"
 import { decrypt } from "../security/encryption"
 import * as identity from "../context/identity"
 import env from "../environment"
-import { BBContext, EndpointMatcher } from "@budibase/types"
+import { Ctx, EndpointMatcher } from "@budibase/types"
 
 const ONE_MINUTE = env.SESSION_UPDATE_PERIOD
   ? parseInt(env.SESSION_UPDATE_PERIOD)
@@ -73,7 +73,7 @@ export default function (
   }
 ) {
   const noAuthOptions = noAuthPatterns ? buildMatcherRegex(noAuthPatterns) : []
-  return async (ctx: BBContext | any, next: any) => {
+  return async (ctx: Ctx | any, next: any) => {
     let publicEndpoint = false
     const version = ctx.request.headers[Header.API_VER]
     // the path is not authenticated
@@ -115,7 +115,8 @@ export default function (
           authenticated = true
         } catch (err: any) {
           authenticated = false
-          console.error("Auth Error", err?.message || err)
+          console.error(`Auth Error: ${err.message}`)
+          console.error(err)
           // remove the cookie as the user does not exist anymore
           clearCookie(ctx, Cookie.Auth)
         }
@@ -148,12 +149,13 @@ export default function (
       finalise(ctx, { authenticated, user, internal, version, publicEndpoint })
 
       if (user && user.email) {
-        return identity.doInUserContext(user, next)
+        return identity.doInUserContext(user, ctx, next)
       } else {
         return next()
       }
     } catch (err: any) {
-      console.error("Auth Error", err?.message || err)
+      console.error(`Auth Error: ${err.message}`)
+      console.error(err)
       // invalid token, clear the cookie
       if (err && err.name === "JsonWebTokenError") {
         clearCookie(ctx, Cookie.Auth)

packages/backend-core/src/middleware/errorHandling.ts

@@ -11,6 +11,7 @@ export async function errorHandling(ctx: any, next: any) {
 
     if (status > 499 || env.ENABLE_4XX_HTTP_LOGGING) {
       ctx.log.error(err)
+      console.trace(err)
     }
 
     const error = errors.getPublicError(err)

packages/backend-core/src/middleware/index.ts

@@ -17,4 +17,5 @@ export { default as builderOrAdmin } from "./builderOrAdmin"
 export { default as builderOnly } from "./builderOnly"
 export { default as logging } from "./logging"
 export { default as errorHandling } from "./errorHandling"
+export { default as querystringToBody } from "./querystringToBody"
 export * as joiValidator from "./joi-validator"

packages/backend-core/src/middleware/passport/datasource/google.ts

@@ -1,9 +1,8 @@
 import * as google from "../sso/google"
-import { Cookie, Config } from "../../../constants"
+import { Cookie } from "../../../constants"
 import { clearCookie, getCookie } from "../../../utils"
-import { getScopedConfig, getPlatformUrl, doWithDB } from "../../../db"
-import environment from "../../../environment"
-import { getGlobalDB } from "../../../context"
+import { doWithDB } from "../../../db"
+import * as configs from "../../../configs"
 import { BBContext, Database, SSOProfile } from "@budibase/types"
 import { ssoSaveUserNoOp } from "../sso/sso"
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
@@ -13,18 +12,12 @@ type Passport = {
 }
 
 async function fetchGoogleCreds() {
-  // try and get the config from the tenant
-  const db = getGlobalDB()
-  const googleConfig = await getScopedConfig(db, {
-    type: Config.GOOGLE,
-  })
-  // or fall back to env variables
-  return (
-    googleConfig || {
-      clientID: environment.GOOGLE_CLIENT_ID,
-      clientSecret: environment.GOOGLE_CLIENT_SECRET,
-    }
-  )
+  let config = await configs.getGoogleDatasourceConfig()
+
+  if (!config) {
+    throw new Error("No google configuration found")
+  }
+  return config
 }
 
 export async function preAuth(
@@ -34,7 +27,7 @@ export async function preAuth(
 ) {
   // get the relevant config
   const googleConfig = await fetchGoogleCreds()
-  const platformUrl = await getPlatformUrl({ tenantAware: false })
+  const platformUrl = await configs.getPlatformUrl({ tenantAware: false })
 
   let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback`
   const strategy = await google.strategyFactory(
@@ -61,7 +54,7 @@ export async function postAuth(
 ) {
   // get the relevant config
   const config = await fetchGoogleCreds()
-  const platformUrl = await getPlatformUrl({ tenantAware: false })
+  const platformUrl = await configs.getPlatformUrl({ tenantAware: false })
 
   let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback`
   const authStateCookie = getCookie(ctx, Cookie.DatasourceAuth)

packages/backend-core/src/middleware/passport/sso/google.ts

@@ -2,12 +2,11 @@ import { ssoCallbackUrl } from "../utils"
 import * as sso from "./sso"
 import {
   ConfigType,
-  GoogleConfig,
-  Database,
   SSOProfile,
   SSOAuthDetails,
   SSOProviderType,
   SaveSSOUserFunction,
+  GoogleInnerConfig,
 } from "@budibase/types"
 const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
 
@@ -45,7 +44,7 @@ export function buildVerifyFn(saveUserFn: SaveSSOUserFunction) {
  * @returns Dynamically configured Passport Google Strategy
  */
 export async function strategyFactory(
-  config: GoogleConfig["config"],
+  config: GoogleInnerConfig,
   callbackUrl: string,
   saveUserFn: SaveSSOUserFunction
 ) {
@@ -73,9 +72,6 @@ export async function strategyFactory(
   }
 }
 
-export async function getCallbackUrl(
-  db: Database,
-  config: { callbackURL?: string }
-) {
-  return ssoCallbackUrl(db, config, ConfigType.GOOGLE)
+export async function getCallbackUrl(config: GoogleInnerConfig) {
+  return ssoCallbackUrl(ConfigType.GOOGLE, config)
 }

packages/backend-core/src/middleware/passport/sso/oidc.ts

@@ -4,7 +4,6 @@ import { ssoCallbackUrl } from "../utils"
 import {
   ConfigType,
   OIDCInnerConfig,
-  Database,
   SSOProfile,
   OIDCStrategyConfiguration,
   SSOAuthDetails,
@@ -157,9 +156,6 @@ export async function fetchStrategyConfig(
   }
 }
 
-export async function getCallbackUrl(
-  db: Database,
-  config: { callbackURL?: string }
-) {
-  return ssoCallbackUrl(db, config, ConfigType.OIDC)
+export async function getCallbackUrl() {
+  return ssoCallbackUrl(ConfigType.OIDC)
 }

packages/backend-core/src/middleware/passport/utils.ts

@@ -1,6 +1,6 @@
-import { isMultiTenant, getTenantId } from "../../context"
-import { getScopedConfig } from "../../db"
-import { ConfigType, Database } from "@budibase/types"
+import { getTenantId, isMultiTenant } from "../../context"
+import * as configs from "../../configs"
+import { ConfigType, GoogleInnerConfig } from "@budibase/types"
 
 /**
  * Utility to handle authentication errors.
@@ -19,17 +19,14 @@ export function authError(done: Function, message: string, err?: any) {
 }
 
 export async function ssoCallbackUrl(
-  db: Database,
-  config?: { callbackURL?: string },
-  type?: ConfigType
+  type: ConfigType,
+  config?: GoogleInnerConfig
 ) {
   // incase there is a callback URL from before
-  if (config && config.callbackURL) {
-    return config.callbackURL
+  if (config && (config as GoogleInnerConfig).callbackURL) {
+    return (config as GoogleInnerConfig).callbackURL as string
   }
-  const publicConfig = await getScopedConfig(db, {
-    type: ConfigType.SETTINGS,
-  })
+  const settingsConfig = await configs.getSettingsConfig()
 
   let callbackUrl = `/api/global/auth`
   if (isMultiTenant()) {
@@ -37,5 +34,5 @@ export async function ssoCallbackUrl(
   }
   callbackUrl += `/${type}/callback`
 
-  return `${publicConfig.platformUrl}${callbackUrl}`
+  return `${settingsConfig.platformUrl}${callbackUrl}`
 }

packages/backend-core/src/middleware/querystringToBody.ts

@@ -0,0 +1,28 @@
+import { Ctx } from "@budibase/types"
+
+/**
+ * Expects a standard "query" query string property which is the JSON body
+ * of the request, which has to be sent via query string due to the requirement
+ * of making an endpoint a GET request e.g. downloading a file stream.
+ */
+export default function (ctx: Ctx, next: any) {
+  const queryString = ctx.request.query?.query as string | undefined
+  if (ctx.request.method.toLowerCase() !== "get") {
+    ctx.throw(
+      500,
+      "Query to download middleware can only be used for get requests."
+    )
+  }
+  if (!queryString) {
+    return next()
+  }
+  const decoded = decodeURIComponent(queryString)
+  let json
+  try {
+    json = JSON.parse(decoded)
+  } catch (err) {
+    return next()
+  }
+  ctx.request.body = json
+  return next()
+}

packages/backend-core/src/migrations/migrations.ts

@@ -87,6 +87,7 @@ export const runMigration = async (
     const lengthStatement = length > 1 ? `[${count}/${length}]` : ""
 
     const db = getDB(dbName)
+
     try {
       const doc = await getMigrationsDoc(db)
 

packages/backend-core/src/platform/tenants.ts

@@ -1,7 +1,7 @@
 import { StaticDatabases } from "../constants"
 import { getPlatformDB } from "./platformDb"
 import { LockName, LockOptions, LockType, Tenants } from "@budibase/types"
-import * as locks from "../redis/redlock"
+import * as locks from "../redis/redlockImpl"
 
 const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
 

packages/backend-core/src/queue/constants.ts

@@ -1,4 +1,5 @@
 export enum JobQueue {
   AUTOMATION = "automationQueue",
   APP_BACKUP = "appBackupQueue",
+  AUDIT_LOG = "auditLogQueue",
 }

packages/backend-core/src/queue/queue.ts

@@ -40,8 +40,10 @@ export function createQueue<T>(
 }
 
 export async function shutdown() {
-  if (QUEUES.length) {
+  if (cleanupInterval) {
     clearInterval(cleanupInterval)
+  }
+  if (QUEUES.length) {
     for (let queue of QUEUES) {
       await queue.close()
     }

packages/backend-core/src/redis/index.ts

@@ -3,4 +3,4 @@
 export { default as Client } from "./redis"
 export * as utils from "./utils"
 export * as clients from "./init"
-export * as locks from "./redlock"
+export * as locks from "./redlockImpl"

packages/backend-core/src/users.ts

@@ -5,19 +5,44 @@ import {
   generateAppUserID,
   queryGlobalView,
   UNICODE_MAX,
+  directCouchFind,
 } from "./db"
 import { BulkDocsResponse, User } from "@budibase/types"
 import { getGlobalDB } from "./context"
 import * as context from "./context"
 
-export const bulkGetGlobalUsersById = async (userIds: string[]) => {
+type GetOpts = { cleanup?: boolean }
+
+function removeUserPassword(users: User | User[]) {
+  if (Array.isArray(users)) {
+    return users.map(user => {
+      if (user) {
+        delete user.password
+        return user
+      }
+    })
+  } else if (users) {
+    delete users.password
+    return users
+  }
+  return users
+}
+
+export const bulkGetGlobalUsersById = async (
+  userIds: string[],
+  opts?: GetOpts
+) => {
   const db = getGlobalDB()
-  return (
+  let users = (
     await db.allDocs({
       keys: userIds,
       include_docs: true,
     })
   ).rows.map(row => row.doc) as User[]
+  if (opts?.cleanup) {
+    users = removeUserPassword(users) as User[]
+  }
+  return users
 }
 
 export const bulkUpdateGlobalUsers = async (users: User[]) => {
@@ -25,18 +50,22 @@ export const bulkUpdateGlobalUsers = async (users: User[]) => {
   return (await db.bulkDocs(users)) as BulkDocsResponse
 }
 
-export async function getById(id: string): Promise<User> {
+export async function getById(id: string, opts?: GetOpts): Promise<User> {
   const db = context.getGlobalDB()
-  return db.get(id)
+  let user = await db.get(id)
+  if (opts?.cleanup) {
+    user = removeUserPassword(user)
+  }
+  return user
 }
 
 /**
  * Given an email address this will use a view to search through
  * all the users to find one with this email address.
- * @param {string} email the email to lookup the user by.
  */
 export const getGlobalUserByEmail = async (
-  email: String
+  email: String,
+  opts?: GetOpts
 ): Promise<User | undefined> => {
   if (email == null) {
     throw "Must supply an email address to view"
@@ -52,10 +81,19 @@ export const getGlobalUserByEmail = async (
     throw new Error(`Multiple users found with email address: ${email}`)
   }
 
-  return response
+  let user = response as User
+  if (opts?.cleanup) {
+    user = removeUserPassword(user) as User
+  }
+
+  return user
 }
 
-export const searchGlobalUsersByApp = async (appId: any, opts: any) => {
+export const searchGlobalUsersByApp = async (
+  appId: any,
+  opts: any,
+  getOpts?: GetOpts
+) => {
   if (typeof appId !== "string") {
     throw new Error("Must provide a string based app ID")
   }
@@ -64,10 +102,54 @@ export const searchGlobalUsersByApp = async (appId: any, opts: any) => {
   })
   params.startkey = opts && opts.startkey ? opts.startkey : params.startkey
   let response = await queryGlobalView(ViewName.USER_BY_APP, params)
+
   if (!response) {
     response = []
   }
-  return Array.isArray(response) ? response : [response]
+  let users: User[] = Array.isArray(response) ? response : [response]
+  if (getOpts?.cleanup) {
+    users = removeUserPassword(users) as User[]
+  }
+  return users
+}
+
+/*
+  Return any user who potentially has access to the application
+  Admins, developers and app users with the explicitly role.
+*/
+export const searchGlobalUsersByAppAccess = async (appId: any, opts: any) => {
+  const roleSelector = `roles.${appId}`
+
+  let orQuery: any[] = [
+    {
+      "builder.global": true,
+    },
+    {
+      "admin.global": true,
+    },
+  ]
+
+  if (appId) {
+    const roleCheck = {
+      [roleSelector]: {
+        $exists: true,
+      },
+    }
+    orQuery.push(roleCheck)
+  }
+
+  let searchOptions = {
+    selector: {
+      $or: orQuery,
+      _id: {
+        $regex: "^us_",
+      },
+    },
+    limit: opts?.limit || 50,
+  }
+
+  const resp = await directCouchFind(context.getGlobalDBName(), searchOptions)
+  return resp?.rows
 }
 
 export const getGlobalUserByAppPage = (appId: string, user: User) => {
@@ -80,7 +162,11 @@ export const getGlobalUserByAppPage = (appId: string, user: User) => {
 /**
  * Performs a starts with search on the global email view.
  */
-export const searchGlobalUsersByEmail = async (email: string, opts: any) => {
+export const searchGlobalUsersByEmail = async (
+  email: string,
+  opts: any,
+  getOpts?: GetOpts
+) => {
   if (typeof email !== "string") {
     throw new Error("Must provide a string to search by")
   }
@@ -95,5 +181,9 @@ export const searchGlobalUsersByEmail = async (email: string, opts: any) => {
   if (!response) {
     response = []
   }
-  return Array.isArray(response) ? response : [response]
+  let users: User[] = Array.isArray(response) ? response : [response]
+  if (getOpts?.cleanup) {
+    users = removeUserPassword(users) as User[]
+  }
+  return users
 }

packages/backend-core/src/utils/utils.ts

@@ -10,7 +10,13 @@ import {
 import env from "../environment"
 import * as tenancy from "../tenancy"
 import * as context from "../context"
-import { App, Ctx, TenantResolutionStrategy } from "@budibase/types"
+import {
+  App,
+  AuditedEventFriendlyName,
+  Ctx,
+  Event,
+  TenantResolutionStrategy,
+} from "@budibase/types"
 import { SetOption } from "cookies"
 const jwt = require("jsonwebtoken")
 
@@ -217,3 +223,7 @@ export async function getBuildersCount() {
 export function timeout(timeMs: number) {
   return new Promise(resolve => setTimeout(resolve, timeMs))
 }
+
+export function isAudited(event: Event) {
+  return !!AuditedEventFriendlyName[event]
+}

packages/backend-core/tests/utilities/mocks/licenses.ts

@@ -70,6 +70,10 @@ export const useBackups = () => {
   return useFeature(Feature.APP_BACKUPS)
 }
 
+export const useEnforceableSSO = () => {
+  return useFeature(Feature.ENFORCEABLE_SSO)
+}
+
 export const useGroups = () => {
   return useFeature(Feature.USER_GROUPS)
 }
@@ -78,6 +82,10 @@ export const useEnvironmentVariables = () => {
   return useFeature(Feature.ENVIRONMENT_VARIABLES)
 }
 
+export const useAuditLogs = () => {
+  return useFeature(Feature.AUDIT_LOGS)
+}
+
 // QUOTAS
 
 export const setAutomationLogsQuota = (value: number) => {

packages/backend-core/tests/utilities/structures/accounts.ts

@@ -8,6 +8,8 @@ import {
   CloudAccount,
   Hosting,
   SSOAccount,
+  CreateAccount,
+  CreatePassswordAccount,
 } from "@budibase/types"
 import _ from "lodash"
 
@@ -29,6 +31,10 @@ export const account = (): Account => {
   }
 }
 
+export function selfHostAccount() {
+  return account()
+}
+
 export const cloudAccount = (): CloudAccount => {
   return {
     ...account(),
@@ -47,9 +53,9 @@ function provider(): AccountSSOProvider {
   return _.sample(Object.values(AccountSSOProvider)) as AccountSSOProvider
 }
 
-export function ssoAccount(): SSOAccount {
+export function ssoAccount(account: Account = cloudAccount()): SSOAccount {
   return {
-    ...cloudAccount(),
+    ...account,
     authType: AuthType.SSO,
     oauth2: {
       accessToken: generator.string(),
@@ -61,3 +67,49 @@ export function ssoAccount(): SSOAccount {
     thirdPartyProfile: {},
   }
 }
+
+export const cloudCreateAccount: CreatePassswordAccount = {
+  email: "cloud@budibase.com",
+  tenantId: "cloud",
+  hosting: Hosting.CLOUD,
+  authType: AuthType.PASSWORD,
+  password: "Password123!",
+  tenantName: "cloud",
+  name: "Budi Armstrong",
+  size: "10+",
+  profession: "Software Engineer",
+}
+
+export const cloudSSOCreateAccount: CreateAccount = {
+  email: "cloud-sso@budibase.com",
+  tenantId: "cloud-sso",
+  hosting: Hosting.CLOUD,
+  authType: AuthType.SSO,
+  tenantName: "cloudsso",
+  name: "Budi Armstrong",
+  size: "10+",
+  profession: "Software Engineer",
+}
+
+export const selfCreateAccount: CreatePassswordAccount = {
+  email: "self@budibase.com",
+  tenantId: "self",
+  hosting: Hosting.SELF,
+  authType: AuthType.PASSWORD,
+  password: "Password123!",
+  tenantName: "self",
+  name: "Budi Armstrong",
+  size: "10+",
+  profession: "Software Engineer",
+}
+
+export const selfSSOCreateAccount: CreateAccount = {
+  email: "self-sso@budibase.com",
+  tenantId: "self-sso",
+  hosting: Hosting.SELF,
+  authType: AuthType.SSO,
+  tenantName: "selfsso",
+  name: "Budi Armstrong",
+  size: "10+",
+  profession: "Software Engineer",
+}

packages/backend-core/tests/utilities/structures/generator.ts

@@ -0,0 +1,2 @@
+import Chance from "chance"
+export const generator = new Chance()

packages/backend-core/tests/utilities/structures/index.ts

@@ -1,8 +1,4 @@
 export * from "./common"
-
-import Chance from "chance"
-export const generator = new Chance()
-
 export * as accounts from "./accounts"
 export * as apps from "./apps"
 export * as db from "./db"
@@ -12,3 +8,4 @@ export * as plugins from "./plugins"
 export * as sso from "./sso"
 export * as tenant from "./tenants"
 export * as users from "./users"
+export { generator } from "./generator"

packages/backend-core/tests/utilities/structures/shared.ts

@@ -0,0 +1,19 @@
+import { User } from "@budibase/types"
+import { generator } from "./generator"
+import { uuid } from "./common"
+
+export const newEmail = () => {
+  return `${uuid()}@test.com`
+}
+
+export const user = (userProps?: any): User => {
+  return {
+    email: newEmail(),
+    password: "test",
+    roles: { app_test: "admin" },
+    firstName: generator.first(),
+    lastName: generator.last(),
+    pictureUrl: "http://test.com",
+    ...userProps,
+  }
+}

packages/backend-core/tests/utilities/structures/sso.ts

@@ -1,6 +1,7 @@
 import {
   GoogleInnerConfig,
   JwtClaims,
+  OAuth2,
   OIDCInnerConfig,
   OIDCWellKnownConfig,
   SSOAuthDetails,
@@ -8,8 +9,40 @@ import {
   SSOProviderType,
   User,
 } from "@budibase/types"
-import { uuid, generator, users, email } from "./index"
+import { generator } from "./generator"
+import { uuid, email } from "./common"
+import * as shared from "./shared"
 import _ from "lodash"
+import { user } from "./shared"
+
+export function OAuth(): OAuth2 {
+  return {
+    refreshToken: generator.string(),
+    accessToken: generator.string(),
+  }
+}
+
+export function authDetails(userDoc?: User): SSOAuthDetails {
+  if (!userDoc) {
+    userDoc = user()
+  }
+
+  const userId = userDoc._id || uuid()
+  const provider = generator.string()
+
+  const profile = ssoProfile(userDoc)
+  profile.provider = provider
+  profile.id = userId
+
+  return {
+    email: userDoc.email,
+    oauth2: OAuth(),
+    profile,
+    provider,
+    providerType: providerType(),
+    userId,
+  }
+}
 
 export function providerType(): SSOProviderType {
   return _.sample(Object.values(SSOProviderType)) as SSOProviderType
@@ -17,7 +50,7 @@ export function providerType(): SSOProviderType {
 
 export function ssoProfile(user?: User): SSOProfile {
   if (!user) {
-    user = users.user()
+    user = shared.user()
   }
   return {
     id: user._id!,
@@ -33,31 +66,6 @@ export function ssoProfile(user?: User): SSOProfile {
   }
 }
 
-export function authDetails(user?: User): SSOAuthDetails {
-  if (!user) {
-    user = users.user()
-  }
-
-  const userId = user._id || uuid()
-  const provider = generator.string()
-
-  const profile = ssoProfile(user)
-  profile.provider = provider
-  profile.id = userId
-
-  return {
-    email: user.email,
-    oauth2: {
-      refreshToken: generator.string(),
-      accessToken: generator.string(),
-    },
-    profile,
-    provider,
-    providerType: providerType(),
-    userId,
-  }
-}
-
 // OIDC
 
 export function oidcConfig(): OIDCInnerConfig {
@@ -69,6 +77,7 @@ export function oidcConfig(): OIDCInnerConfig {
     configUrl: "http://someconfigurl",
     clientID: generator.string(),
     clientSecret: generator.string(),
+    scopes: [],
   }
 }
 

packages/backend-core/tests/utilities/structures/users.ts

@@ -1,29 +1,13 @@
-import { generator } from "../"
 import {
   AdminUser,
   BuilderUser,
   SSOAuthDetails,
   SSOUser,
-  User,
 } from "@budibase/types"
-import { v4 as uuid } from "uuid"
-import * as sso from "./sso"
+import { user } from "./shared"
+import { authDetails } from "./sso"
 
-export const newEmail = () => {
-  return `${uuid()}@test.com`
-}
-
-export const user = (userProps?: any): User => {
-  return {
-    email: newEmail(),
-    password: "test",
-    roles: { app_test: "admin" },
-    firstName: generator.first(),
-    lastName: generator.last(),
-    pictureUrl: "http://test.com",
-    ...userProps,
-  }
-}
+export { user, newEmail } from "./shared"
 
 export const adminUser = (userProps?: any): AdminUser => {
   return {
@@ -53,7 +37,7 @@ export function ssoUser(
   delete base.password
 
   if (!opts.details) {
-    opts.details = sso.authDetails(base)
+    opts.details = authDetails(base)
   }
 
   return {

packages/backend-core/tests/utilities/testContainerUtils.ts

@@ -1,3 +1,31 @@
+import { execSync } from "child_process"
+
+let dockerPsResult: string | undefined
+
+function formatDockerPsResult(serverName: string, port: number) {
+  const lines = dockerPsResult?.split("\n")
+  let first = true
+  if (!lines) {
+    return null
+  }
+  for (let line of lines) {
+    if (first) {
+      first = false
+      continue
+    }
+    let toLookFor = serverName.split("-service")[0]
+    if (!line.includes(toLookFor)) {
+      continue
+    }
+    const regex = new RegExp(`0.0.0.0:([0-9]*)->${port}`, "g")
+    const found = line.match(regex)
+    if (found) {
+      return found[0].split(":")[1].split("->")[0]
+    }
+  }
+  return null
+}
+
 function getTestContainerSettings(
   serverName: string,
   key: string
@@ -14,10 +42,22 @@ function getTestContainerSettings(
 }
 
 function getContainerInfo(containerName: string, port: number) {
-  const assignedPort = getTestContainerSettings(
+  let assignedPort = getTestContainerSettings(
     containerName.toUpperCase(),
     `PORT_${port}`
   )
+  if (!dockerPsResult) {
+    try {
+      const outputBuffer = execSync("docker ps")
+      dockerPsResult = outputBuffer.toString("utf8")
+    } catch (err) {
+      //no-op
+    }
+  }
+  const possiblePort = formatDockerPsResult(containerName, port)
+  if (possiblePort) {
+    assignedPort = possiblePort
+  }
   const host = getTestContainerSettings(containerName.toUpperCase(), "IP")
   return {
     port: assignedPort,
@@ -39,12 +79,15 @@ function getRedisConfig() {
 }
 
 export function setupEnv(...envs: any[]) {
+  const couch = getCouchConfig(),
+    minio = getCouchConfig(),
+    redis = getRedisConfig()
   const configs = [
-    { key: "COUCH_DB_PORT", value: getCouchConfig().port },
-    { key: "COUCH_DB_URL", value: getCouchConfig().url },
-    { key: "MINIO_PORT", value: getMinioConfig().port },
-    { key: "MINIO_URL", value: getMinioConfig().url },
-    { key: "REDIS_URL", value: getRedisConfig().url },
+    { key: "COUCH_DB_PORT", value: couch.port },
+    { key: "COUCH_DB_URL", value: couch.url },
+    { key: "MINIO_PORT", value: minio.port },
+    { key: "MINIO_URL", value: minio.url },
+    { key: "REDIS_URL", value: redis.url },
   ]
 
   for (const config of configs.filter(x => !!x.value)) {

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.3.18-alpha.7",
+  "version": "2.3.18-alpha.29",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/string-templates": "2.3.18-alpha.7",
+    "@budibase/string-templates": "2.3.18-alpha.29",
     "@spectrum-css/accordion": "3.0.24",
     "@spectrum-css/actionbutton": "1.0.1",
     "@spectrum-css/actiongroup": "1.0.1",

packages/bbui/src/ActionButton/ActionButton.svelte

@@ -1,6 +1,9 @@
 <script>
   import "@spectrum-css/actionbutton/dist/index-vars.css"
   import { createEventDispatcher } from "svelte"
+  import Tooltip from "../Tooltip/Tooltip.svelte"
+  import { fade } from "svelte/transition"
+
   const dispatch = createEventDispatcher()
 
   export let quiet = false
@@ -13,6 +16,9 @@
   export let active = false
   export let fullWidth = false
   export let noPadding = false
+  export let tooltip = ""
+
+  let showTooltip = false
 
   function longPress(element) {
     if (!longPressable) return
@@ -35,42 +41,54 @@
   }
 </script>
 
-<button
-  use:longPress
-  class:spectrum-ActionButton--quiet={quiet}
-  class:spectrum-ActionButton--emphasized={emphasized}
-  class:is-selected={selected}
-  class:noPadding
-  class:fullWidth
-  class="spectrum-ActionButton spectrum-ActionButton--size{size}"
-  class:active
-  {disabled}
-  on:longPress
-  on:click|preventDefault
+<span
+  class="btn-wrap"
+  on:mouseover={() => (showTooltip = true)}
+  on:mouseleave={() => (showTooltip = false)}
+  on:focus={() => (showTooltip = true)}
 >
-  {#if longPressable}
-    <svg
-      class="spectrum-Icon spectrum-UIIcon-CornerTriangle100 spectrum-ActionButton-hold"
-      focusable="false"
-      aria-hidden="true"
-    >
-      <use xlink:href="#spectrum-css-icon-CornerTriangle100" />
-    </svg>
-  {/if}
-  {#if icon}
-    <svg
-      class="spectrum-Icon spectrum-Icon--size{size}"
-      focusable="false"
-      aria-hidden="true"
-      aria-label={icon}
-    >
-      <use xlink:href="#spectrum-icon-18-{icon}" />
-    </svg>
-  {/if}
-  {#if $$slots}
-    <span class="spectrum-ActionButton-label"><slot /></span>
-  {/if}
-</button>
+  <button
+    use:longPress
+    class:spectrum-ActionButton--quiet={quiet}
+    class:spectrum-ActionButton--emphasized={emphasized}
+    class:is-selected={selected}
+    class:noPadding
+    class:fullWidth
+    class="spectrum-ActionButton spectrum-ActionButton--size{size}"
+    class:active
+    {disabled}
+    on:longPress
+    on:click|preventDefault
+  >
+    {#if longPressable}
+      <svg
+        class="spectrum-Icon spectrum-UIIcon-CornerTriangle100 spectrum-ActionButton-hold"
+        focusable="false"
+        aria-hidden="true"
+      >
+        <use xlink:href="#spectrum-css-icon-CornerTriangle100" />
+      </svg>
+    {/if}
+    {#if icon}
+      <svg
+        class="spectrum-Icon spectrum-Icon--size{size}"
+        focusable="false"
+        aria-hidden="true"
+        aria-label={icon}
+      >
+        <use xlink:href="#spectrum-icon-18-{icon}" />
+      </svg>
+    {/if}
+    {#if $$slots}
+      <span class="spectrum-ActionButton-label"><slot /></span>
+    {/if}
+    {#if tooltip && showTooltip}
+      <div class="tooltip" in:fade={{ duration: 130, delay: 250 }}>
+        <Tooltip textWrapping direction="bottom" text={tooltip} />
+      </div>
+    {/if}
+  </button>
+</span>
 
 <style>
   .fullWidth {
@@ -98,4 +116,14 @@
   .is-selected:not(.emphasized) .spectrum-Icon {
     color: var(--spectrum-global-color-gray-900);
   }
+  .tooltip {
+    position: absolute;
+    pointer-events: none;
+    left: 50%;
+    top: calc(100% + 4px);
+    width: 100vw;
+    max-width: 150px;
+    transform: translateX(-50%);
+    text-align: center;
+  }
 </style>

packages/bbui/src/Form/Core/Multiselect.svelte

@@ -14,6 +14,9 @@
   export let autocomplete = false
   export let sort = false
   export let autoWidth = false
+  export let fetchTerm = null
+  export let useFetch = false
+  export let customPopoverHeight
 
   const dispatch = createEventDispatcher()
 
@@ -83,10 +86,13 @@
   {options}
   isPlaceholder={!value?.length}
   {autocomplete}
+  bind:fetchTerm
+  {useFetch}
   {isOptionSelected}
   {getOptionLabel}
   {getOptionValue}
   onSelectOption={toggleOption}
   {sort}
   {autoWidth}
+  {customPopoverHeight}
 />

packages/bbui/src/Form/Core/Picker.svelte

@@ -24,6 +24,7 @@
   export let getOptionLabel = option => option
   export let getOptionValue = option => option
   export let getOptionIcon = () => null
+  export let useOptionIconImage = false
   export let getOptionColour = () => null
   export let open = false
   export let readonly = false
@@ -31,6 +32,11 @@
   export let autoWidth = false
   export let autocomplete = false
   export let sort = false
+  export let fetchTerm = null
+  export let useFetch = false
+  export let customPopoverHeight
+  export let align = "left"
+  export let footer = null
 
   const dispatch = createEventDispatcher()
 
@@ -71,7 +77,7 @@
   }
 
   const getFilteredOptions = (options, term, getLabel) => {
-    if (autocomplete && term) {
+    if (autocomplete && term && !fetchTerm) {
       const lowerCaseTerm = term.toLowerCase()
       return options.filter(option => {
         return `${getLabel(option)}`.toLowerCase().includes(lowerCaseTerm)
@@ -130,12 +136,13 @@
 
 <Popover
   anchor={button}
-  align="left"
+  align={align || "left"}
   bind:this={popover}
   {open}
   on:close={() => (open = false)}
   useAnchorWidth={!autoWidth}
   maxWidth={autoWidth ? 400 : null}
+  customHeight={customPopoverHeight}
 >
   <div
     class="popover-content"
@@ -144,8 +151,9 @@
   >
     {#if autocomplete}
       <Search
-        value={searchTerm}
-        on:change={event => (searchTerm = event.detail)}
+        value={useFetch ? fetchTerm : searchTerm}
+        on:change={event =>
+          useFetch ? (fetchTerm = event.detail) : (searchTerm = event.detail)}
         {disabled}
         placeholder="Search"
       />
@@ -183,7 +191,16 @@
           >
             {#if getOptionIcon(option, idx)}
               <span class="option-extra icon">
-                <Icon size="S" name={getOptionIcon(option, idx)} />
+                {#if useOptionIconImage}
+                  <img
+                    src={getOptionIcon(option, idx)}
+                    alt="icon"
+                    width="15"
+                    height="15"
+                  />
+                {:else}
+                  <Icon size="S" name={getOptionIcon(option, idx)} />
+                {/if}
               </span>
             {/if}
             {#if getOptionColour(option, idx)}
@@ -205,6 +222,12 @@
         {/each}
       {/if}
     </ul>
+
+    {#if footer}
+      <div class="footer">
+        {footer}
+      </div>
+    {/if}
   </div>
 </Popover>
 
@@ -247,7 +270,7 @@
   }
   .popover-content.auto-width .spectrum-Menu-itemLabel {
     white-space: nowrap;
-    overflow: hidden;
+    overflow: none;
     text-overflow: ellipsis;
   }
   .popover-content:not(.auto-width) .spectrum-Menu-itemLabel {
@@ -281,4 +304,11 @@
   .popover-content :global(.spectrum-Search .spectrum-Textfield-icon) {
     top: 9px;
   }
+
+  .footer {
+    padding: 4px 12px 12px 12px;
+    font-style: italic;
+    max-width: 170px;
+    font-size: 12px;
+  }
 </style>

packages/bbui/src/Form/Core/Select.svelte

@@ -11,6 +11,7 @@
   export let getOptionLabel = option => option
   export let getOptionValue = option => option
   export let getOptionIcon = () => null
+  export let useOptionIconImage = false
   export let getOptionColour = () => null
   export let isOptionEnabled
   export let readonly = false
@@ -18,6 +19,8 @@
   export let autoWidth = false
   export let autocomplete = false
   export let sort = false
+  export let align
+  export let footer = null
 
   const dispatch = createEventDispatcher()
 
@@ -41,7 +44,7 @@
   const getFieldText = (value, options, placeholder) => {
     // Always use placeholder if no value
     if (value == null || value === "") {
-      return placeholder || "Choose an option"
+      return placeholder !== false ? "Choose an option" : ""
     }
 
     return getFieldAttribute(getOptionLabel, value, options)
@@ -66,15 +69,18 @@
   {fieldColour}
   {options}
   {autoWidth}
+  {align}
+  {footer}
   {getOptionLabel}
   {getOptionValue}
   {getOptionIcon}
+  {useOptionIconImage}
   {getOptionColour}
   {isOptionEnabled}
   {autocomplete}
   {sort}
   isPlaceholder={value == null || value === ""}
-  placeholderOption={placeholder}
+  placeholderOption={placeholder === false ? null : placeholder}
   isOptionSelected={option => option === value}
   onSelectOption={selectOption}
 />

packages/bbui/src/Form/Multiselect.svelte

@@ -15,6 +15,11 @@
   export let getOptionValue = option => option
   export let sort = false
   export let autoWidth = false
+  export let autocomplete = false
+  export let fetchTerm = null
+  export let useFetch = false
+  export let customPopoverHeight
+
   const dispatch = createEventDispatcher()
   const onChange = e => {
     value = e.detail
@@ -34,6 +39,10 @@
     {getOptionLabel}
     {getOptionValue}
     {autoWidth}
+    {autocomplete}
+    {customPopoverHeight}
+    bind:fetchTerm
+    {useFetch}
     on:change={onChange}
     on:click
   />

packages/bbui/src/Form/Select.svelte

@@ -14,12 +14,17 @@
   export let getOptionLabel = option => extractProperty(option, "label")
   export let getOptionValue = option => extractProperty(option, "value")
   export let getOptionIcon = option => option?.icon
+  export let useOptionIconImage = false
   export let getOptionColour = option => option?.colour
   export let isOptionEnabled
   export let quiet = false
   export let autoWidth = false
   export let sort = false
   export let tooltip = ""
+  export let autocomplete = false
+  export let customPopoverHeight
+  export let align
+  export let footer = null
 
   const dispatch = createEventDispatcher()
   const onChange = e => {
@@ -46,11 +51,16 @@
     {placeholder}
     {autoWidth}
     {sort}
+    {align}
+    {footer}
     {getOptionLabel}
     {getOptionValue}
     {getOptionIcon}
     {getOptionColour}
+    {useOptionIconImage}
     {isOptionEnabled}
+    {autocomplete}
+    {customPopoverHeight}
     on:change={onChange}
     on:click
   />

packages/bbui/src/Icon/Icon.svelte

@@ -67,6 +67,9 @@
     color: var(--spectrum-alias-icon-color-selected-hover) !important;
     cursor: pointer;
   }
+  svg.hoverable:active {
+    color: var(--spectrum-global-color-blue-400) !important;
+  }
 
   svg.disabled {
     color: var(--spectrum-global-color-gray-500) !important;

packages/bbui/src/InlineAlert/InlineAlert.svelte

@@ -57,5 +57,7 @@
     --spectrum-semantic-negative-icon-color: #e34850;
     min-width: 100px;
     margin: 0;
+    border-color: var(--spectrum-global-color-gray-400);
+    border-width: 1px;
   }
 </style>

packages/bbui/src/Label/Label.svelte

@@ -21,7 +21,7 @@
   label {
     padding: 0;
     white-space: nowrap;
-    color: var(--spectrum-global-color-gray-600);
+    color: var(--spectrum-global-color-gray-700);
   }
 
   .muted {

packages/bbui/src/Modal/Modal.svelte

@@ -1,7 +1,7 @@
 <script>
   import "@spectrum-css/modal/dist/index-vars.css"
   import "@spectrum-css/underlay/dist/index-vars.css"
-  import { createEventDispatcher, setContext, tick } from "svelte"
+  import { createEventDispatcher, setContext, tick, onMount } from "svelte"
   import { fade, fly } from "svelte/transition"
   import Portal from "svelte-portal"
   import Context from "../context"
@@ -62,9 +62,14 @@
   }
 
   setContext(Context.Modal, { show, hide, cancel })
-</script>
 
-<svelte:window on:keydown={handleKey} />
+  onMount(() => {
+    document.addEventListener("keydown", handleKey)
+    return () => {
+      document.removeEventListener("keydown", handleKey)
+    }
+  })
+</script>
 
 {#if inline}
   {#if visible}

packages/bbui/src/Popover/Popover.svelte

@@ -18,6 +18,7 @@
   export let useAnchorWidth = false
   export let dismissible = true
   export let offset = 5
+  export let customHeight
 
   $: target = portalTarget || getContext(Context.PopoverRoot) || ".spectrum"
 
@@ -74,6 +75,7 @@
       on:keydown={handleEscape}
       class="spectrum-Popover is-open"
       role="presentation"
+      style="height: {customHeight}"
       transition:fly|local={{ y: -20, duration: 200 }}
     >
       <slot />

packages/bbui/src/helpers.js

@@ -104,6 +104,9 @@ export const deepSet = (obj, key, value) => {
  * @param obj the object to clone
  */
 export const cloneDeep = obj => {
+  if (!obj) {
+    return obj
+  }
   return JSON.parse(JSON.stringify(obj))
 }
 

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.3.18-alpha.7",
+  "version": "2.3.18-alpha.29",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -58,10 +58,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.3.18-alpha.7",
-    "@budibase/client": "2.3.18-alpha.7",
-    "@budibase/frontend-core": "2.3.18-alpha.7",
-    "@budibase/string-templates": "2.3.18-alpha.7",
+    "@budibase/bbui": "2.3.18-alpha.29",
+    "@budibase/client": "2.3.18-alpha.29",
+    "@budibase/frontend-core": "2.3.18-alpha.29",
+    "@budibase/string-templates": "2.3.18-alpha.29",
     "@fortawesome/fontawesome-svg-core": "^6.2.1",
     "@fortawesome/free-brands-svg-icons": "^6.2.1",
     "@fortawesome/free-solid-svg-icons": "^6.2.1",
@@ -72,6 +72,7 @@
     "codemirror": "^5.59.0",
     "dayjs": "^1.11.2",
     "downloadjs": "1.4.7",
+    "fast-json-patch": "^3.1.1",
     "lodash": "4.17.21",
     "posthog-js": "^1.36.0",
     "remixicon": "2.5.0",

packages/builder/src/builderStore/index.js

@@ -5,12 +5,47 @@ import { getThemeStore } from "./store/theme"
 import { derived } from "svelte/store"
 import { findComponent, findComponentPath } from "./componentUtils"
 import { RoleUtils } from "@budibase/frontend-core"
+import { createHistoryStore } from "builderStore/store/history"
+import { get } from "svelte/store"
 
 export const store = getFrontendStore()
 export const automationStore = getAutomationStore()
 export const themeStore = getThemeStore()
 export const temporalStore = getTemporalStore()
 
+// Setup history for screens
+export const screenHistoryStore = createHistoryStore({
+  getDoc: id => get(store).screens?.find(screen => screen._id === id),
+  selectDoc: store.actions.screens.select,
+  afterAction: () => {
+    // Ensure a valid component is selected
+    if (!get(selectedComponent)) {
+      store.update(state => ({
+        ...state,
+        selectedComponentId: get(selectedScreen)?.props._id,
+      }))
+    }
+  },
+})
+store.actions.screens.save = screenHistoryStore.wrapSaveDoc(
+  store.actions.screens.save
+)
+store.actions.screens.delete = screenHistoryStore.wrapDeleteDoc(
+  store.actions.screens.delete
+)
+
+// Setup history for automations
+export const automationHistoryStore = createHistoryStore({
+  getDoc: automationStore.actions.getDefinition,
+  selectDoc: automationStore.actions.select,
+})
+automationStore.actions.save = automationHistoryStore.wrapSaveDoc(
+  automationStore.actions.save
+)
+automationStore.actions.delete = automationHistoryStore.wrapDeleteDoc(
+  automationStore.actions.delete
+)
+
 export const selectedScreen = derived(store, $store => {
   return $store.screens.find(screen => screen._id === $store.selectedScreenId)
 })
@@ -71,3 +106,13 @@ export const selectedComponentPath = derived(
     ).map(component => component._id)
   }
 )
+
+// Derived automation state
+export const selectedAutomation = derived(automationStore, $automationStore => {
+  if (!$automationStore.selectedAutomationId) {
+    return null
+  }
+  return $automationStore.automations?.find(
+    x => x._id === $automationStore.selectedAutomationId
+  )
+})

packages/builder/src/builderStore/store/automation/Automation.js

@@ -1,69 +0,0 @@
-import { generate } from "shortid"
-
-/**
- * Class responsible for the traversing of the automation definition.
- * Automation definitions are stored in linked lists.
- */
-export default class Automation {
-  constructor(automation) {
-    this.automation = automation
-  }
-
-  hasTrigger() {
-    return this.automation.definition.trigger
-  }
-
-  addTestData(data) {
-    this.automation.testData = { ...this.automation.testData, ...data }
-  }
-
-  addBlock(block, idx) {
-    // Make sure to add trigger if doesn't exist
-    if (!this.hasTrigger() && block.type === "TRIGGER") {
-      const trigger = { id: generate(), ...block }
-      this.automation.definition.trigger = trigger
-      return trigger
-    }
-
-    const newBlock = { id: generate(), ...block }
-    this.automation.definition.steps.splice(idx, 0, newBlock)
-    return newBlock
-  }
-
-  updateBlock(updatedBlock, id) {
-    const { steps, trigger } = this.automation.definition
-
-    if (trigger && trigger.id === id) {
-      this.automation.definition.trigger = updatedBlock
-      return
-    }
-
-    const stepIdx = steps.findIndex(step => step.id === id)
-    if (stepIdx < 0) throw new Error("Block not found.")
-    steps.splice(stepIdx, 1, updatedBlock)
-    this.automation.definition.steps = steps
-  }
-
-  deleteBlock(id) {
-    const { steps, trigger } = this.automation.definition
-
-    if (trigger && trigger.id === id) {
-      this.automation.definition.trigger = null
-      return
-    }
-
-    const stepIdx = steps.findIndex(step => step.id === id)
-    if (stepIdx < 0) throw new Error("Block not found.")
-    steps.splice(stepIdx, 1)
-    this.automation.definition.steps = steps
-  }
-
-  constructBlock(type, stepId, blockDefinition) {
-    return {
-      ...blockDefinition,
-      inputs: blockDefinition.inputs || {},
-      stepId,
-      type,
-    }
-  }
-}

packages/builder/src/builderStore/store/automation/index.js

@@ -1,16 +1,18 @@
-import { writable } from "svelte/store"
+import { writable, get } from "svelte/store"
 import { API } from "api"
-import Automation from "./Automation"
 import { cloneDeep } from "lodash/fp"
+import { generate } from "shortid"
+import { selectedAutomation } from "builderStore"
 
 const initialAutomationState = {
   automations: [],
+  testResults: null,
   showTestPanel: false,
   blockDefinitions: {
     TRIGGER: [],
     ACTION: [],
   },
-  selectedAutomation: null,
+  selectedAutomationId: null,
 }
 
 export const getAutomationStore = () => {
@@ -37,49 +39,41 @@ const automationActions = store => ({
       API.getAutomationDefinitions(),
     ])
     store.update(state => {
-      let selected = state.selectedAutomation?.automation
       state.automations = responses[0]
+      state.automations.sort((a, b) => {
+        return a.name < b.name ? -1 : 1
+      })
       state.blockDefinitions = {
         TRIGGER: responses[1].trigger,
         ACTION: responses[1].action,
       }
-      // If previously selected find the new obj and select it
-      if (selected) {
-        selected = responses[0].filter(
-          automation => automation._id === selected._id
-        )
-        state.selectedAutomation = new Automation(selected[0])
-      }
       return state
     })
   },
-  create: async ({ name }) => {
+  create: async (name, trigger) => {
     const automation = {
       name,
       type: "automation",
       definition: {
         steps: [],
+        trigger,
       },
     }
-    const response = await API.createAutomation(automation)
-    store.update(state => {
-      state.automations = [...state.automations, response.automation]
-      store.actions.select(response.automation)
-      return state
-    })
+    const response = await store.actions.save(automation)
+    await store.actions.fetch()
+    store.actions.select(response._id)
+    return response
   },
   duplicate: async automation => {
-    const response = await API.createAutomation({
+    const response = await store.actions.save({
       ...automation,
       name: `${automation.name} - copy`,
       _id: undefined,
       _ref: undefined,
     })
-    store.update(state => {
-      state.automations = [...state.automations, response.automation]
-      store.actions.select(response.automation)
-      return state
-    })
+    await store.actions.fetch()
+    store.actions.select(response._id)
+    return response
   },
   save: async automation => {
     const response = await API.updateAutomation(automation)
@@ -90,11 +84,13 @@ const automationActions = store => ({
       )
       if (existingIdx !== -1) {
         state.automations.splice(existingIdx, 1, updatedAutomation)
-        state.automations = [...state.automations]
-        store.actions.select(updatedAutomation)
         return state
+      } else {
+        state.automations = [...state.automations, updatedAutomation]
       }
+      return state
     })
+    return response.automation
   },
   delete: async automation => {
     await API.deleteAutomation({
@@ -102,34 +98,83 @@ const automationActions = store => ({
       automationRev: automation?._rev,
     })
     store.update(state => {
-      const existingIdx = state.automations.findIndex(
-        existing => existing._id === automation?._id
+      // Remove the automation
+      state.automations = state.automations.filter(
+        x => x._id !== automation._id
       )
-      state.automations.splice(existingIdx, 1)
-      state.automations = [...state.automations]
-      state.selectedAutomation = null
-      state.selectedBlock = null
+      // Select a new automation if required
+      if (automation._id === state.selectedAutomationId) {
+        store.actions.select(state.automations[0]?._id)
+      }
       return state
     })
+    await store.actions.fetch()
+  },
+  updateBlockInputs: async (block, data) => {
+    // Create new modified block
+    let newBlock = {
+      ...block,
+      inputs: {
+        ...block.inputs,
+        ...data,
+      },
+    }
+
+    // Remove any nullish or empty string values
+    Object.keys(newBlock.inputs).forEach(key => {
+      const val = newBlock.inputs[key]
+      if (val == null || val === "") {
+        delete newBlock.inputs[key]
+      }
+    })
+
+    // Create new modified automation
+    const automation = get(selectedAutomation)
+    const newAutomation = store.actions.getUpdatedDefinition(
+      automation,
+      newBlock
+    )
+
+    // Don't save if no changes were made
+    if (JSON.stringify(newAutomation) === JSON.stringify(automation)) {
+      return
+    }
+    await store.actions.save(newAutomation)
   },
   test: async (automation, testData) => {
-    store.update(state => {
-      state.selectedAutomation.testResults = null
-      return state
-    })
     const result = await API.testAutomation({
       automationId: automation?._id,
       testData,
     })
+    if (!result?.trigger && !result?.steps?.length) {
+      throw "Something went wrong testing your automation"
+    }
     store.update(state => {
-      state.selectedAutomation.testResults = result
+      state.testResults = result
       return state
     })
   },
-  select: automation => {
+  getDefinition: id => {
+    return get(store).automations?.find(x => x._id === id)
+  },
+  getUpdatedDefinition: (automation, block) => {
+    let newAutomation = cloneDeep(automation)
+    if (automation.definition.trigger?.id === block.id) {
+      newAutomation.definition.trigger = block
+    } else {
+      const idx = automation.definition.steps.findIndex(x => x.id === block.id)
+      newAutomation.definition.steps.splice(idx, 1, block)
+    }
+    return newAutomation
+  },
+  select: id => {
+    if (!id || id === get(store).selectedAutomationId) {
+      return
+    }
     store.update(state => {
-      state.selectedAutomation = new Automation(cloneDeep(automation))
-      state.selectedBlock = null
+      state.selectedAutomationId = id
+      state.testResults = null
+      state.showTestPanel = false
       return state
     })
   },
@@ -147,48 +192,57 @@ const automationActions = store => ({
       appId,
     })
   },
-  addTestDataToAutomation: data => {
-    store.update(state => {
-      state.selectedAutomation.addTestData(data)
-      return state
-    })
+  addTestDataToAutomation: async data => {
+    let newAutomation = cloneDeep(get(selectedAutomation))
+    newAutomation.testData = {
+      ...newAutomation.testData,
+      ...data,
+    }
+    await store.actions.save(newAutomation)
   },
-  addBlockToAutomation: (block, blockIdx) => {
-    store.update(state => {
-      state.selectedBlock = state.selectedAutomation.addBlock(
-        cloneDeep(block),
-        blockIdx
-      )
-      return state
-    })
+  constructBlock(type, stepId, blockDefinition) {
+    return {
+      ...blockDefinition,
+      inputs: blockDefinition.inputs || {},
+      stepId,
+      type,
+      id: generate(),
+    }
   },
-  toggleFieldControl: value => {
-    store.update(state => {
-      state.selectedBlock.rowControl = value
-      return state
-    })
+  addBlockToAutomation: async (block, blockIdx) => {
+    const automation = get(selectedAutomation)
+    let newAutomation = cloneDeep(automation)
+    if (!automation) {
+      return
+    }
+    newAutomation.definition.steps.splice(blockIdx, 0, block)
+    await store.actions.save(newAutomation)
   },
-  deleteAutomationBlock: block => {
-    store.update(state => {
-      const idx =
-        state.selectedAutomation.automation.definition.steps.findIndex(
-          x => x.id === block.id
-        )
-      state.selectedAutomation.deleteBlock(block.id)
+  /**
+   * "rowControl" appears to be the name of the flag used to determine whether
+   * a certain automation block uses values or bindings as inputs
+   */
+  toggleRowControl: async (block, rowControl) => {
+    const newBlock = { ...block, rowControl }
+    const newAutomation = store.actions.getUpdatedDefinition(
+      get(selectedAutomation),
+      newBlock
+    )
+    await store.actions.save(newAutomation)
+  },
+  deleteAutomationBlock: async block => {
+    const automation = get(selectedAutomation)
+    let newAutomation = cloneDeep(automation)
 
-      // Select next closest step
-      const steps = state.selectedAutomation.automation.definition.steps
-      let nextSelectedBlock
-      if (steps[idx] != null) {
-        nextSelectedBlock = steps[idx]
-      } else if (steps[idx - 1] != null) {
-        nextSelectedBlock = steps[idx - 1]
-      } else {
-        nextSelectedBlock =
-          state.selectedAutomation.automation.definition.trigger || null
-      }
-      state.selectedBlock = nextSelectedBlock
-      return state
-    })
+    // Delete trigger if required
+    if (newAutomation.definition.trigger?.id === block.id) {
+      delete newAutomation.definition.trigger
+    } else {
+      // Otherwise remove step
+      newAutomation.definition.steps = newAutomation.definition.steps.filter(
+        step => step.id !== block.id
+      )
+    }
+    await store.actions.save(newAutomation)
   },
 })

packages/builder/src/builderStore/store/automation/tests/Automation.spec.js

@@ -1,48 +0,0 @@
-import Automation from "../Automation"
-import TEST_AUTOMATION from "./testAutomation"
-
-const TEST_BLOCK = {
-  id: "AUXJQGZY7",
-  name: "Delay",
-  icon: "ri-time-fill",
-  tagline: "Delay for <b>{{time}}</b> milliseconds",
-  description: "Delay the automation until an amount of time has passed.",
-  params: { time: "number" },
-  type: "LOGIC",
-  args: { time: "5000" },
-  stepId: "DELAY",
-}
-
-describe("Automation Data Object", () => {
-  let automation
-
-  beforeEach(() => {
-    automation = new Automation({ ...TEST_AUTOMATION })
-  })
-
-  it("adds a automation block to the automation", () => {
-    automation.addBlock(TEST_BLOCK)
-    expect(automation.automation.definition)
-  })
-
-  it("updates a automation block with new attributes", () => {
-    const firstBlock = automation.automation.definition.steps[0]
-    const updatedBlock = {
-      ...firstBlock,
-      name: "UPDATED",
-    }
-    automation.updateBlock(updatedBlock, firstBlock.id)
-    expect(automation.automation.definition.steps[0]).toEqual(updatedBlock)
-  })
-
-  it("deletes a automation block successfully", () => {
-    const { steps } = automation.automation.definition
-    const originalLength = steps.length
-
-    const lastBlock = steps[steps.length - 1]
-    automation.deleteBlock(lastBlock.id)
-    expect(automation.automation.definition.steps.length).toBeLessThan(
-      originalLength
-    )
-  })
-})

packages/builder/src/builderStore/store/automation/tests/testAutomation.js

@@ -1,78 +0,0 @@
-export default {
-  name: "Test automation",
-  definition: {
-    steps: [
-      {
-        id: "ANBDINAPS",
-        description: "Send an email.",
-        tagline: "Send email to <b>{{to}}</b>",
-        icon: "ri-mail-open-fill",
-        name: "Send Email",
-        params: {
-          to: "string",
-          from: "string",
-          subject: "longText",
-          text: "longText",
-        },
-        type: "ACTION",
-        args: {
-          text: "A user was created!",
-          subject: "New Budibase User",
-          from: "budimaster@budibase.com",
-          to: "test@test.com",
-        },
-        stepId: "SEND_EMAIL",
-      },
-    ],
-    trigger: {
-      id: "iRzYMOqND",
-      name: "Row Saved",
-      event: "row:save",
-      icon: "ri-save-line",
-      tagline: "Row is added to <b>{{table.name}}</b>",
-      description: "Fired when a row is saved to your database.",
-      params: { table: "table" },
-      type: "TRIGGER",
-      args: {
-        table: {
-          type: "table",
-          views: {},
-          name: "users",
-          schema: {
-            name: {
-              type: "string",
-              constraints: {
-                type: "string",
-                length: { maximum: 123 },
-                presence: { allowEmpty: false },
-              },
-              name: "name",
-            },
-            age: {
-              type: "number",
-              constraints: {
-                type: "number",
-                presence: { allowEmpty: false },
-                numericality: {
-                  greaterThanOrEqualTo: "",
-                  lessThanOrEqualTo: "",
-                },
-              },
-              name: "age",
-            },
-          },
-          _id: "c6b4e610cd984b588837bca27188a451",
-          _rev: "7-b8aa1ce0b53e88928bb88fc11bdc0aff",
-        },
-      },
-      stepId: "ROW_SAVED",
-    },
-  },
-  type: "automation",
-  ok: true,
-  id: "b384f861f4754e1693835324a7fcca62",
-  rev: "1-aa1c2cbd868ef02e26f8fad531dd7e37",
-  live: false,
-  _id: "b384f861f4754e1693835324a7fcca62",
-  _rev: "108-4116829ec375e0481d0ecab9e83a2caf",
-}

packages/builder/src/builderStore/store/frontend.js

@@ -1,6 +1,11 @@
 import { get, writable } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
-import { selectedScreen, selectedComponent } from "builderStore"
+import {
+  selectedScreen,
+  selectedComponent,
+  screenHistoryStore,
+  automationHistoryStore,
+} from "builderStore"
 import {
   datasources,
   integrations,
@@ -67,6 +72,8 @@ const INITIAL_FRONTEND_STATE = {
   // onboarding
   onboarding: false,
   tourNodes: null,
+
+  builderSidePanel: false,
 }
 
 export const getFrontendStore = () => {
@@ -122,6 +129,8 @@ export const getFrontendStore = () => {
         navigation: application.navigation || {},
         usedPlugins: application.usedPlugins || [],
       }))
+      screenHistoryStore.reset()
+      automationHistoryStore.reset()
 
       // Initialise backend stores
       database.set(application.instance)
@@ -179,10 +188,7 @@ export const getFrontendStore = () => {
         }
 
         // Check screen isn't already selected
-        if (
-          state.selectedScreenId === screen._id &&
-          state.selectedComponentId === screen.props?._id
-        ) {
+        if (state.selectedScreenId === screen._id) {
           return
         }
 
@@ -256,7 +262,7 @@ export const getFrontendStore = () => {
         }
       },
       save: async screen => {
-        /* 
+        /*
           Temporarily disabled to accomodate migration issues.
           store.actions.screens.validate(screen)
         */
@@ -347,6 +353,7 @@ export const getFrontendStore = () => {
 
           return state
         })
+        return null
       },
       updateSetting: async (screen, name, value) => {
         if (!screen || !name) {

packages/builder/src/builderStore/store/history.js

@@ -0,0 +1,319 @@
+import * as jsonpatch from "fast-json-patch/index.mjs"
+import { writable, derived, get } from "svelte/store"
+
+const Operations = {
+  Add: "Add",
+  Delete: "Delete",
+  Change: "Change",
+}
+
+const initialState = {
+  history: [],
+  position: 0,
+  loading: false,
+}
+
+export const createHistoryStore = ({
+  getDoc,
+  selectDoc,
+  beforeAction,
+  afterAction,
+}) => {
+  // Use a derived store to check if we are able to undo or redo any operations
+  const store = writable(initialState)
+  const derivedStore = derived(store, $store => {
+    return {
+      ...$store,
+      canUndo: $store.position > 0,
+      canRedo: $store.position < $store.history.length,
+    }
+  })
+
+  // Wrapped versions of essential functions which we call ourselves when using
+  // undo and redo
+  let saveFn
+  let deleteFn
+
+  /**
+   * Internal util to set the loading flag
+   */
+  const startLoading = () => {
+    store.update(state => {
+      state.loading = true
+      return state
+    })
+  }
+
+  /**
+   * Internal util to unset the loading flag
+   */
+  const stopLoading = () => {
+    store.update(state => {
+      state.loading = false
+      return state
+    })
+  }
+
+  /**
+   * Resets history state
+   */
+  const reset = () => {
+    store.set(initialState)
+  }
+
+  /**
+   * Adds or updates an operation in history.
+   * For internal use only.
+   * @param operation the operation to save
+   */
+  const saveOperation = operation => {
+    store.update(state => {
+      // Update history
+      let history = state.history
+      let position = state.position
+      if (!operation.id) {
+        // Every time a new operation occurs we discard any redo potential
+        operation.id = Math.random()
+        history = [...history.slice(0, state.position), operation]
+        position += 1
+      } else {
+        // If this is a redo/undo of an existing operation, just update history
+        // to replace the doc object as revisions may have changed
+        const idx = history.findIndex(op => op.id === operation.id)
+        history[idx].doc = operation.doc
+      }
+      return { history, position }
+    })
+  }
+
+  /**
+   * Wraps the save function, which asynchronously updates a doc.
+   * The returned function is an enriched version of the real save function so
+   * that we can control history.
+   * @param fn the save function
+   * @returns {function} a wrapped version of the save function
+   */
+  const wrapSaveDoc = fn => {
+    saveFn = async (doc, operationId) => {
+      // Only works on a single doc at a time
+      if (!doc || Array.isArray(doc)) {
+        return
+      }
+      startLoading()
+      try {
+        const oldDoc = getDoc(doc._id)
+        const newDoc = jsonpatch.deepClone(await fn(doc))
+
+        // Store the change
+        if (!oldDoc) {
+          // If no old doc, this is an add operation
+          saveOperation({
+            type: Operations.Add,
+            doc: newDoc,
+            id: operationId,
+          })
+        } else {
+          // Otherwise this is a change operation
+          saveOperation({
+            type: Operations.Change,
+            forwardPatch: jsonpatch.compare(oldDoc, doc),
+            backwardsPatch: jsonpatch.compare(doc, oldDoc),
+            doc: newDoc,
+            id: operationId,
+          })
+        }
+        stopLoading()
+        return newDoc
+      } catch (error) {
+        // We want to allow errors to propagate up to normal handlers, but we
+        // want to stop loading first
+        stopLoading()
+        throw error
+      }
+    }
+    return saveFn
+  }
+
+  /**
+   * Wraps the delete function, which asynchronously deletes a doc.
+   * The returned function is an enriched version of the real delete function so
+   * that we can control history.
+   * @param fn the delete function
+   * @returns {function} a wrapped version of the delete function
+   */
+  const wrapDeleteDoc = fn => {
+    deleteFn = async (doc, operationId) => {
+      // Only works on a single doc at a time
+      if (!doc || Array.isArray(doc)) {
+        return
+      }
+      startLoading()
+      try {
+        const oldDoc = jsonpatch.deepClone(doc)
+        await fn(doc)
+        saveOperation({
+          type: Operations.Delete,
+          doc: oldDoc,
+          id: operationId,
+        })
+        stopLoading()
+      } catch (error) {
+        // We want to allow errors to propagate up to normal handlers, but we
+        // want to stop loading first
+        stopLoading()
+        throw error
+      }
+    }
+    return deleteFn
+  }
+
+  /**
+   * Asynchronously undoes the previous operation.
+   * Optionally selects the changed document so that changes are visible.
+   * @returns {Promise<void>}
+   */
+  const undo = async () => {
+    // Sanity checks
+    const { canUndo, history, position, loading } = get(derivedStore)
+    if (!canUndo || loading) {
+      return
+    }
+    const operation = history[position - 1]
+    if (!operation) {
+      return
+    }
+    startLoading()
+
+    // Before hook
+    await beforeAction?.(operation)
+
+    // Update state immediately to prevent further clicks and to prevent bad
+    // history in the event of an update failing
+    store.update(state => {
+      return {
+        ...state,
+        position: state.position - 1,
+      }
+    })
+
+    // Undo the operation
+    try {
+      // Undo ADD
+      if (operation.type === Operations.Add) {
+        // Try to get the latest doc version to delete
+        const latestDoc = getDoc(operation.doc._id)
+        const doc = latestDoc || operation.doc
+        await deleteFn(doc, operation.id)
+      }
+
+      // Undo DELETE
+      else if (operation.type === Operations.Delete) {
+        // Delete the _rev from the deleted doc so that we can save it as a new
+        // doc again without conflicts
+        let doc = jsonpatch.deepClone(operation.doc)
+        delete doc._rev
+        const created = await saveFn(doc, operation.id)
+        selectDoc?.(created?._id || doc._id)
+      }
+
+      // Undo CHANGE
+      else {
+        // Get the current doc and apply the backwards patch on top of it
+        let doc = jsonpatch.deepClone(getDoc(operation.doc._id))
+        if (doc) {
+          jsonpatch.applyPatch(
+            doc,
+            jsonpatch.deepClone(operation.backwardsPatch)
+          )
+          await saveFn(doc, operation.id)
+          selectDoc?.(doc._id)
+        }
+      }
+      stopLoading()
+    } catch (error) {
+      stopLoading()
+      throw error
+    }
+
+    // After hook
+    await afterAction?.(operation)
+  }
+
+  /**
+   * Asynchronously redoes the previous undo.
+   * Optionally selects the changed document so that changes are visible.
+   * @returns {Promise<void>}
+   */
+  const redo = async () => {
+    // Sanity checks
+    const { canRedo, history, position, loading } = get(derivedStore)
+    if (!canRedo || loading) {
+      return
+    }
+    const operation = history[position]
+    if (!operation) {
+      return
+    }
+    startLoading()
+
+    // Before hook
+    await beforeAction?.(operation)
+
+    // Update state immediately to prevent further clicks and to prevent bad
+    // history in the event of an update failing
+    store.update(state => {
+      return {
+        ...state,
+        position: state.position + 1,
+      }
+    })
+
+    // Redo the operation
+    try {
+      // Redo ADD
+      if (operation.type === Operations.Add) {
+        // Delete the _rev from the deleted doc so that we can save it as a new
+        // doc again without conflicts
+        let doc = jsonpatch.deepClone(operation.doc)
+        delete doc._rev
+        const created = await saveFn(doc, operation.id)
+        selectDoc?.(created?._id || doc._id)
+      }
+
+      // Redo DELETE
+      else if (operation.type === Operations.Delete) {
+        // Try to get the latest doc version to delete
+        const latestDoc = getDoc(operation.doc._id)
+        const doc = latestDoc || operation.doc
+        await deleteFn(doc, operation.id)
+      }
+
+      // Redo CHANGE
+      else {
+        // Get the current doc and apply the forwards patch on top of it
+        let doc = jsonpatch.deepClone(getDoc(operation.doc._id))
+        if (doc) {
+          jsonpatch.applyPatch(doc, jsonpatch.deepClone(operation.forwardPatch))
+          await saveFn(doc, operation.id)
+          selectDoc?.(doc._id)
+        }
+      }
+      stopLoading()
+    } catch (error) {
+      stopLoading()
+      throw error
+    }
+
+    // After hook
+    await afterAction?.(operation)
+  }
+
+  return {
+    subscribe: derivedStore.subscribe,
+    wrapSaveDoc,
+    wrapDeleteDoc,
+    reset,
+    undo,
+    redo,
+  }
+}

packages/builder/src/components/automation/AutomationBuilder/AutomationBuilder.svelte

@@ -1,10 +1,10 @@
 <script>
-  import { automationStore } from "builderStore"
+  import { selectedAutomation } from "builderStore"
   import Flowchart from "./FlowChart/FlowChart.svelte"
-
-  $: automation = $automationStore.selectedAutomation?.automation
 </script>
 
-{#if automation}
-  <Flowchart {automation} />
+{#if $selectedAutomation}
+  {#key $selectedAutomation._id}
+    <Flowchart automation={$selectedAutomation} />
+  {/key}
 {/if}

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte

@@ -5,7 +5,6 @@
     Detail,
     Body,
     Icon,
-    Tooltip,
     notifications,
   } from "@budibase/bbui"
   import { automationStore } from "builderStore"
@@ -13,7 +12,6 @@
   import { externalActions } from "./ExternalActions"
 
   export let blockIdx
-  export let blockComplete
 
   const disabled = {
     SEND_EMAIL_SMTP: {
@@ -50,15 +48,12 @@
 
   async function addBlockToAutomation() {
     try {
-      const newBlock = $automationStore.selectedAutomation.constructBlock(
+      const newBlock = automationStore.actions.constructBlock(
         "ACTION",
         actionVal.stepId,
         actionVal
       )
-      automationStore.actions.addBlockToAutomation(newBlock, blockIdx + 1)
-      await automationStore.actions.save(
-        $automationStore.selectedAutomation?.automation
-      )
+      await automationStore.actions.addBlockToAutomation(newBlock, blockIdx + 1)
     } catch (error) {
       notifications.error("Error saving automation")
     }
@@ -66,20 +61,14 @@
 </script>
 
 <ModalContent
-  title="Create Automation"
+  title="Add automation step"
   confirmText="Save"
   size="M"
   disabled={!selectedAction}
-  onConfirm={() => {
-    blockComplete = true
-    addBlockToAutomation()
-  }}
+  onConfirm={addBlockToAutomation}
 >
-  <Body size="XS">Select an app or event.</Body>
-
-  <Layout noPadding>
-    <Body size="S">Apps</Body>
-
+  <Layout noPadding gap="XS">
+    <Detail size="S">Apps</Detail>
     <div class="item-list">
       {#each Object.entries(external) as [idx, action]}
         <div
@@ -95,64 +84,45 @@
               alt="zapier"
             />
             <span class="icon-spacing">
-              <Body size="XS">{idx.charAt(0).toUpperCase() + idx.slice(1)}</Body
-              ></span
-            >
+              <Body size="XS">
+                {idx.charAt(0).toUpperCase() + idx.slice(1)}
+              </Body>
+            </span>
           </div>
         </div>
       {/each}
     </div>
+  </Layout>
 
+  <Layout noPadding gap="XS">
     <Detail size="S">Actions</Detail>
-
     <div class="item-list">
       {#each Object.entries(internal) as [idx, action]}
-        {#if disabled[idx] && disabled[idx].disabled}
-          <Tooltip text={disabled[idx].message} direction="bottom">
-            <div
-              class="item"
-              class:selected={selectedAction === action.name}
-              class:disabled={true}
-              on:click={() => selectAction(action)}
-            >
-              <div class="item-body">
-                <Icon name={action.icon} />
-                <span class="icon-spacing">
-                  <Body size="XS">{action.name}</Body></span
-                >
-              </div>
-            </div>
-          </Tooltip>
-        {:else}
-          <div
-            class="item"
-            class:selected={selectedAction === action.name}
-            on:click={() => selectAction(action)}
-          >
-            <div class="item-body">
-              <Icon name={action.icon} />
-              <span class="icon-spacing">
-                <Body size="XS">{action.name}</Body></span
-              >
-            </div>
+        {@const isDisabled = disabled[idx] && disabled[idx].disabled}
+        <div
+          class="item"
+          class:disabled={isDisabled}
+          class:selected={selectedAction === action.name}
+          on:click={isDisabled ? null : () => selectAction(action)}
+        >
+          <div class="item-body">
+            <Icon name={action.icon} />
+            <Body size="XS">{action.name}</Body>
+            {#if isDisabled}
+              <Icon name="Help" tooltip={disabled[idx].message} />
+            {/if}
           </div>
-        {/if}
+        </div>
       {/each}
     </div>
   </Layout>
 </ModalContent>
 
 <style>
-  .disabled {
-    opacity: 0.3;
-    pointer-events: none;
-  }
-  .icon-spacing {
-    margin-left: var(--spacing-m);
-  }
   .item-body {
     display: flex;
     margin-left: var(--spacing-m);
+    gap: var(--spacing-m);
   }
   .item-list {
     display: grid;
@@ -171,8 +141,15 @@
     box-sizing: border-box;
     border-width: 2px;
   }
-  .item:hover,
+  .item:not(.disabled):hover,
   .selected {
     background: var(--spectrum-alias-background-color-tertiary);
   }
+  .disabled {
+    background: var(--spectrum-global-color-gray-200);
+    color: var(--spectrum-global-color-gray-500);
+  }
+  .disabled :global(.spectrum-Body) {
+    color: var(--spectrum-global-color-gray-600);
+  }
 </style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { automationStore } from "builderStore"
+  import { automationStore, selectedAutomation } from "builderStore"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import FlowItem from "./FlowItem.svelte"
   import TestDataModal from "./TestDataModal.svelte"
@@ -13,27 +13,28 @@
     Modal,
   } from "@budibase/bbui"
   import { ActionStepID } from "constants/backend/automations"
+  import UndoRedoControl from "components/common/UndoRedoControl.svelte"
+  import { automationHistoryStore } from "builderStore"
 
   export let automation
+
   let testDataModal
-  let blocks
   let confirmDeleteDialog
 
-  $: {
-    blocks = []
-    if (automation) {
-      if (automation.definition.trigger) {
-        blocks.push(automation.definition.trigger)
-      }
-      blocks = blocks.concat(automation.definition.steps || [])
+  $: blocks = getBlocks(automation)
+
+  const getBlocks = automation => {
+    let blocks = []
+    if (automation.definition.trigger) {
+      blocks.push(automation.definition.trigger)
     }
+    blocks = blocks.concat(automation.definition.steps || [])
+    return blocks
   }
 
   async function deleteAutomation() {
     try {
-      await automationStore.actions.delete(
-        $automationStore.selectedAutomation?.automation
-      )
+      await automationStore.actions.delete($selectedAutomation)
     } catch (error) {
       notifications.error("Error deleting automation")
     }
@@ -41,20 +42,17 @@
 </script>
 
 <div class="canvas">
-  <div style="float: left; padding-left: var(--spacing-xl);">
+  <div class="header">
     <Heading size="S">{automation.name}</Heading>
-  </div>
-  <div style="float: right; padding-right: var(--spacing-xl);" class="title">
-    <div class="subtitle">
-      <div style="display:flex; align-items: center;">
-        <div class="icon">
-          <Icon
-            on:click={confirmDeleteDialog.show}
-            hoverable
-            size="M"
-            name="DeleteOutline"
-          />
-        </div>
+    <div class="controls">
+      <UndoRedoControl store={automationHistoryStore} />
+      <Icon
+        on:click={confirmDeleteDialog.show}
+        hoverable
+        size="M"
+        name="DeleteOutline"
+      />
+      <div class="buttons">
         <ActionButton
           on:click={() => {
             testDataModal.show()
@@ -62,15 +60,13 @@
           icon="MultipleCheck"
           size="M">Run test</ActionButton
         >
-        <div style="padding-left: var(--spacing-m);">
-          <ActionButton
-            disabled={!$automationStore.selectedAutomation?.testResults}
-            on:click={() => {
-              $automationStore.showTestPanel = true
-            }}
-            size="M">Test Details</ActionButton
-          >
-        </div>
+        <ActionButton
+          disabled={!$automationStore.testResults}
+          on:click={() => {
+            $automationStore.showTestPanel = true
+          }}
+          size="M">Test Details</ActionButton
+        >
       </div>
     </div>
   </div>
@@ -80,7 +76,7 @@
     <div
       class="block"
       animate:flip={{ duration: 500 }}
-      in:fly|local={{ x: 500, duration: 500 }}
+      in:fly={{ x: 500, duration: 500 }}
       out:fly|local={{ x: 500, duration: 500 }}
     >
       {#if block.stepId !== ActionStepID.LOOP}
@@ -105,6 +101,9 @@
 </Modal>
 
 <style>
+  .canvas {
+    padding: var(--spacing-l) var(--spacing-xl);
+  }
   /* Fix for firefox not respecting bottom padding in scrolling containers */
   .canvas > *:last-child {
     padding-bottom: 40px;
@@ -122,18 +121,19 @@
     text-align: left;
   }
 
-  .title {
-    padding-bottom: var(--spacing-xl);
-  }
-
-  .subtitle {
-    padding-bottom: var(--spacing-xl);
+  .header {
     display: flex;
     justify-content: space-between;
     align-items: center;
   }
-  .icon {
-    cursor: pointer;
-    padding-right: var(--spacing-m);
+  .controls,
+  .buttons {
+    display: flex;
+    justify-content: flex-end;
+    align-items: center;
+    gap: var(--spacing-xl);
+  }
+  .buttons {
+    gap: var(--spacing-s);
   }
 </style>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { automationStore } from "builderStore"
+  import { automationStore, selectedAutomation } from "builderStore"
   import {
     Icon,
     Divider,
@@ -23,36 +23,26 @@
   export let block
   export let testDataModal
   export let idx
+
   let selected
   let webhookModal
   let actionModal
-  let blockComplete
+  let open = true
   let showLooping = false
   let role
 
-  $: automationId = $automationStore.selectedAutomation?.automation._id
+  $: automationId = $selectedAutomation?._id
   $: showBindingPicker =
     block.stepId === ActionStepID.CREATE_ROW ||
     block.stepId === ActionStepID.UPDATE_ROW
-
   $: isTrigger = block.type === "TRIGGER"
-
-  $: selected = $automationStore.selectedBlock?.id === block.id
-  $: steps =
-    $automationStore.selectedAutomation?.automation?.definition?.steps ?? []
-
+  $: steps = $selectedAutomation?.definition?.steps ?? []
   $: blockIdx = steps.findIndex(step => step.id === block.id)
   $: lastStep = !isTrigger && blockIdx + 1 === steps.length
-
-  $: totalBlocks =
-    $automationStore.selectedAutomation?.automation?.definition?.steps.length +
-    1
-
-  $: loopingSelected =
-    $automationStore.selectedAutomation?.automation.definition.steps.find(
-      x => x.blockToLoop === block.id
-    )
-
+  $: totalBlocks = $selectedAutomation?.definition?.steps.length + 1
+  $: loopBlock = $selectedAutomation?.definition.steps.find(
+    x => x.blockToLoop === block.id
+  )
   $: isAppAction = block?.stepId === TriggerStepID.APP
   $: isAppAction && setPermissions(role)
   $: isAppAction && getPermissions(automationId)
@@ -81,76 +71,54 @@
   }
 
   async function removeLooping() {
-    loopingSelected = false
-    let loopBlock =
-      $automationStore.selectedAutomation?.automation.definition.steps.find(
-        x => x.blockToLoop === block.id
-      )
-    automationStore.actions.deleteAutomationBlock(loopBlock)
-    await automationStore.actions.save(
-      $automationStore.selectedAutomation?.automation
+    let loopBlock = $selectedAutomation?.definition.steps.find(
+      x => x.blockToLoop === block.id
     )
+    try {
+      await automationStore.actions.deleteAutomationBlock(loopBlock)
+    } catch (error) {
+      notifications.error("Error saving automation")
+    }
   }
 
   async function deleteStep() {
-    let loopBlock =
-      $automationStore.selectedAutomation?.automation.definition.steps.find(
-        x => x.blockToLoop === block.id
-      )
+    let loopBlock = $selectedAutomation?.definition.steps.find(
+      x => x.blockToLoop === block.id
+    )
 
     try {
       if (loopBlock) {
-        automationStore.actions.deleteAutomationBlock(loopBlock)
+        await automationStore.actions.deleteAutomationBlock(loopBlock)
       }
-      automationStore.actions.deleteAutomationBlock(block)
-      await automationStore.actions.save(
-        $automationStore.selectedAutomation?.automation
-      )
+      await automationStore.actions.deleteAutomationBlock(block)
     } catch (error) {
-      notifications.error("Error saving notification")
+      notifications.error("Error saving automation")
     }
   }
-  function toggleFieldControl(evt) {
-    onSelect(block)
-    let rowControl
-    if (evt.detail === "Use values") {
-      rowControl = false
-    } else {
-      rowControl = true
-    }
-    automationStore.actions.toggleFieldControl(rowControl)
-    automationStore.actions.save(
-      $automationStore.selectedAutomation?.automation
-    )
+
+  /**
+   * "rowControl" appears to be the name of the flag used to determine whether
+   * a certain automation block uses values or bindings as inputs
+   */
+  function toggleRowControl(evt) {
+    const rowControl = evt.detail !== "Use values"
+    automationStore.actions.toggleRowControl(block, rowControl)
   }
 
   async function addLooping() {
-    loopingSelected = true
     const loopDefinition = $automationStore.blockDefinitions.ACTION.LOOP
-
-    const loopBlock = $automationStore.selectedAutomation.constructBlock(
+    const loopBlock = automationStore.actions.constructBlock(
       "ACTION",
       "LOOP",
       loopDefinition
     )
     loopBlock.blockToLoop = block.id
-    block.loopBlock = loopBlock.id
-    automationStore.actions.addBlockToAutomation(loopBlock, blockIdx)
-    await automationStore.actions.save(
-      $automationStore.selectedAutomation?.automation
-    )
-  }
-
-  async function onSelect(block) {
-    await automationStore.update(state => {
-      state.selectedBlock = block
-      return state
-    })
+    await automationStore.actions.addBlockToAutomation(loopBlock, blockIdx)
   }
 </script>
 
 <div class={`block ${block.type} hoverable`} class:selected on:click={() => {}}>
-  {#if loopingSelected}
+  {#if loopBlock}
     <div class="blockSection">
       <div
         on:click={() => {
@@ -174,13 +142,8 @@
         </div>
 
         <div class="blockTitle">
-          <div
-            style="margin-left: 10px;"
-            on:click={() => {
-              onSelect(block)
-            }}
-          >
-            <Icon name={showLooping ? "ChevronUp" : "ChevronDown"} />
+          <div style="margin-left: 10px;" on:click={() => {}}>
+            <Icon hoverable name={showLooping ? "ChevronDown" : "ChevronUp"} />
           </div>
         </div>
       </div>
@@ -198,9 +161,7 @@
               $automationStore.blockDefinitions.ACTION.LOOP.schema.inputs
                 .properties
             )}
-            block={$automationStore.selectedAutomation?.automation.definition.steps.find(
-              x => x.blockToLoop === block.id
-            )}
+            block={loopBlock}
             {webhookModal}
           />
         </Layout>
@@ -209,22 +170,28 @@
     {/if}
   {/if}
 
-  <FlowItemHeader bind:blockComplete {block} {testDataModal} {idx} />
-  {#if !blockComplete}
+  <FlowItemHeader
+    {open}
+    {block}
+    {testDataModal}
+    {idx}
+    on:toggle={() => (open = !open)}
+  />
+  {#if open}
     <Divider noMargin />
     <div class="blockSection">
       <Layout noPadding gap="S">
         {#if !isTrigger}
           <div>
             <div class="block-options">
-              {#if !loopingSelected}
-                <ActionButton on:click={() => addLooping()} icon="Reuse"
-                  >Add Looping</ActionButton
-                >
+              {#if !loopBlock}
+                <ActionButton on:click={() => addLooping()} icon="Reuse">
+                  Add Looping
+                </ActionButton>
               {/if}
               {#if showBindingPicker}
                 <Select
-                  on:change={toggleFieldControl}
+                  on:change={toggleRowControl}
                   defaultValue="Use values"
                   autoWidth
                   value={block.rowControl ? "Use bindings" : "Use values"}
@@ -250,16 +217,16 @@
           {webhookModal}
         />
         {#if lastStep}
-          <Button on:click={() => testDataModal.show()} cta
-            >Finish and test automation</Button
-          >
+          <Button on:click={() => testDataModal.show()} cta>
+            Finish and test automation
+          </Button>
         {/if}
       </Layout>
     </div>
   {/if}
 
   <Modal bind:this={actionModal} width="30%">
-    <ActionModal {blockIdx} bind:blockComplete />
+    <ActionModal {blockIdx} />
   </Modal>
 
   <Modal bind:this={webhookModal} width="30%">

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte

@@ -2,21 +2,22 @@
   import { automationStore } from "builderStore"
   import { Icon, Body, Detail, StatusLight } from "@budibase/bbui"
   import { externalActions } from "./ExternalActions"
+  import { createEventDispatcher } from "svelte"
 
   export let block
-  export let blockComplete
+  export let open
   export let showTestStatus = false
-  export let showParameters = {}
   export let testResult
   export let isTrigger
   export let idx
 
+  const dispatch = createEventDispatcher()
+
   $: {
     if (!testResult) {
-      testResult =
-        $automationStore.selectedAutomation?.testResults?.steps.filter(step =>
-          block.id ? step.id === block.id : step.stepId === block.stepId
-        )[0]
+      testResult = $automationStore.testResults?.steps?.filter(step =>
+        block.id ? step.id === block.id : step.stepId === block.stepId
+      )?.[0]
     }
   }
   $: isTrigger = isTrigger || block.type === "TRIGGER"
@@ -45,13 +46,7 @@
 </script>
 
 <div class="blockSection">
-  <div
-    on:click={() => {
-      blockComplete = !blockComplete
-      showParameters[block.id] = blockComplete
-    }}
-    class="splitHeader"
-  >
+  <div on:click={() => dispatch("toggle")} class="splitHeader">
     <div class="center-items">
       {#if externalActions[block.stepId]}
         <img
@@ -99,7 +94,7 @@
           onSelect(block)
         }}
       >
-        <Icon hoverable name={blockComplete ? "ChevronUp" : "ChevronDown"} />
+        <Icon hoverable name={open ? "ChevronUp" : "ChevronDown"} />
       </div>
     </div>
   </div>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/TestDataModal.svelte

@@ -7,7 +7,7 @@
     Label,
     notifications,
   } from "@budibase/bbui"
-  import { automationStore } from "builderStore"
+  import { automationStore, selectedAutomation } from "builderStore"
   import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
   import { cloneDeep } from "lodash/fp"
 
@@ -17,9 +17,7 @@
 
   $: {
     // clone the trigger so we're not mutating the reference
-    trigger = cloneDeep(
-      $automationStore.selectedAutomation.automation.definition.trigger
-    )
+    trigger = cloneDeep($selectedAutomation.definition.trigger)
 
     // get the outputs so we can define the fields
     let schema = Object.entries(trigger.schema?.outputs?.properties || {})
@@ -32,7 +30,7 @@
   }
 
   // check to see if there is existing test data in the store
-  $: testData = $automationStore.selectedAutomation.automation.testData || {}
+  $: testData = $selectedAutomation.testData || {}
 
   // Check the schema to see if required fields have been entered
   $: isError = !trigger.schema.outputs.required.every(
@@ -51,10 +49,7 @@
 
   const testAutomation = async () => {
     try {
-      await automationStore.actions.test(
-        $automationStore.selectedAutomation?.automation,
-        testData
-      )
+      await automationStore.actions.test($selectedAutomation, testData)
       $automationStore.showTestPanel = true
     } catch (error) {
       notifications.error("Error testing automation")
@@ -70,8 +65,8 @@
   onConfirm={testAutomation}
   cancelText="Cancel"
 >
-  <Tabs selected="Form" quiet
-    ><Tab icon="Form" title="Form">
+  <Tabs selected="Form" quiet>
+    <Tab icon="Form" title="Form">
       <div class="tab-content-padding">
         <AutomationBlockSetup
           {testData}
@@ -86,11 +81,7 @@
         <Label>JSON</Label>
         <div class="text-area-container">
           <TextArea
-            value={JSON.stringify(
-              $automationStore.selectedAutomation.automation.testData,
-              null,
-              2
-            )}
+            value={JSON.stringify($selectedAutomation.testData, null, 2)}
             error={failedParse}
             on:change={e => parseTestJSON(e)}
           />

packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte

@@ -7,7 +7,7 @@
   export let testResults
   export let width = "400px"
 
-  let showParameters
+  let openBlocks = {}
   let blocks
 
   function prepTestResults(results) {
@@ -48,14 +48,15 @@
     <div class="block" style={width ? `width: ${width}` : ""}>
       {#if block.stepId !== ActionStepID.LOOP}
         <FlowItemHeader
-          showTestStatus={true}
-          bind:showParameters
-          {block}
+          open={!!openBlocks[block.id]}
+          on:toggle={() => (openBlocks[block.id] = !openBlocks[block.id])}
           isTrigger={idx === 0}
-          {idx}
           testResult={filteredResults?.[idx]}
+          showTestStatus
+          {block}
+          {idx}
         />
-        {#if showParameters && showParameters[block.id]}
+        {#if openBlocks[block.id]}
           <Divider noMargin />
           {#if filteredResults?.[idx]?.outputs.iterations}
             <div style="display: flex; padding: 10px 10px 0px 12px;">

packages/builder/src/components/automation/AutomationBuilder/TestPanel.svelte

@@ -2,26 +2,8 @@
   import { Icon, Divider } from "@budibase/bbui"
   import TestDisplay from "./TestDisplay.svelte"
   import { automationStore } from "builderStore"
-  import { ActionStepID } from "constants/backend/automations"
 
   export let automation
-
-  let blocks, testResults
-
-  $: {
-    blocks = []
-    if (automation) {
-      if (automation.definition.trigger) {
-        blocks.push(automation.definition.trigger)
-      }
-      blocks = blocks
-        .concat(automation.definition.steps || [])
-        .filter(x => x.stepId !== ActionStepID.LOOP)
-    } else if ($automationStore.selectedAutomation) {
-      automation = $automationStore.selectedAutomation
-    }
-  }
-  $: testResults = $automationStore.selectedAutomation?.testResults
 </script>
 
 <div class="title">
@@ -42,7 +24,7 @@
 
 <Divider />
 
-<TestDisplay {automation} {testResults} />
+<TestDisplay {automation} testResults={$automationStore.testResults} />
 
 <style>
   .title {

packages/builder/src/components/automation/AutomationPanel/AutomationList.svelte

@@ -1,12 +1,11 @@
 <script>
   import { onMount } from "svelte"
-  import { goto } from "@roxi/routify"
-  import { automationStore } from "builderStore"
+  import { automationStore, selectedAutomation } from "builderStore"
   import NavItem from "components/common/NavItem.svelte"
   import EditAutomationPopover from "./EditAutomationPopover.svelte"
   import { notifications } from "@budibase/bbui"
 
-  $: selectedAutomationId = $automationStore.selectedAutomation?.automation?._id
+  $: selectedAutomationId = $selectedAutomation?._id
 
   onMount(async () => {
     try {
@@ -16,9 +15,8 @@
     }
   })
 
-  function selectAutomation(automation) {
-    automationStore.actions.select(automation)
-    $goto(`./${automation._id}`)
+  function selectAutomation(id) {
+    automationStore.actions.select(id)
   }
 </script>
 
@@ -29,7 +27,7 @@
       icon="ShareAndroid"
       text={automation.name}
       selected={automation._id === selectedAutomationId}
-      on:click={() => selectAutomation(automation)}
+      on:click={() => selectAutomation(automation._id)}
     >
       <EditAutomationPopover {automation} />
     </NavItem>
@@ -42,5 +40,6 @@
     flex-direction: column;
     justify-content: flex-start;
     align-items: stretch;
+    margin: 0 calc(-1 * var(--spacing-xl));
   }
 </style>

packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte

@@ -1,36 +1,20 @@
 <script>
   import AutomationList from "./AutomationList.svelte"
   import CreateAutomationModal from "./CreateAutomationModal.svelte"
-  import { Modal, Tabs, Tab, Button, Layout } from "@budibase/bbui"
+  import { Modal, Button, Layout } from "@budibase/bbui"
+  import Panel from "components/design/Panel.svelte"
 
   export let modal
   export let webhookModal
 </script>
 
-<div class="nav">
-  <Tabs selected="Automations">
-    <Tab title="Automations">
-      <Layout paddingX="L" paddingY="L" gap="S">
-        <Button cta wide on:click={modal.show}>Add automation</Button>
-      </Layout>
-      <AutomationList />
-      <Modal bind:this={modal}>
-        <CreateAutomationModal {webhookModal} />
-      </Modal>
-    </Tab>
-  </Tabs>
-</div>
+<Panel title="Automations" borderRight>
+  <Layout paddingX="L" paddingY="XL" gap="S">
+    <Button cta on:click={modal.show}>Add automation</Button>
+    <AutomationList />
+  </Layout>
+</Panel>
 
-<style>
-  .nav {
-    overflow-y: auto;
-    background: var(--background);
-    display: flex;
-    flex-direction: column;
-    justify-content: flex-start;
-    align-items: stretch;
-    position: relative;
-    border-right: var(--border-light);
-    padding-bottom: 60px;
-  }
-</style>
+<Modal bind:this={modal}>
+  <CreateAutomationModal {webhookModal} />
+</Modal>

packages/builder/src/components/automation/AutomationPanel/CreateAutomationModal.svelte

@@ -1,6 +1,4 @@
 <script>
-  import { goto } from "@roxi/routify"
-  import { database } from "stores/backend"
   import { automationStore } from "builderStore"
   import { notifications } from "@budibase/bbui"
   import {
@@ -10,48 +8,37 @@
     Layout,
     Body,
     Icon,
+    Label,
   } from "@budibase/bbui"
   import { TriggerStepID } from "constants/backend/automations"
 
+  export let webhookModal
+
   let name
   let selectedTrigger
   let nameTouched = false
   let triggerVal
-  export let webhookModal
 
-  $: instanceId = $database._id
   $: nameError =
     nameTouched && !name ? "Please specify a name for the automation." : null
+  $: triggers = Object.entries($automationStore.blockDefinitions.TRIGGER)
 
   async function createAutomation() {
     try {
-      await automationStore.actions.create({
-        name,
-        instanceId,
-      })
-      const newBlock = $automationStore.selectedAutomation.constructBlock(
+      const trigger = automationStore.actions.constructBlock(
         "TRIGGER",
         triggerVal.stepId,
         triggerVal
       )
-
-      automationStore.actions.addBlockToAutomation(newBlock)
+      await automationStore.actions.create(name, trigger)
       if (triggerVal.stepId === TriggerStepID.WEBHOOK) {
-        webhookModal.show
+        webhookModal.show()
       }
-
-      await automationStore.actions.save(
-        $automationStore.selectedAutomation?.automation
-      )
-
       notifications.success(`Automation ${name} created`)
-
-      $goto(`./${$automationStore.selectedAutomation.automation._id}`)
     } catch (error) {
       notifications.error("Error creating automation")
     }
   }
-  $: triggers = Object.entries($automationStore.blockDefinitions.TRIGGER)
 
   const selectTrigger = trigger => {
     triggerVal = trigger
@@ -70,9 +57,9 @@
     header="You must publish your app to activate your automations."
     message="To test your automation before publishing, you can use the 'Run Test' functionality on the next screen."
   />
-  <Body size="XS"
-    >Please name your automation, then select a trigger. Every automation must
-    start with a trigger.
+  <Body size="S">
+    Please name your automation, then select a trigger.<br />
+    Every automation must start with a trigger.
   </Body>
   <Input
     bind:value={name}
@@ -81,9 +68,8 @@
     label="Name"
   />
 
-  <Layout noPadding>
-    <Body size="S">Triggers</Body>
-
+  <Layout noPadding gap="XS">
+    <Label size="S">Trigger</Label>
     <div class="item-list">
       {#each triggers as [idx, trigger]}
         <div

packages/builder/src/components/automation/AutomationPanel/EditAutomationPopover.svelte

@@ -1,5 +1,4 @@
 <script>
-  import { goto } from "@roxi/routify"
   import { automationStore } from "builderStore"
   import { ActionMenu, MenuItem, notifications, Icon } from "@budibase/bbui"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
@@ -14,7 +13,6 @@
     try {
       await automationStore.actions.delete(automation)
       notifications.success("Automation deleted successfully")
-      $goto("../automate")
     } catch (error) {
       notifications.error("Error deleting automation")
     }
@@ -24,7 +22,6 @@
     try {
       await automationStore.actions.duplicate(automation)
       notifications.success("Automation has been duplicated successfully")
-      $goto(`./${$automationStore.selectedAutomation.automation._id}`)
     } catch (error) {
       notifications.error("Error duplicating automation")
     }

packages/builder/src/components/automation/AutomationPanel/UpdateAutomationModal.svelte

@@ -3,13 +3,13 @@
   import { notifications } from "@budibase/bbui"
   import { Icon, Input, ModalContent, Modal } from "@budibase/bbui"
 
+  export let automation
+  export let onCancel = undefined
+
   let name
   let error = ""
   let modal
 
-  export let automation
-  export let onCancel = undefined
-
   export const show = () => {
     name = automation?.name
     modal.show()