Merge branch 'BUDI-9011' of https://github.com/budibase/budibase into BUDI-9011

.gitattributes

@@ -0,0 +1 @@
+scripts/resources/minio filter=lfs diff=lfs merge=lfs -text

.github/workflows/budibase_ci.yml

@@ -30,7 +30,7 @@ env:
 
 jobs:
   lint:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -47,7 +47,7 @@ jobs:
       - run: yarn lint
 
   build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -76,7 +76,7 @@ jobs:
           fi
 
   helm-lint:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -88,7 +88,7 @@ jobs:
       - run: cd charts/budibase && helm lint .
 
   test-libraries:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -122,7 +122,7 @@ jobs:
           fi
 
   test-worker:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -151,11 +151,22 @@ jobs:
           yarn test --verbose --reporters=default --reporters=github-actions
 
   test-server:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         datasource:
-          [mssql, mysql, postgres, postgres_legacy, mongodb, mariadb, oracle, sqs, none]
+          [
+            mssql,
+            mysql,
+            postgres,
+            postgres_legacy,
+            mongodb,
+            mariadb,
+            oracle,
+            sqs,
+            elasticsearch,
+            none,
+          ]
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -192,6 +203,8 @@ jobs:
             docker pull budibase/oracle-database:23.2-slim-faststart
           elif [ "${{ matrix.datasource }}" == "postgres_legacy" ]; then
             docker pull postgres:9.5.25
+          elif [ "${{ matrix.datasource }}" == "elasticsearch" ]; then
+            docker pull elasticsearch@${{ steps.dotenv.outputs.ELASTICSEARCH_SHA }}
           fi
           docker pull minio/minio &
           docker pull redis &
@@ -240,7 +253,7 @@ jobs:
           yarn test --filter $FILTER --verbose --reporters=default --reporters=github-actions
 
   check-pro-submodule:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
     steps:
       - name: Checkout repo and submodules
@@ -299,7 +312,7 @@ jobs:
           fi
 
   check-lockfile:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
     steps:
       - name: Checkout repo

.github/workflows/readme-openapi.yml

@@ -20,7 +20,7 @@ jobs:
       - run: yarn --frozen-lockfile
 
       - name: Install OpenAPI pkg
-        run: yarn global add openapi
+        run: yarn global add rdme@8.6.6
 
       - name: update specs
-        run: cd packages/server && yarn specs && openapi specs/openapi.yaml --key=${{ secrets.README_API_KEY }} --id=6728a74f5918b50036c61841
+        run: cd packages/server && yarn specs && rdme openapi specs/openapi.yaml --key=${{ secrets.README_API_KEY }} --id=67c16880add6da002352069a

docs/CONTRIBUTING.md

@@ -108,7 +108,7 @@ You can install them following any of the steps described below:
   - Installation steps: https://asdf-vm.com/guide/getting-started.html
   - asdf plugin add nodejs
   - asdf plugin add python
-  - npm install -g yarn
+  - asdf plugin add yarn
 
 ### Using NVM and pyenv
 

eslint.config.mjs

@@ -94,6 +94,15 @@ export default [
         allowImportExportEverywhere: true,
       },
     },
+
+    plugins: {
+      ...config.plugins,
+      "@typescript-eslint": tseslint.plugin,
+    },
+    rules: {
+      ...config.rules,
+      "@typescript-eslint/consistent-type-imports": "error",
+    },
   })),
   ...tseslint.configs.strict.map(config => ({
     ...config,

hosting/nginx.dev.conf

@@ -63,6 +63,11 @@ http {
       proxy_send_timeout 120s;
       proxy_http_version 1.1;
 
+      # Enable buffering for potentially large OIDC configs
+      proxy_buffering on;
+      proxy_buffer_size 16k;
+      proxy_buffers 4 32k;
+
       proxy_set_header Host $host;
       proxy_set_header Connection "";
 

hosting/single/Dockerfile

@@ -1,5 +1,5 @@
 ARG BASEIMG=budibase/couchdb:v3.3.3-sqs-v2.1.1
-FROM node:20-slim as build
+FROM node:20-slim AS build
 
 # install node-gyp dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends g++ make python3 jq
@@ -34,13 +34,13 @@ COPY packages/worker/dist packages/worker/dist
 COPY packages/worker/pm2.config.js packages/worker/pm2.config.js
 
 
-FROM $BASEIMG as runner
+FROM $BASEIMG AS runner
 ARG TARGETARCH
-ENV TARGETARCH $TARGETARCH
+ENV TARGETARCH=$TARGETARCH
 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service)
 # e.g. docker build --build-arg TARGETBUILD=aas ....
 ARG TARGETBUILD=single
-ENV TARGETBUILD $TARGETBUILD
+ENV TARGETBUILD=$TARGETBUILD
 
 # install base dependencies
 RUN apt-get update && \
@@ -67,6 +67,12 @@ RUN mkdir -p /var/log/nginx && \
 
 # setup minio
 WORKDIR /minio
+
+# a 2022 version of minio that supports gateway mode
+COPY scripts/resources/minio /minio
+RUN chmod +x minio
+
+# handles the installation of minio in non-aas environments
 COPY scripts/install-minio.sh ./install.sh
 RUN chmod +x install.sh && ./install.sh
 
@@ -86,7 +92,7 @@ COPY hosting/single/ssh/sshd_config /etc/
 COPY hosting/single/ssh/ssh_setup.sh /tmp
 
 #  setup letsencrypt certificate
-RUN apt-get install -y certbot python3-certbot-nginx
+RUN apt-get update && apt-get install -y certbot python3-certbot-nginx
 COPY hosting/letsencrypt /app/letsencrypt
 RUN chmod +x /app/letsencrypt/certificate-request.sh /app/letsencrypt/certificate-renew.sh
 

hosting/single/runner.sh

@@ -1,53 +1,61 @@
 #!/bin/bash
-declare -a ENV_VARS=("COUCHDB_USER" "COUCHDB_PASSWORD" "DATA_DIR" "MINIO_ACCESS_KEY" "MINIO_SECRET_KEY" "INTERNAL_API_KEY" "JWT_SECRET" "REDIS_PASSWORD")
-declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONMENT" "CLUSTER_PORT" "DEPLOYMENT_ENVIRONMENT" "MINIO_URL" "NODE_ENV" "POSTHOG_TOKEN" "REDIS_URL" "SELF_HOSTED" "WORKER_PORT" "WORKER_URL" "TENANT_FEATURE_FLAGS" "ACCOUNT_PORTAL_URL")
-# Check the env vars set in Dockerfile have come through, AAS seems to drop them
-[[ -z "${APP_PORT}" ]] && export APP_PORT=4001
-[[ -z "${ARCHITECTURE}" ]] && export ARCHITECTURE=amd
-[[ -z "${BUDIBASE_ENVIRONMENT}" ]] && export BUDIBASE_ENVIRONMENT=PRODUCTION
-[[ -z "${CLUSTER_PORT}" ]] && export CLUSTER_PORT=80
-[[ -z "${DEPLOYMENT_ENVIRONMENT}" ]] && export DEPLOYMENT_ENVIRONMENT=docker
-[[ -z "${MINIO_URL}" ]] && [[ -z "${USE_S3}" ]] && export MINIO_URL=http://127.0.0.1:9000
-[[ -z "${NODE_ENV}" ]] && export NODE_ENV=production
-[[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
-[[ -z "${ACCOUNT_PORTAL_URL}" ]] && export ACCOUNT_PORTAL_URL=https://account.budibase.app
-[[ -z "${REDIS_URL}" ]] && export REDIS_URL=127.0.0.1:6379
-[[ -z "${SELF_HOSTED}" ]] && export SELF_HOSTED=1
-[[ -z "${WORKER_PORT}" ]] && export WORKER_PORT=4002
-[[ -z "${WORKER_URL}" ]] && export WORKER_URL=http://127.0.0.1:4002
-[[ -z "${APPS_URL}" ]] && export APPS_URL=http://127.0.0.1:4001
-[[ -z "${SERVER_TOP_LEVEL_PATH}" ]] && export SERVER_TOP_LEVEL_PATH=/app
-#  export CUSTOM_DOMAIN=budi001.custom.com
 
-# Azure App Service customisations
-if [[ "${TARGETBUILD}" = "aas" ]]; then
-    export DATA_DIR="${DATA_DIR:-/home}"
-    WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
-    /etc/init.d/ssh start
-else
-    export DATA_DIR=${DATA_DIR:-/data}
+echo "Starting runner.sh..."
+
+# set defaults for Docker-related variables
+export APP_PORT="${APP_PORT:-4001}"
+export ARCHITECTURE="${ARCHITECTURE:-amd}"
+export BUDIBASE_ENVIRONMENT="${BUDIBASE_ENVIRONMENT:-PRODUCTION}"
+export CLUSTER_PORT="${CLUSTER_PORT:-80}"
+export DEPLOYMENT_ENVIRONMENT="${DEPLOYMENT_ENVIRONMENT:-docker}"
+
+# only set MINIO_URL if neither MINIO_URL nor USE_S3 is set
+if [[ -z "${MINIO_URL}" && -z "${USE_S3}" ]]; then
+  export MINIO_URL="http://127.0.0.1:9000"
 fi
-mkdir -p ${DATA_DIR}
-# Mount NFS or GCP Filestore if env vars exist for it
-if [[ ! -z ${FILESHARE_IP} && ! -z ${FILESHARE_NAME} ]]; then
+
+export NODE_ENV="${NODE_ENV:-production}"
+export POSTHOG_TOKEN="${POSTHOG_TOKEN:-phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU}"
+export ACCOUNT_PORTAL_URL="${ACCOUNT_PORTAL_URL:-https://account.budibase.app}"
+export REDIS_URL="${REDIS_URL:-127.0.0.1:6379}"
+export SELF_HOSTED="${SELF_HOSTED:-1}"
+export WORKER_PORT="${WORKER_PORT:-4002}"
+export WORKER_URL="${WORKER_URL:-http://127.0.0.1:4002}"
+export APPS_URL="${APPS_URL:-http://127.0.0.1:4001}"
+export SERVER_TOP_LEVEL_PATH="${SERVER_TOP_LEVEL_PATH:-/app}"
+
+# set DATA_DIR and ensure the directory exists
+if [[ ${TARGETBUILD} == "aas" ]]; then
+    export DATA_DIR="/home"
+else
+    export DATA_DIR="${DATA_DIR:-/data}"
+fi
+mkdir -p "${DATA_DIR}"
+
+# mount NFS or GCP Filestore if FILESHARE_IP and FILESHARE_NAME are set
+if [[ -n "${FILESHARE_IP}" && -n "${FILESHARE_NAME}" ]]; then
     echo "Mounting NFS share"
     apt update && apt install -y nfs-common nfs-kernel-server
     echo "Mount file share ${FILESHARE_IP}:/${FILESHARE_NAME} to ${DATA_DIR}"
-    mount -o nolock ${FILESHARE_IP}:/${FILESHARE_NAME} ${DATA_DIR}
+    mount -o nolock "${FILESHARE_IP}:/${FILESHARE_NAME}" "${DATA_DIR}"
     echo "Mounting result: $?"
 fi
 
-if [ -f "${DATA_DIR}/.env" ]; then
-    # Read in the .env file and export the variables
-    for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done
+# source environment variables from a .env file if it exists in DATA_DIR
+if [[ -f "${DATA_DIR}/.env" ]]; then
+    set -a  # Automatically export all variables loaded from .env
+    source "${DATA_DIR}/.env"
+    set +a
 fi
-# randomise any unset environment variables
-for ENV_VAR in "${ENV_VARS[@]}"
-do
-    if [[ -z "${!ENV_VAR}" ]]; then
-        eval "export $ENV_VAR=$(uuidgen | sed -e 's/-//g')"
+
+# randomize any unset sensitive environment variables using uuidgen
+env_vars=(COUCHDB_USER COUCHDB_PASSWORD MINIO_ACCESS_KEY MINIO_SECRET_KEY INTERNAL_API_KEY JWT_SECRET REDIS_PASSWORD)
+for var in "${env_vars[@]}"; do
+    if [[ -z "${!var}" ]]; then
+        export "$var"="$(uuidgen | tr -d '-')"
     fi
 done
+
 if [[ -z "${COUCH_DB_URL}" ]]; then
     export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@127.0.0.1:5984
 fi
@@ -58,17 +66,15 @@ fi
 
 if [ ! -f "${DATA_DIR}/.env" ]; then
     touch ${DATA_DIR}/.env
-    for ENV_VAR in "${ENV_VARS[@]}"
-    do
+    for ENV_VAR in "${ENV_VARS[@]}"; do
         temp=$(eval "echo \$$ENV_VAR")
-        echo "$ENV_VAR=$temp" >> ${DATA_DIR}/.env
+        echo "$ENV_VAR=$temp" >>${DATA_DIR}/.env
     done
-    for ENV_VAR in "${DOCKER_VARS[@]}"
-    do
+    for ENV_VAR in "${DOCKER_VARS[@]}"; do
         temp=$(eval "echo \$$ENV_VAR")
-        echo "$ENV_VAR=$temp" >> ${DATA_DIR}/.env
+        echo "$ENV_VAR=$temp" >>${DATA_DIR}/.env
     done
-    echo "COUCH_DB_URL=${COUCH_DB_URL}" >> ${DATA_DIR}/.env
+    echo "COUCH_DB_URL=${COUCH_DB_URL}" >>${DATA_DIR}/.env
 fi
 
 # Read in the .env file and export the variables
@@ -79,31 +85,44 @@ ln -s ${DATA_DIR}/.env /worker/.env
 # make these directories in runner, incase of mount
 mkdir -p ${DATA_DIR}/minio
 mkdir -p ${DATA_DIR}/redis
+mkdir -p ${DATA_DIR}/couch
 chown -R couchdb:couchdb ${DATA_DIR}/couch
 
 REDIS_CONFIG="/etc/redis/redis.conf"
 sed -i "s#DATA_DIR#${DATA_DIR}#g" "${REDIS_CONFIG}"
 
 if [[ -n "${USE_DEFAULT_REDIS_CONFIG}" ]]; then
-  REDIS_CONFIG=""
+    REDIS_CONFIG=""
 fi
 
 if [[ -n "${REDIS_PASSWORD}" ]]; then
-  redis-server "${REDIS_CONFIG}" --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
+    redis-server "${REDIS_CONFIG}" --requirepass $REDIS_PASSWORD >/dev/stdout 2>&1 &
 else
-  redis-server "${REDIS_CONFIG}" > /dev/stdout 2>&1 &
+    redis-server "${REDIS_CONFIG}" >/dev/stdout 2>&1 &
 fi
-/bbcouch-runner.sh &
+
+echo "Starting callback CouchDB runner..."
+./bbcouch-runner.sh &
 
 # only start minio if use s3 isn't passed
 if [[ -z "${USE_S3}" ]]; then
-  /minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
+    if [[ ${TARGETBUILD} == aas ]]; then
+        echo "Starting MinIO in Azure Gateway mode"
+        if [[ -z "${AZURE_STORAGE_ACCOUNT}" || -z "${AZURE_STORAGE_KEY}" || -z "${MINIO_ACCESS_KEY}" || -z "${MINIO_SECRET_KEY}" ]]; then
+            echo "The following environment variables must be set: AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_KEY, MINIO_ACCESS_KEY, MINIO_SECRET_KEY"
+            exit 1
+        fi
+        /minio/minio gateway azure --console-address ":9001" >/dev/stdout 2>&1 &
+    else
+        echo "Starting MinIO in standalone mode"
+        /minio/minio server --console-address ":9001" ${DATA_DIR}/minio >/dev/stdout 2>&1 &
+    fi
 fi
 
 /etc/init.d/nginx restart
 if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then
     # Add monthly cron job to renew certbot certificate
-    echo -n "* * 2 * * root exec /app/letsencrypt/certificate-renew.sh ${CUSTOM_DOMAIN}" >> /etc/cron.d/certificate-renew
+    echo -n "* * 2 * * root exec /app/letsencrypt/certificate-renew.sh ${CUSTOM_DOMAIN}" >>/etc/cron.d/certificate-renew
     chmod +x /etc/cron.d/certificate-renew
     # Request the certbot certificate
     /app/letsencrypt/certificate-request.sh ${CUSTOM_DOMAIN}

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.4.11",
+  "version": "3.4.22",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/backend-core/src/context/mainContext.ts

@@ -123,7 +123,7 @@ export async function doInAutomationContext<T>(params: {
   task: () => T
 }): Promise<T> {
   await ensureSnippetContext()
-  return newContext(
+  return await newContext(
     {
       tenantId: getTenantIDFromAppID(params.appId),
       appId: params.appId,
@@ -266,9 +266,9 @@ export const getProdAppId = () => {
   return conversions.getProdAppID(appId)
 }
 
-export function doInEnvironmentContext(
+export function doInEnvironmentContext<T>(
   values: Record<string, string>,
-  task: any
+  task: () => T
 ) {
   if (!values) {
     throw new Error("Must supply environment variables.")

packages/backend-core/src/objectStore/objectStore.ts

@@ -478,7 +478,7 @@ export async function deleteFolder(
   if (existingObjectsResponse.Contents?.length === 0) {
     return
   }
-  const deleteParams: any = {
+  const deleteParams: { Bucket: string; Delete: { Objects: any[] } } = {
     Bucket: bucketName,
     Delete: {
       Objects: [],
@@ -489,10 +489,12 @@ export async function deleteFolder(
     deleteParams.Delete.Objects.push({ Key: content.Key })
   })
 
-  const deleteResponse = await client.deleteObjects(deleteParams)
-  // can only empty 1000 items at once
-  if (deleteResponse.Deleted?.length === 1000) {
-    return deleteFolder(bucketName, folder)
+  if (deleteParams.Delete.Objects.length) {
+    const deleteResponse = await client.deleteObjects(deleteParams)
+    // can only empty 1000 items at once
+    if (deleteResponse.Deleted?.length === 1000) {
+      return deleteFolder(bucketName, folder)
+    }
   }
 }
 

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -3,6 +3,7 @@ import { newid } from "../utils"
 import { Queue, QueueOptions, JobOptions } from "./queue"
 import { helpers } from "@budibase/shared-core"
 import { Job, JobId, JobInformation } from "bull"
+import { cloneDeep } from "lodash"
 
 function jobToJobInformation(job: Job): JobInformation {
   let cron = ""
@@ -33,12 +34,13 @@ function jobToJobInformation(job: Job): JobInformation {
   }
 }
 
-interface JobMessage<T = any> extends Partial<Job<T>> {
+export interface TestQueueMessage<T = any> extends Partial<Job<T>> {
   id: string
   timestamp: number
   queue: Queue<T>
   data: any
   opts?: JobOptions
+  manualTrigger?: boolean
 }
 
 /**
@@ -47,15 +49,15 @@ interface JobMessage<T = any> extends Partial<Job<T>> {
  * internally to register when messages are available to the consumers - in can
  * support many inputs and many consumers.
  */
-class InMemoryQueue implements Partial<Queue> {
+export class InMemoryQueue<T = any> implements Partial<Queue<T>> {
   _name: string
   _opts?: QueueOptions
-  _messages: JobMessage[]
+  _messages: TestQueueMessage<T>[]
   _queuedJobIds: Set<string>
   _emitter: NodeJS.EventEmitter<{
-    message: [JobMessage]
-    completed: [Job]
-    removed: [JobMessage]
+    message: [TestQueueMessage<T>]
+    completed: [Job<T>]
+    removed: [TestQueueMessage<T>]
   }>
   _runCount: number
   _addCount: number
@@ -86,10 +88,12 @@ class InMemoryQueue implements Partial<Queue> {
    */
   async process(concurrencyOrFunc: number | any, func?: any) {
     func = typeof concurrencyOrFunc === "number" ? func : concurrencyOrFunc
-    this._emitter.on("message", async message => {
+    this._emitter.on("message", async msg => {
+      const message = cloneDeep(msg)
+
       // For the purpose of testing, don't trigger cron jobs immediately.
       // Require the test to trigger them manually with timestamps.
-      if (message.opts?.repeat != null) {
+      if (!message.manualTrigger && message.opts?.repeat != null) {
         return
       }
 
@@ -107,7 +111,7 @@ class InMemoryQueue implements Partial<Queue> {
       if (resp.then != null) {
         try {
           await retryFunc(resp)
-          this._emitter.emit("completed", message as Job)
+          this._emitter.emit("completed", message as Job<T>)
         } catch (e: any) {
           console.error(e)
         }
@@ -124,7 +128,6 @@ class InMemoryQueue implements Partial<Queue> {
     return this as any
   }
 
-  // simply puts a message to the queue and emits to the queue for processing
   /**
    * Simple function to replicate the add message functionality of Bull, putting
    * a new message on the queue. This then emits an event which will be used to
@@ -133,7 +136,13 @@ class InMemoryQueue implements Partial<Queue> {
    * a JSON message as this is required by Bull.
    * @param repeat serves no purpose for the import queue.
    */
-  async add(data: any, opts?: JobOptions) {
+  async add(data: T | string, optsOrT?: JobOptions | T) {
+    if (typeof data === "string") {
+      throw new Error("doesn't support named jobs")
+    }
+
+    const opts = optsOrT as JobOptions
+
     const jobId = opts?.jobId?.toString()
     if (jobId && this._queuedJobIds.has(jobId)) {
       console.log(`Ignoring already queued job ${jobId}`)
@@ -148,7 +157,7 @@ class InMemoryQueue implements Partial<Queue> {
     }
 
     const pushMessage = () => {
-      const message: JobMessage = {
+      const message: TestQueueMessage = {
         id: newid(),
         timestamp: Date.now(),
         queue: this as unknown as Queue,
@@ -176,7 +185,7 @@ class InMemoryQueue implements Partial<Queue> {
 
   async removeRepeatableByKey(id: string) {
     for (const [idx, message] of this._messages.entries()) {
-      if (message.opts?.jobId?.toString() === id) {
+      if (message.id === id) {
         this._messages.splice(idx, 1)
         this._emitter.emit("removed", message)
         return
@@ -204,6 +213,16 @@ class InMemoryQueue implements Partial<Queue> {
     return null
   }
 
+  manualTrigger(id: JobId) {
+    for (const message of this._messages) {
+      if (message.id === id) {
+        this._emitter.emit("message", { ...message, manualTrigger: true })
+        return
+      }
+    }
+    throw new Error(`Job with id ${id} not found`)
+  }
+
   on(event: string, callback: (...args: any[]) => void): Queue {
     // @ts-expect-error - this callback can be one of many types
     this._emitter.on(event, callback)

packages/backend-core/src/queue/index.ts

@@ -1,2 +1,3 @@
 export * from "./queue"
 export * from "./constants"
+export * from "./inMemoryQueue"

packages/backend-core/src/sql/utils.ts

@@ -5,10 +5,10 @@ import {
   SqlQuery,
   Table,
   TableSourceType,
+  SEPARATOR,
 } from "@budibase/types"
 import { DEFAULT_BB_DATASOURCE_ID } from "../constants"
 import { Knex } from "knex"
-import { SEPARATOR } from "../db"
 import environment from "../environment"
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`

packages/backend-core/src/utils/Duration.ts

@@ -15,23 +15,27 @@ const conversion: Record<DurationType, number> = {
 }
 
 export class Duration {
+  constructor(public ms: number) {}
+
+  to(type: DurationType) {
+    return this.ms / conversion[type]
+  }
+
+  toMs() {
+    return this.ms
+  }
+
+  toSeconds() {
+    return this.to(DurationType.SECONDS)
+  }
+
   static convert(from: DurationType, to: DurationType, duration: number) {
     const milliseconds = duration * conversion[from]
     return milliseconds / conversion[to]
   }
 
   static from(from: DurationType, duration: number) {
-    return {
-      to: (to: DurationType) => {
-        return Duration.convert(from, to, duration)
-      },
-      toMs: () => {
-        return Duration.convert(from, DurationType.MILLISECONDS, duration)
-      },
-      toSeconds: () => {
-        return Duration.convert(from, DurationType.SECONDS, duration)
-      },
-    }
+    return new Duration(duration * conversion[from])
   }
 
   static fromSeconds(duration: number) {

packages/backend-core/src/utils/index.ts

@@ -2,3 +2,4 @@ export * from "./hashing"
 export * from "./utils"
 export * from "./stringUtils"
 export * from "./Duration"
+export * from "./time"

packages/backend-core/src/utils/time.ts

@@ -0,0 +1,7 @@
+import { Duration } from "./Duration"
+
+export async function time<T>(f: () => Promise<T>): Promise<[T, Duration]> {
+  const start = performance.now()
+  const result = await f()
+  return [result, Duration.fromMilliseconds(performance.now() - start)]
+}

packages/bbui/src/Actions/position_dropdown.ts

@@ -27,7 +27,7 @@ export type UpdateHandler = (
 
 interface Opts {
   anchor?: HTMLElement
-  align: PopoverAlignment
+  align: PopoverAlignment | `${PopoverAlignment}`
   maxHeight?: number
   maxWidth?: number
   minWidth?: number

packages/bbui/src/Badge/Badge.svelte

@@ -11,6 +11,7 @@
   export let active = false
   export let inactive = false
   export let hoverable = false
+  export let outlineColor = null
 </script>
 
 <!-- svelte-ignore a11y-no-static-element-interactions -->
@@ -29,6 +30,7 @@
   class:spectrum-Label--seafoam={seafoam}
   class:spectrum-Label--active={active}
   class:spectrum-Label--inactive={inactive}
+  style={outlineColor ? `border: 2px solid ${outlineColor}` : ""}
 >
   <slot />
 </span>

packages/bbui/src/ClearButton/ClearButton.svelte

@@ -1,6 +1,6 @@
-<script>
-  export let small = false
-  export let disabled
+<script lang="ts">
+  export let small: boolean = false
+  export let disabled: boolean = false
 </script>
 
 <button

packages/bbui/src/Drawer/Drawer.svelte

@@ -64,7 +64,7 @@
   import { setContext, createEventDispatcher, onDestroy } from "svelte"
   import { generate } from "shortid"
 
-  export let title
+  export let title = ""
   export let forceModal = false
 
   const dispatch = createEventDispatcher()

packages/bbui/src/Form/Checkbox.svelte

@@ -1,24 +1,24 @@
-<script>
+<script lang="ts">
   import Field from "./Field.svelte"
   import Checkbox from "./Core/Checkbox.svelte"
   import { createEventDispatcher } from "svelte"
 
-  export let value = null
-  export let label = null
-  export let labelPosition = "above"
-  export let text = null
-  export let disabled = false
-  export let error = null
-  export let size = "M"
-  export let helpText = null
+  export let value: boolean | undefined = undefined
+  export let label: string | undefined = undefined
+  export let labelPosition: "above" | "below" = "above"
+  export let text: string | undefined = undefined
+  export let disabled: boolean = false
+  export let error: string | undefined = undefined
+  export let size: "S" | "M" | "L" | "XL" = "M"
+  export let helpText: string | undefined = undefined
 
   const dispatch = createEventDispatcher()
-  const onChange = e => {
+  const onChange = (e: CustomEvent<boolean>) => {
     value = e.detail
     dispatch("change", e.detail)
   }
 </script>
 
 <Field {helpText} {label} {labelPosition} {error}>
-  <Checkbox {error} {disabled} {text} {value} {size} on:change={onChange} />
+  <Checkbox {disabled} {text} {value} {size} on:change={onChange} />
 </Field>

packages/bbui/src/Icon/Icon.svelte

@@ -28,6 +28,9 @@
     <svg
       on:contextmenu
       on:click
+      on:mouseover
+      on:mouseleave
+      on:focus
       class:hoverable
       class:disabled
       class="spectrum-Icon spectrum-Icon--size{size}"

packages/bbui/src/Layout/Page.svelte

@@ -47,7 +47,7 @@
     overflow-x: hidden;
   }
   .main {
-    overflow-y: scroll;
+    overflow-y: auto;
   }
   .content {
     display: flex;

packages/bbui/src/Link/Link.svelte

@@ -10,7 +10,7 @@
   export let secondary: boolean = false
   export let overBackground: boolean = false
   export let target: string | undefined = undefined
-  export let download: boolean | undefined = undefined
+  export let download: string | undefined = undefined
   export let disabled: boolean = false
   export let tooltip: string | null = null
 

packages/bbui/src/Popover/Popover.svelte

@@ -18,8 +18,9 @@
   import type { KeyboardEventHandler } from "svelte/elements"
   import { PopoverAlignment } from "../constants"
 
-  export let anchor: HTMLElement
-  export let align: PopoverAlignment = PopoverAlignment.Right
+  export let anchor: HTMLElement | undefined
+  export let align: PopoverAlignment | `${PopoverAlignment}` =
+    PopoverAlignment.Right
   export let portalTarget: string | undefined = undefined
   export let minWidth: number | undefined = undefined
   export let maxWidth: number | undefined = undefined

packages/bbui/src/SideNavigation/Item.svelte

@@ -1,63 +0,0 @@
-<script>
-  import { getContext } from "svelte"
-
-  const multilevel = getContext("sidenav-type")
-  import Badge from "../Badge/Badge.svelte"
-
-  export let href = ""
-  export let external = false
-  export let heading = ""
-  export let icon = ""
-  export let selected = false
-  export let disabled = false
-  export let badge = ""
-</script>
-
-<!-- svelte-ignore a11y-click-events-have-key-events -->
-<!-- svelte-ignore a11y-no-noninteractive-element-interactions -->
-<li
-  class="spectrum-SideNav-item"
-  class:is-selected={selected}
-  class:is-disabled={disabled}
-  on:click
->
-  {#if heading}
-    <h2 class="spectrum-SideNav-heading" id="nav-heading-{heading}">
-      {heading}
-    </h2>
-  {/if}
-  <a
-    target={external ? "_blank" : ""}
-    {href}
-    class="spectrum-SideNav-itemLink"
-    aria-current="page"
-  >
-    {#if icon}
-      <svg
-        class="spectrum-Icon spectrum-Icon--sizeM spectrum-SideNav-itemIcon"
-        focusable="false"
-        aria-hidden="true"
-      >
-        <use xlink:href="#spectrum-icon-18-{icon}" />
-      </svg>
-    {/if}
-    <slot />
-    {#if badge}
-      <div class="badge">
-        <Badge active size="S">{badge}</Badge>
-      </div>
-    {/if}
-  </a>
-
-  {#if multilevel && $$slots.subnav}
-    <ul class="spectrum-SideNav">
-      <slot name="subnav" />
-    </ul>
-  {/if}
-</li>
-
-<style>
-  .badge {
-    margin-left: 10px;
-  }
-</style>

packages/bbui/src/SideNavigation/Navigation.svelte

@@ -1,13 +0,0 @@
-<script>
-  import { setContext } from "svelte"
-  import "@spectrum-css/sidenav/dist/index-vars.css"
-
-  export let multilevel = false
-  setContext("sidenav-type", multilevel)
-</script>
-
-<nav>
-  <ul class="spectrum-SideNav" class:spectrum-SideNav--multiLevel={multilevel}>
-    <slot />
-  </ul>
-</nav>

packages/bbui/src/Stores/banner.ts

@@ -7,17 +7,26 @@ export const BANNER_TYPES = {
   WARNING: "warning",
 }
 
+interface BannerConfig {
+  message?: string
+  type?: string
+  extraButtonText?: string
+  extraButtonAction?: () => void
+  onChange?: () => void
+}
+
+interface DefaultConfig {
+  messages: BannerConfig[]
+}
+
 export function createBannerStore() {
-  const DEFAULT_CONFIG = {
+  const DEFAULT_CONFIG: DefaultConfig = {
     messages: [],
   }
 
-  const banner = writable(DEFAULT_CONFIG)
+  const banner = writable<DefaultConfig>(DEFAULT_CONFIG)
 
-  const show = async (
-    // eslint-disable-next-line
-    config = { message, type, extraButtonText, extraButtonAction, onChange }
-  ) => {
+  const show = async (config: BannerConfig = {}) => {
     banner.update(store => {
       return {
         ...store,
@@ -27,7 +36,7 @@ export function createBannerStore() {
   }
 
   const showStatus = async () => {
-    const config = {
+    const config: BannerConfig = {
       message: "Some systems are experiencing issues",
       type: BANNER_TYPES.NEGATIVE,
       extraButtonText: "View Status",
@@ -37,18 +46,24 @@ export function createBannerStore() {
     await queue([config])
   }
 
-  const queue = async entries => {
-    const priority = {
+  const queue = async (entries: Array<BannerConfig>) => {
+    const priority: Record<string, number> = {
       [BANNER_TYPES.NEGATIVE]: 0,
       [BANNER_TYPES.WARNING]: 1,
       [BANNER_TYPES.INFO]: 2,
     }
     banner.update(store => {
       const sorted = [...store.messages, ...entries].sort((a, b) => {
-        if (priority[a.type] == priority[b.type]) {
+        if (
+          priority[a.type as keyof typeof priority] ===
+          priority[b.type as keyof typeof priority]
+        ) {
           return 0
         }
-        return priority[a.type] < priority[b.type] ? -1 : 1
+        return priority[a.type as keyof typeof priority] <
+          priority[b.type as keyof typeof priority]
+          ? -1
+          : 1
       })
       return {
         ...store,

packages/bbui/src/Stores/notifications.ts

@@ -2,9 +2,21 @@ import { writable } from "svelte/store"
 
 const NOTIFICATION_TIMEOUT = 3000
 
+interface Notification {
+  id: string
+  type: string
+  message: string
+  icon: string
+  dismissable: boolean
+  action: (() => void) | null
+  actionMessage: string | null
+  wide: boolean
+  dismissTimeout: number
+}
+
 export const createNotificationStore = () => {
-  const timeoutIds = new Set()
-  const _notifications = writable([], () => {
+  const timeoutIds = new Set<ReturnType<typeof setTimeout>>()
+  const _notifications = writable<Notification[]>([], () => {
     return () => {
       // clear all the timers
       timeoutIds.forEach(timeoutId => {
@@ -21,7 +33,7 @@ export const createNotificationStore = () => {
   }
 
   const send = (
-    message,
+    message: string,
     {
       type = "default",
       icon = "",
@@ -30,7 +42,15 @@ export const createNotificationStore = () => {
       actionMessage = null,
       wide = false,
       dismissTimeout = NOTIFICATION_TIMEOUT,
-    }
+    }: {
+      type?: string
+      icon?: string
+      autoDismiss?: boolean
+      action?: (() => void) | null
+      actionMessage?: string | null
+      wide?: boolean
+      dismissTimeout?: number
+    } = {}
   ) => {
     if (block) {
       return
@@ -60,7 +80,7 @@ export const createNotificationStore = () => {
     }
   }
 
-  const dismissNotification = id => {
+  const dismissNotification = (id: string) => {
     _notifications.update(state => {
       return state.filter(n => n.id !== id)
     })
@@ -71,17 +91,18 @@ export const createNotificationStore = () => {
   return {
     subscribe,
     send,
-    info: msg => send(msg, { type: "info", icon: "Info" }),
-    error: msg =>
+    info: (msg: string) => send(msg, { type: "info", icon: "Info" }),
+    error: (msg: string) =>
       send(msg, { type: "error", icon: "Alert", autoDismiss: false }),
-    warning: msg => send(msg, { type: "warning", icon: "Alert" }),
-    success: msg => send(msg, { type: "success", icon: "CheckmarkCircle" }),
+    warning: (msg: string) => send(msg, { type: "warning", icon: "Alert" }),
+    success: (msg: string) =>
+      send(msg, { type: "success", icon: "CheckmarkCircle" }),
     blockNotifications,
     dismiss: dismissNotification,
   }
 }
 
-function id() {
+function id(): string {
   return "_" + Math.random().toString(36).slice(2, 9)
 }
 

packages/bbui/src/Table/ArrayRenderer.svelte

@@ -1,8 +1,8 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/label/dist/index-vars.css"
   import Badge from "../Badge/Badge.svelte"
 
-  export let value
+  export let value: string | string[]
 
   const displayLimit = 5
 

packages/bbui/src/Table/AttachmentRenderer.svelte

@@ -1,15 +1,15 @@
-<script>
+<script lang="ts">
   import Link from "../Link/Link.svelte"
 
-  export let value
+  export let value: { name: string; url: string; extension: string }[]
 
   const displayLimit = 5
   $: attachments = value?.slice(0, displayLimit) ?? []
   $: leftover = (value?.length ?? 0) - attachments.length
 
-  const imageExtensions = ["png", "tiff", "gif", "raw", "jpg", "jpeg"]
-  const isImage = extension => {
-    return imageExtensions.includes(extension?.toLowerCase())
+  const imageExtensions: string[] = ["png", "tiff", "gif", "raw", "jpg", "jpeg"]
+  const isImage = (extension: string | undefined): boolean => {
+    return imageExtensions.includes(extension?.toLowerCase() ?? "")
   }
 </script>
 

packages/bbui/src/Table/BoldRenderer.svelte

@@ -1,5 +1,5 @@
-<script>
-  export let value
+<script lang="ts">
+  export let value: string
 </script>
 
 <div class="bold">{value}</div>

packages/bbui/src/Table/BooleanRenderer.svelte

@@ -1,7 +1,7 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/checkbox/dist/index-vars.css"
 
-  export let value
+  export let value: boolean
 </script>
 
 <label

packages/bbui/src/Table/CellRenderer.svelte

@@ -1,4 +1,4 @@
-<script>
+<script lang="ts">
   import StringRenderer from "./StringRenderer.svelte"
   import BooleanRenderer from "./BooleanRenderer.svelte"
   import DateTimeRenderer from "./DateTimeRenderer.svelte"
@@ -8,14 +8,14 @@
   import InternalRenderer from "./InternalRenderer.svelte"
   import { processStringSync } from "@budibase/string-templates"
 
-  export let row
-  export let schema
-  export let value
-  export let customRenderers = []
-  export let snippets
+  export let row: Record<string, any>
+  export let schema: Record<string, any>
+  export let value: Record<string, any>
+  export let customRenderers: { column: string; component: any }[] = []
+  export let snippets: any
 
-  let renderer
-  const typeMap = {
+  let renderer: any
+  const typeMap: Record<string, any> = {
     boolean: BooleanRenderer,
     datetime: DateTimeRenderer,
     link: RelationshipRenderer,
@@ -33,7 +33,7 @@
   $: renderer = customRenderer?.component ?? typeMap[type] ?? StringRenderer
   $: cellValue = getCellValue(value, schema.template)
 
-  const getType = schema => {
+  const getType = (schema: any): string => {
     // Use a string renderer for dates if we use a custom template
     if (schema?.type === "datetime" && schema?.template) {
       return "string"
@@ -41,7 +41,7 @@
     return schema?.type || "string"
   }
 
-  const getCellValue = (value, template) => {
+  const getCellValue = (value: any, template: string | undefined): any => {
     if (!template) {
       return value
     }

packages/bbui/src/Table/CodeRenderer.svelte

@@ -1,5 +1,5 @@
-<script>
-  export let value
+<script lang="ts">
+  export let value: string
 </script>
 
 <code>{value}</code>

packages/bbui/src/Table/DateTimeRenderer.svelte

@@ -1,13 +1,13 @@
-<script>
+<script lang="ts">
   import dayjs from "dayjs"
 
-  export let value
-  export let schema
+  export let value: string | Date
+  export let schema: { timeOnly?: boolean; dateOnly?: boolean }
 
   // adding the 0- will turn a string like 00:00:00 into a valid ISO
   // date, but will make actual ISO dates invalid
   $: time = new Date(`0-${value}`)
-  $: isTimeOnly = !isNaN(time) || schema?.timeOnly
+  $: isTimeOnly = !isNaN(time as any) || schema?.timeOnly
   $: isDateOnly = schema?.dateOnly
   $: format = isTimeOnly
     ? "HH:mm:ss"

packages/bbui/src/Table/InternalRenderer.svelte

@@ -1,11 +1,11 @@
-<script>
+<script lang="ts">
   import Icon from "../Icon/Icon.svelte"
   import { copyToClipboard } from "../helpers"
   import { notifications } from "../Stores/notifications"
 
-  export let value
+  export let value: string
 
-  const onClick = async e => {
+  const onClick = async (e: MouseEvent): Promise<void> => {
     e.stopPropagation()
     try {
       await copyToClipboard(value)

packages/bbui/src/Table/RelationshipRenderer.svelte

@@ -1,11 +1,11 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/label/dist/index-vars.css"
   import { createEventDispatcher } from "svelte"
   import Badge from "../Badge/Badge.svelte"
 
-  export let row
-  export let value
-  export let schema
+  export let row: { tableId: string; _id: string }
+  export let value: { primaryDisplay?: string }[] | undefined
+  export let schema: { name?: string }
 
   const dispatch = createEventDispatcher()
   const displayLimit = 5
@@ -13,7 +13,7 @@
   $: relationships = value?.slice(0, displayLimit) ?? []
   $: leftover = (value?.length ?? 0) - relationships.length
 
-  const onClick = e => {
+  const onClick = (e: MouseEvent) => {
     e.stopPropagation()
     dispatch("clickrelationship", {
       tableId: row.tableId,

packages/bbui/src/Table/SelectEditRenderer.svelte

@@ -1,12 +1,12 @@
-<script>
+<script lang="ts">
   import Checkbox from "../Form/Checkbox.svelte"
   import ActionButton from "../ActionButton/ActionButton.svelte"
 
-  export let selected
-  export let onEdit
-  export let allowSelectRows = false
-  export let allowEditRows = false
-  export let data
+  export let selected: boolean | undefined
+  export let onEdit: (_e: Event) => void
+  export let allowSelectRows: boolean = false
+  export let allowEditRows: boolean = false
+  export let data: Record<string, any>
 </script>
 
 <div>

packages/bbui/src/Table/StringRenderer.svelte

@@ -1,6 +1,6 @@
-<script>
-  export let value
-  export let schema
+<script lang="ts">
+  export let value: string | object
+  export let schema: { capitalise?: boolean }
 </script>
 
 <div class:capitalise={schema?.capitalise}>

packages/bbui/src/Table/Table.svelte

@@ -1,4 +1,4 @@
-<script>
+<script lang="ts">
   import { createEventDispatcher } from "svelte"
   import "@spectrum-css/table/dist/index-vars.css"
   import CellRenderer from "./CellRenderer.svelte"
@@ -8,6 +8,7 @@
   import Checkbox from "../Form/Checkbox.svelte"
 
   /**
+   /**
    * The expected schema is our normal couch schemas for our tables.
    * Each field schema can be enriched with a few extra properties to customise
    * the behaviour.
@@ -24,42 +25,42 @@
    * borderLeft: show a left border
    * borderRight: show a right border
    */
-  export let data = []
-  export let schema = {}
-  export let showAutoColumns = false
-  export let rowCount = 0
-  export let quiet = false
-  export let loading = false
-  export let allowSelectRows
-  export let allowEditRows = true
-  export let allowEditColumns = true
-  export let allowClickRows = true
-  export let selectedRows = []
-  export let customRenderers = []
-  export let disableSorting = false
-  export let autoSortColumns = true
-  export let compact = false
-  export let customPlaceholder = false
-  export let showHeaderBorder = true
-  export let placeholderText = "No rows found"
-  export let snippets = []
-  export let defaultSortColumn = undefined
-  export let defaultSortOrder = "Ascending"
+  export let data: any[] = []
+  export let schema: Record<string, any> = {}
+  export let showAutoColumns: boolean = false
+  export let rowCount: number = 0
+  export let quiet: boolean = false
+  export let loading: boolean = false
+  export let allowSelectRows: boolean = false
+  export let allowEditRows: boolean = true
+  export let allowEditColumns: boolean = true
+  export let allowClickRows: boolean = true
+  export let selectedRows: any[] = []
+  export let customRenderers: any[] = []
+  export let disableSorting: boolean = false
+  export let autoSortColumns: boolean = true
+  export let compact: boolean = false
+  export let customPlaceholder: boolean = false
+  export let showHeaderBorder: boolean = true
+  export let placeholderText: string = "No rows found"
+  export let snippets: any[] = []
+  export let defaultSortColumn: string | undefined = undefined
+  export let defaultSortOrder: "Ascending" | "Descending" = "Ascending"
 
   const dispatch = createEventDispatcher()
 
   // Config
-  const headerHeight = 36
+  const headerHeight: number = 36
   $: rowHeight = compact ? 46 : 55
 
   // Sorting state
-  let sortColumn
-  let sortOrder
+  let sortColumn: string | undefined
+  let sortOrder: "Ascending" | "Descending" | undefined
 
   // Table state
-  let height = 0
-  let loaded = false
-  let checkboxStatus = false
+  let height: number = 0
+  let loaded: boolean = false
+  let checkboxStatus: boolean = false
 
   $: schema = fixSchema(schema)
   $: if (!loading) loaded = true
@@ -95,8 +96,8 @@
     }
   }
 
-  const fixSchema = schema => {
-    let fixedSchema = {}
+  const fixSchema = (schema: Record<string, any>): Record<string, any> => {
+    let fixedSchema: Record<string, any> = {}
     Object.entries(schema || {}).forEach(([fieldName, fieldSchema]) => {
       if (typeof fieldSchema === "string") {
         fixedSchema[fieldName] = {
@@ -120,7 +121,13 @@
     return fixedSchema
   }
 
-  const getVisibleRowCount = (loaded, height, allRows, rowCount, rowHeight) => {
+  const getVisibleRowCount = (
+    loaded: boolean,
+    height: number,
+    allRows: number,
+    rowCount: number,
+    rowHeight: number
+  ): number => {
     if (!loaded) {
       return rowCount || 0
     }
@@ -131,12 +138,12 @@
   }
 
   const getHeightStyle = (
-    visibleRowCount,
-    rowCount,
-    totalRowCount,
-    rowHeight,
-    loading
-  ) => {
+    visibleRowCount: number,
+    rowCount: number,
+    totalRowCount: number,
+    rowHeight: number,
+    loading: boolean
+  ): string => {
     if (loading) {
       return `height: ${headerHeight + visibleRowCount * rowHeight}px;`
     }
@@ -146,7 +153,11 @@
     return `height: ${headerHeight + visibleRowCount * rowHeight}px;`
   }
 
-  const getGridStyle = (fields, schema, showEditColumn) => {
+  const getGridStyle = (
+    fields: string[],
+    schema: Record<string, any>,
+    showEditColumn: boolean
+  ): string => {
     let style = "grid-template-columns:"
     if (showEditColumn) {
       style += " auto"
@@ -163,7 +174,11 @@
     return style
   }
 
-  const sortRows = (rows, sortColumn, sortOrder) => {
+  const sortRows = (
+    rows: any[],
+    sortColumn: string | undefined,
+    sortOrder: string | undefined
+  ): any[] => {
     sortColumn = sortColumn ?? defaultSortColumn
     sortOrder = sortOrder ?? defaultSortOrder
     if (!sortColumn || !sortOrder || disableSorting) {
@@ -180,7 +195,7 @@
     })
   }
 
-  const sortBy = fieldSchema => {
+  const sortBy = (fieldSchema: Record<string, any>): void => {
     if (fieldSchema.sortable === false) {
       return
     }
@@ -193,7 +208,7 @@
     dispatch("sort", { column: sortColumn, order: sortOrder })
   }
 
-  const getDisplayName = schema => {
+  const getDisplayName = (schema: Record<string, any>): string => {
     let name = schema?.displayName
     if (schema && name === undefined) {
       name = schema.name
@@ -201,9 +216,13 @@
     return name || ""
   }
 
-  const getFields = (schema, showAutoColumns, autoSortColumns) => {
-    let columns = []
-    let autoColumns = []
+  const getFields = (
+    schema: Record<string, any>,
+    showAutoColumns: boolean,
+    autoSortColumns: boolean
+  ): string[] => {
+    let columns: any[] = []
+    let autoColumns: any[] = []
     Object.entries(schema || {}).forEach(([field, fieldSchema]) => {
       if (!field || !fieldSchema) {
         return
@@ -235,17 +254,17 @@
       .map(column => column.name)
   }
 
-  const editColumn = (e, field) => {
+  const editColumn = (e: Event, field: any): void => {
     e.stopPropagation()
     dispatch("editcolumn", field)
   }
 
-  const editRow = (e, row) => {
+  const editRow = (e: Event, row: any): void => {
     e.stopPropagation()
     dispatch("editrow", cloneDeep(row))
   }
 
-  const toggleSelectRow = row => {
+  const toggleSelectRow = (row: any): void => {
     if (!allowSelectRows) {
       return
     }
@@ -258,7 +277,7 @@
     }
   }
 
-  const toggleSelectAll = e => {
+  const toggleSelectAll = (e: CustomEvent): void => {
     const select = !!e.detail
     if (select) {
       // Add any rows which are not already in selected rows
@@ -278,8 +297,10 @@
     }
   }
 
-  const computeCellStyles = schema => {
-    let styles = {}
+  const computeCellStyles = (
+    schema: Record<string, any>
+  ): Record<string, string> => {
+    let styles: Record<string, string> = {}
     Object.keys(schema || {}).forEach(field => {
       styles[field] = ""
       if (schema[field].color) {

packages/bbui/src/Tabs/Tabs.svelte

@@ -1,28 +1,48 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/tabs/dist/index-vars.css"
   import { writable } from "svelte/store"
   import { onMount, setContext, createEventDispatcher } from "svelte"
 
-  export let selected
-  export let vertical = false
-  export let noPadding = false
-  // added as a separate option as noPadding is used for vertical padding
-  export let noHorizPadding = false
-  export let quiet = false
-  export let emphasized = false
-  export let onTop = false
-  export let size = "M"
-  export let beforeSwitch = null
+  interface TabInfo {
+    width?: number
+    height?: number
+    left?: number
+    top?: number
+  }
 
-  let thisSelected = undefined
+  interface TabState {
+    title: string
+    id: string
+    emphasized: boolean
+    info?: TabInfo
+  }
+
+  export let selected: string
+  export let vertical: boolean = false
+  export let noPadding: boolean = false
+  // added as a separate option as noPadding is used for vertical padding
+  export let noHorizPadding: boolean = false
+  export let quiet: boolean = false
+  export let emphasized: boolean = false
+  export let onTop: boolean = false
+  export let size: "S" | "M" | "L" = "M"
+  export let beforeSwitch: ((_title: string) => boolean) | null = null
+
+  let thisSelected: string | undefined = undefined
+  let container: HTMLElement | undefined
 
   let _id = id()
-  const tab = writable({ title: selected, id: _id, emphasized })
+  const tab = writable<TabState>({ title: selected, id: _id, emphasized })
   setContext("tab", tab)
 
-  let container
+  let top: string | undefined
+  let left: string | undefined
+  let width: string | undefined
+  let height: string | undefined
 
-  const dispatch = createEventDispatcher()
+  const dispatch = createEventDispatcher<{
+    select: string
+  }>()
 
   $: {
     if (thisSelected !== selected) {
@@ -44,29 +64,34 @@
     }
     if ($tab.title !== thisSelected) {
       tab.update(state => {
-        state.title = thisSelected
+        state.title = thisSelected as string
         return state
       })
     }
   }
 
-  let top, left, width, height
-  $: calculateIndicatorLength($tab)
-  $: calculateIndicatorOffset($tab)
+  $: $tab && calculateIndicatorLength()
+  $: $tab && calculateIndicatorOffset()
 
   function calculateIndicatorLength() {
     if (!vertical) {
-      width = $tab.info?.width + "px"
+      width = ($tab.info?.width ?? 0) + "px"
     } else {
-      height = $tab.info?.height + 4 + "px"
+      height = ($tab.info?.height ?? 0) + 4 + "px"
     }
   }
 
   function calculateIndicatorOffset() {
     if (!vertical) {
-      left = $tab.info?.left - container?.getBoundingClientRect().left + "px"
+      left =
+        ($tab.info?.left ?? 0) -
+        (container?.getBoundingClientRect().left ?? 0) +
+        "px"
     } else {
-      top = $tab.info?.top - container?.getBoundingClientRect().top + "px"
+      top =
+        ($tab.info?.top ?? 0) -
+        (container?.getBoundingClientRect().top ?? 0) +
+        "px"
     }
   }
 
@@ -75,7 +100,7 @@
     calculateIndicatorOffset()
   })
 
-  function id() {
+  function id(): string {
     return "_" + Math.random().toString(36).slice(2, 9)
   }
 </script>

packages/bbui/src/Tags/Tag.svelte

@@ -1,13 +1,13 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/tags/dist/index-vars.css"
   import Avatar from "../Avatar/Avatar.svelte"
   import ClearButton from "../ClearButton/ClearButton.svelte"
 
-  export let icon = ""
-  export let avatar = ""
-  export let invalid = false
-  export let disabled = false
-  export let closable = false
+  export let icon: string = ""
+  export let avatar: string = ""
+  export let invalid: boolean = false
+  export let disabled: boolean = false
+  export let closable: boolean = false
 </script>
 
 <div

packages/bbui/src/Tags/Tags.svelte

@@ -1,4 +1,4 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/tags/dist/index-vars.css"
 </script>
 

packages/bbui/src/Tooltip/TooltipWrapper.svelte

@@ -1,10 +1,10 @@
-<script>
+<script lang="ts">
   import Icon from "../Icon/Icon.svelte"
   import AbsTooltip from "./AbsTooltip.svelte"
 
-  export let tooltip = ""
-  export let size = "M"
-  export let disabled = true
+  export let tooltip: string = ""
+  export let size: "S" | "M" = "M"
+  export let disabled: boolean = true
 </script>
 
 <div class:container={!!tooltip}>

packages/bbui/src/TreeView/Item.svelte

@@ -1,9 +1,9 @@
-<script>
-  export let selected = false
-  export let open = false
-  export let href = false
-  export let title
-  export let icon
+<script lang="ts">
+  export let selected: boolean = false
+  export let open: boolean = false
+  export let href: string | null = null
+  export let title: string
+  export let icon: string | undefined
 </script>
 
 <li

packages/bbui/src/TreeView/Tree.svelte

@@ -1,9 +1,9 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/treeview/dist/index-vars.css"
 
-  export let quiet = false
-  export let standalone = true
-  export let width = "250px"
+  export let quiet: boolean = false
+  export let standalone: boolean = true
+  export let width: string = "250px"
 </script>
 
 <ul

packages/bbui/src/Typography/Code.svelte

@@ -1,8 +1,8 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/typography/dist/index-vars.css"
 
   // Sizes
-  export let size = "M"
+  export let size: "S" | "M" | "L" = "M"
 </script>
 
 <code class="spectrum-Code spectrum-Code--size{size}">

packages/bbui/src/Typography/Detail.svelte

@@ -1,9 +1,9 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/typography/dist/index-vars.css"
 
-  export let size = "M"
-  export let serif = false
-  export let weight = 600
+  export let size: "S" | "M" | "L" = "M"
+  export let serif: boolean = false
+  export let weight: number | null = null
 </script>
 
 <p

packages/bbui/src/index.ts

@@ -65,8 +65,6 @@ export { default as Modal } from "./Modal/Modal.svelte"
 export { default as ModalContent, keepOpen } from "./Modal/ModalContent.svelte"
 export { default as NotificationDisplay } from "./Notification/NotificationDisplay.svelte"
 export { default as Notification } from "./Notification/Notification.svelte"
-export { default as SideNavigation } from "./SideNavigation/Navigation.svelte"
-export { default as SideNavigationItem } from "./SideNavigation/Item.svelte"
 export { default as Context } from "./context"
 export { default as Table } from "./Table/Table.svelte"
 export { default as Tabs } from "./Tabs/Tabs.svelte"

packages/builder/src/components/backend/DataTable/formula.js

@@ -2,6 +2,7 @@ import { FieldType } from "@budibase/types"
 import { FIELDS } from "@/constants/backend"
 import { tables } from "@/stores/builder"
 import { get as svelteGet } from "svelte/store"
+import { makeReadableKeyPropSafe } from "@/dataBinding"
 
 // currently supported level of relationship depth (server side)
 const MAX_DEPTH = 1
@@ -26,7 +27,7 @@ export function getBindings({
   if (!table) {
     return bindings
   }
-  for (let [column, schema] of Object.entries(table.schema)) {
+  for (const [column, schema] of Object.entries(table.schema)) {
     const isRelationship = schema.type === FieldType.LINK
     // skip relationships after a certain depth and types which
     // can't bind to
@@ -62,6 +63,10 @@ export function getBindings({
 
     const label = path == null ? column : `${path}.0.${column}`
     const binding = path == null ? `[${column}]` : `[${path}].0.[${column}]`
+    const readableBinding = (path == null ? [column] : [path, "0", column])
+      .map(makeReadableKeyPropSafe)
+      .join(".")
+
     // only supply a description for relationship paths
     const description =
       path == null
@@ -75,7 +80,7 @@ export function getBindings({
       description,
       // don't include path, it messes things up, relationship path
       // will be replaced by the main array binding
-      readableBinding: label,
+      readableBinding,
       runtimeBinding: binding,
       display: {
         name: label,

packages/builder/src/components/common/CodeEditor/CodeEditor.svelte

@@ -49,6 +49,7 @@
   import type { EditorMode } from "@budibase/types"
   import type { BindingCompletion, CodeValidator } from "@/types"
   import { validateHbsTemplate } from "./validator/hbs"
+  import { validateJsTemplate } from "./validator/js"
 
   export let label: string | undefined = undefined
   export let completions: BindingCompletion[] = []
@@ -356,6 +357,9 @@
     if (mode === EditorModes.Handlebars) {
       const diagnostics = validateHbsTemplate(value, validations)
       editor.dispatch(setDiagnostics(editor.state, diagnostics))
+    } else if (mode === EditorModes.JS) {
+      const diagnostics = validateJsTemplate(value, validations)
+      editor.dispatch(setDiagnostics(editor.state, diagnostics))
     }
   }
 

packages/builder/src/components/common/CodeEditor/validator/hbs.ts

@@ -45,7 +45,10 @@ export function validateHbsTemplate(
     ) {
       const ignoreMissing = options?.ignoreMissing || false
       nodes.forEach(node => {
-        if (isMustacheStatement(node) && isPathExpression(node.path)) {
+        if (
+          (isMustacheStatement(node) || isBlockStatement(node)) &&
+          isPathExpression(node.path)
+        ) {
           const helperName = node.path.original
 
           const from =
@@ -75,21 +78,64 @@ export function validateHbsTemplate(
               message: `Helper "${helperName}" requires a body:\n{{#${helperName} ...}} [body] {{/${helperName}}}`,
             })
             return
+          } else if (!requiresBlock && isBlockStatement(node)) {
+            diagnostics.push({
+              from,
+              to,
+              severity: "error",
+              message: `Helper "${helperName}" should not contain a body.`,
+            })
+            return
           }
 
-          const providedParams = node.params
+          let providedParamsCount = node.params.length
+          if (isBlockStatement(node)) {
+            // Block body counts as a parameter
+            providedParamsCount++
+          }
 
-          if (providedParams.length !== expectedArguments.length) {
+          const optionalArgMatcher = new RegExp(/^\[(.+)\]$/)
+          const optionalArgs = expectedArguments.filter(a =>
+            optionalArgMatcher.test(a)
+          )
+
+          if (
+            !optionalArgs.length &&
+            providedParamsCount !== expectedArguments.length
+          ) {
             diagnostics.push({
               from,
               to,
               severity: "error",
               message: `Helper "${helperName}" expects ${
                 expectedArguments.length
-              } parameters (${expectedArguments.join(", ")}), but got ${
-                providedParams.length
-              }.`,
+              } parameters (${expectedArguments.join(
+                ", "
+              )}), but got ${providedParamsCount}.`,
             })
+          } else if (optionalArgs.length) {
+            const maxArgs = expectedArguments.length
+            const minArgs = maxArgs - optionalArgs.length
+            if (
+              minArgs > providedParamsCount ||
+              maxArgs < providedParamsCount
+            ) {
+              const parameters = expectedArguments
+                .map(a => {
+                  const test = optionalArgMatcher.exec(a)
+                  if (!test?.[1]) {
+                    return a
+                  }
+                  return `${test[1]} (optional)`
+                })
+                .join(", ")
+              diagnostics.push({
+                from,
+                to,
+                severity: "error",
+                message: `Helper "${helperName}" expects between ${minArgs} to ${expectedArguments.length} parameters (${parameters}), but got ${providedParamsCount}.`,
+              })
+            }
           }
         }
 

packages/builder/src/components/common/CodeEditor/validator/js.ts

@@ -0,0 +1,101 @@
+import { Parser } from "acorn"
+import * as walk from "acorn-walk"
+
+import type { Diagnostic } from "@codemirror/lint"
+import { CodeValidator } from "@/types"
+
+export function validateJsTemplate(
+  code: string,
+  validations: CodeValidator
+): Diagnostic[] {
+  const diagnostics: Diagnostic[] = []
+
+  try {
+    const ast = Parser.parse(code, {
+      ecmaVersion: "latest",
+      locations: true,
+      allowReturnOutsideFunction: true,
+    })
+
+    const lineOffsets: number[] = []
+    let offset = 0
+    for (const line of code.split("\n")) {
+      lineOffsets.push(offset)
+      offset += line.length + 1 // +1 for newline character
+    }
+
+    let hasReturnStatement = false
+    walk.ancestor(ast, {
+      ReturnStatement(node, _state, ancestors) {
+        if (
+          // it returns a value
+          node.argument &&
+          // and it is top level
+          ancestors.length === 2 &&
+          ancestors[0].type === "Program" &&
+          ancestors[1].type === "ReturnStatement"
+        ) {
+          hasReturnStatement = true
+        }
+      },
+      CallExpression(node) {
+        const callee: any = node.callee
+        if (
+          node.type === "CallExpression" &&
+          callee.object?.name === "helpers" &&
+          node.loc
+        ) {
+          const functionName = callee.property.name
+          const from =
+            lineOffsets[node.loc.start.line - 1] + node.loc.start.column
+          const to = lineOffsets[node.loc.end.line - 1] + node.loc.end.column
+
+          if (!(functionName in validations)) {
+            diagnostics.push({
+              from,
+              to,
+              severity: "warning",
+              message: `"${functionName}" function does not exist.`,
+            })
+            return
+          }
+
+          const { arguments: expectedArguments } = validations[functionName]
+          if (
+            expectedArguments &&
+            node.arguments.length !== expectedArguments.length
+          ) {
+            diagnostics.push({
+              from,
+              to,
+              severity: "error",
+              message: `Function "${functionName}" expects ${
+                expectedArguments.length
+              } parameters (${expectedArguments.join(", ")}), but got ${
+                node.arguments.length
+              }.`,
+            })
+          }
+        }
+      },
+    })
+
+    if (!hasReturnStatement) {
+      diagnostics.push({
+        from: 0,
+        to: code.length,
+        severity: "error",
+        message: "Your code must return a value.",
+      })
+    }
+  } catch (e: any) {
+    diagnostics.push({
+      from: 0,
+      to: code.length,
+      severity: "error",
+      message: `Syntax error: ${e.message}`,
+    })
+  }
+
+  return diagnostics
+}

packages/builder/src/components/common/CodeEditor/validator/tests/hbs.spec.ts

@@ -2,7 +2,7 @@ import { validateHbsTemplate } from "../hbs"
 import { CodeValidator } from "@/types"
 
 describe("hbs validator", () => {
-  it("validate empty strings", () => {
+  it("validates empty strings", () => {
     const text = ""
     const validators = {}
 
@@ -10,7 +10,7 @@ describe("hbs validator", () => {
     expect(result).toHaveLength(0)
   })
 
-  it("validate strings without hbs expressions", () => {
+  it("validates strings without hbs expressions", () => {
     const text = "first line\nand another one"
     const validators = {}
 
@@ -23,7 +23,7 @@ describe("hbs validator", () => {
       fieldName: {},
     }
 
-    it("validate valid expressions", () => {
+    it("validates valid expressions", () => {
       const text = "{{ fieldName }}"
 
       const result = validateHbsTemplate(text, validators)
@@ -98,45 +98,193 @@ describe("hbs validator", () => {
   })
 
   describe("expressions with parameters", () => {
-    const validators: CodeValidator = {
-      helperFunction: {
-        arguments: ["a", "b", "c"],
-      },
-    }
+    describe("basic expression", () => {
+      const validators: CodeValidator = {
+        helperFunction: {
+          arguments: ["a", "b", "c"],
+        },
+      }
 
-    it("validate valid params", () => {
-      const text = "{{ helperFunction 1 99 'a' }}"
+      it("validates valid params", () => {
+        const text = "{{ helperFunction 1 99 'a' }}"
 
-      const result = validateHbsTemplate(text, validators)
-      expect(result).toHaveLength(0)
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toHaveLength(0)
+      })
+
+      it("throws on too few params", () => {
+        const text = "{{ helperFunction 100 }}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toEqual([
+          {
+            from: 0,
+            message: `Helper "helperFunction" expects 3 parameters (a, b, c), but got 1.`,
+            severity: "error",
+            to: 24,
+          },
+        ])
+      })
+
+      it("throws on too many params", () => {
+        const text = "{{ helperFunction  1 99 'a' 100 }}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toEqual([
+          {
+            from: 0,
+            message: `Helper "helperFunction" expects 3 parameters (a, b, c), but got 4.`,
+            severity: "error",
+            to: 34,
+          },
+        ])
+      })
     })
 
-    it("throws on too few params", () => {
-      const text = "{{ helperFunction 100 }}"
-
-      const result = validateHbsTemplate(text, validators)
-      expect(result).toEqual([
-        {
-          from: 0,
-          message: `Helper "helperFunction" expects 3 parameters (a, b, c), but got 1.`,
-          severity: "error",
-          to: 24,
+    describe("body expressions", () => {
+      const validators: CodeValidator = {
+        bodyFunction: {
+          arguments: ["a", "b", "c"],
+          requiresBlock: true,
         },
-      ])
+        nonBodyFunction: {
+          arguments: ["a", "b"],
+        },
+      }
+
+      it("validates valid params", () => {
+        const text = "{{#bodyFunction 1 99  }}body{{/bodyFunction}}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toHaveLength(0)
+      })
+
+      it("validates empty bodies", () => {
+        const text = "{{#bodyFunction 1 99  }}{{/bodyFunction}}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toHaveLength(0)
+      })
+
+      it("validates too little parameters", () => {
+        const text = "{{#bodyFunction 1 }}{{/bodyFunction}}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toEqual([
+          {
+            from: 0,
+            message: `Helper "bodyFunction" expects 3 parameters (a, b, c), but got 2.`,
+            severity: "error",
+            to: 37,
+          },
+        ])
+      })
+
+      it("validates too many parameters", () => {
+        const text = "{{#bodyFunction 1 99 'a' 0 }}{{/bodyFunction}}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toEqual([
+          {
+            from: 0,
+            message: `Helper "bodyFunction" expects 3 parameters (a, b, c), but got 5.`,
+            severity: "error",
+            to: 46,
+          },
+        ])
+      })
+
+      it("validates non-supported body usages", () => {
+        const text = "{{#nonBodyFunction 1 99}}{{/nonBodyFunction}}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toEqual([
+          {
+            from: 0,
+            message: `Helper "nonBodyFunction" should not contain a body.`,
+            severity: "error",
+            to: 45,
+          },
+        ])
+      })
     })
 
-    it("throws on too many params", () => {
-      const text = "{{ helperFunction  1 99 'a' 100 }}"
+    describe("optional parameters", () => {
+      it("supports empty parameters", () => {
+        const validators: CodeValidator = {
+          helperFunction: {
+            arguments: ["a", "b", "[c]"],
+          },
+        }
+        const text = "{{ helperFunction 1 99 }}"
 
-      const result = validateHbsTemplate(text, validators)
-      expect(result).toEqual([
-        {
-          from: 0,
-          message: `Helper "helperFunction" expects 3 parameters (a, b, c), but got 4.`,
-          severity: "error",
-          to: 34,
-        },
-      ])
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toHaveLength(0)
+      })
+
+      it("supports valid parameters", () => {
+        const validators: CodeValidator = {
+          helperFunction: {
+            arguments: ["a", "b", "[c]"],
+          },
+        }
+        const text = "{{ helperFunction 1 99 'a' }}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toHaveLength(0)
+      })
+
+      it("returns a valid message on missing parameters", () => {
+        const validators: CodeValidator = {
+          helperFunction: {
+            arguments: ["a", "b", "[c]"],
+          },
+        }
+        const text = "{{ helperFunction 1 }}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toEqual([
+          {
+            from: 0,
+            message: `Helper "helperFunction" expects between 2 to 3 parameters (a, b, c (optional)), but got 1.`,
+            severity: "error",
+            to: 22,
+          },
+        ])
+      })
+
+      it("returns a valid message on too many parameters", () => {
+        const validators: CodeValidator = {
+          helperFunction: {
+            arguments: ["a", "b", "[c]"],
+          },
+        }
+        const text = "{{ helperFunction 1 2 3 4 }}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toEqual([
+          {
+            from: 0,
+            message: `Helper "helperFunction" expects between 2 to 3 parameters (a, b, c (optional)), but got 4.`,
+            severity: "error",
+            to: 28,
+          },
+        ])
+      })
     })
   })
+
+  it("validates wrong hbs code", () => {
+    const text = "{{#fieldName}}{{/wrong}}"
+
+    const result = validateHbsTemplate(text, {})
+    expect(result).toEqual([
+      {
+        from: 0,
+        message: `The handlebars code is not valid:\nfieldName doesn't match wrong - 1:3`,
+        severity: "error",
+        to: text.length,
+      },
+    ])
+  })
 })

packages/builder/src/components/common/CodeEditor/validator/tests/js.spec.ts

@@ -0,0 +1,156 @@
+import { validateJsTemplate } from "../js"
+import { CodeValidator } from "@/types"
+
+describe("js validator", () => {
+  it("validates valid code", () => {
+    const text = "return 7"
+    const validators = {}
+
+    const result = validateJsTemplate(text, validators)
+    expect(result).toEqual([])
+  })
+
+  it("does not validate runtime errors", () => {
+    const text = "return a"
+    const validators = {}
+
+    const result = validateJsTemplate(text, validators)
+    expect(result).toEqual([])
+  })
+
+  it("validates multiline code", () => {
+    const text = "const foo='bar'\nreturn 123"
+    const validators = {}
+
+    const result = validateJsTemplate(text, validators)
+    expect(result).toEqual([])
+  })
+
+  it("allows return not being on the last line", () => {
+    const text = "const foo='bar'\nreturn 123\nconsole.log(foo)"
+    const validators = {}
+
+    const result = validateJsTemplate(text, validators)
+    expect(result).toEqual([])
+  })
+
+  it("throws on missing return", () => {
+    const text = "const foo='bar'\nbar='foo'"
+    const validators = {}
+
+    const result = validateJsTemplate(text, validators)
+    expect(result).toEqual([
+      {
+        from: 0,
+        message: "Your code must return a value.",
+        severity: "error",
+        to: 25,
+      },
+    ])
+  })
+
+  it("checks that returns are at top level", () => {
+    const text = `
+    function call(){
+      return 1
+    }`
+    const validators = {}
+
+    const result = validateJsTemplate(text, validators)
+    expect(result).toEqual([
+      {
+        from: 0,
+        message: "Your code must return a value.",
+        severity: "error",
+        to: text.length,
+      },
+    ])
+  })
+
+  describe("helpers", () => {
+    const validators: CodeValidator = {
+      helperFunction: {
+        arguments: ["a", "b", "c"],
+      },
+    }
+
+    it("validates helpers with valid params", () => {
+      const text = "return helpers.helperFunction(1, 99, 'a')"
+
+      const result = validateJsTemplate(text, validators)
+      expect(result).toEqual([])
+    })
+
+    it("throws on too few params", () => {
+      const text = "return helpers.helperFunction(100)"
+
+      const result = validateJsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 7,
+          message: `Function "helperFunction" expects 3 parameters (a, b, c), but got 1.`,
+          severity: "error",
+          to: 34,
+        },
+      ])
+    })
+
+    it("throws on too many params", () => {
+      const text = "return helpers.helperFunction( 1, 99, 'a', 100)"
+
+      const result = validateJsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 7,
+          message: `Function "helperFunction" expects 3 parameters (a, b, c), but got 4.`,
+          severity: "error",
+          to: 47,
+        },
+      ])
+    })
+
+    it("validates helpers on inner functions", () => {
+      const text = `function call(){
+                      return helpers.helperFunction(1, 99)
+                    }
+                    return call()`
+
+      const result = validateJsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 46,
+          message: `Function "helperFunction" expects 3 parameters (a, b, c), but got 2.`,
+          severity: "error",
+          to: 75,
+        },
+      ])
+    })
+
+    it("validates multiple helpers", () => {
+      const text =
+        "return helpers.helperFunction(1, 99, 'a') + helpers.helperFunction(1) + helpers.another(1) + helpers.another()"
+      const validators: CodeValidator = {
+        helperFunction: {
+          arguments: ["a", "b", "c"],
+        },
+        another: { arguments: [] },
+      }
+
+      const result = validateJsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 44,
+          message: `Function "helperFunction" expects 3 parameters (a, b, c), but got 1.`,
+          severity: "error",
+          to: 69,
+        },
+        {
+          from: 72,
+          message: `Function "another" expects 0 parameters (), but got 1.`,
+          severity: "error",
+          to: 90,
+        },
+      ])
+    })
+  })
+})

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -27,12 +27,11 @@
   } from "../CodeEditor"
   import BindingSidePanel from "./BindingSidePanel.svelte"
   import EvaluationSidePanel from "./EvaluationSidePanel.svelte"
-  import SnippetSidePanel from "./SnippetSidePanel.svelte"
   import { BindingHelpers } from "./utils"
   import { capitalise } from "@/helpers"
   import { Utils, JsonFormatter } from "@budibase/frontend-core"
   import { licensing } from "@/stores/portal"
-  import { BindingMode, SidePanel } from "@budibase/types"
+  import { BindingMode } from "@budibase/types"
   import type {
     EnrichedBinding,
     Snippet,
@@ -44,6 +43,8 @@
   import type { Log } from "@budibase/string-templates"
   import type { CodeValidator } from "@/types"
 
+  type SidePanel = "Bindings" | "Evaluation"
+
   const dispatch = createEventDispatcher()
 
   export let bindings: EnrichedBinding[] = []
@@ -55,7 +56,7 @@
   export let context = null
   export let snippets: Snippet[] | null = null
   export let autofocusEditor = false
-  export let placeholder = null
+  export let placeholder: string | null = null
   export let showTabBar = true
 
   let mode: BindingMode
@@ -71,14 +72,13 @@
   let expressionError: string | undefined
   let evaluating = false
 
-  $: useSnippets = allowSnippets && !$licensing.isFreePlan
+  const SidePanelIcons: Record<SidePanel, string> = {
+    Bindings: "FlashOn",
+    Evaluation: "Play",
+  }
+
   $: editorModeOptions = getModeOptions(allowHBS, allowJS)
-  $: sidePanelOptions = getSidePanelOptions(
-    bindings,
-    context,
-    allowSnippets,
-    mode
-  )
+  $: sidePanelOptions = getSidePanelOptions(bindings, context)
   $: enrichedBindings = enrichBindings(bindings, context, snippets)
   $: usingJS = mode === BindingMode.JavaScript
   $: editorMode =
@@ -90,10 +90,12 @@
   $: requestEval(runtimeExpression, context, snippets)
   $: bindingHelpers = new BindingHelpers(getCaretPosition, insertAtPos)
 
-  $: bindingOptions = bindingsToCompletions(bindings, editorMode)
+  $: bindingOptions = bindingsToCompletions(enrichedBindings, editorMode)
   $: helperOptions = allowHelpers ? getHelperCompletions(editorMode) : []
   $: snippetsOptions =
-    usingJS && useSnippets && snippets?.length ? snippets : []
+    usingJS && allowSnippets && !$licensing.isFreePlan && snippets?.length
+      ? snippets
+      : []
 
   $: completions = !usingJS
     ? [hbAutocomplete([...bindingOptions, ...helperOptions])]
@@ -137,21 +139,13 @@
     return options
   }
 
-  const getSidePanelOptions = (
-    bindings: EnrichedBinding[],
-    context: any,
-    useSnippets: boolean,
-    mode: BindingMode | null
-  ) => {
-    let options = []
+  const getSidePanelOptions = (bindings: EnrichedBinding[], context: any) => {
+    let options: SidePanel[] = []
     if (bindings?.length) {
-      options.push(SidePanel.Bindings)
+      options.push("Bindings")
     }
     if (context && Object.keys(context).length > 0) {
-      options.push(SidePanel.Evaluation)
-    }
-    if (useSnippets && mode === BindingMode.JavaScript) {
-      options.push(SidePanel.Snippets)
+      options.push("Evaluation")
     }
     return options
   }
@@ -342,14 +336,15 @@
             {/each}
           </div>
           <div class="side-tabs">
-            {#each sidePanelOptions as panel}
+            {#each sidePanelOptions as panelOption}
               <ActionButton
                 size="M"
                 quiet
-                selected={sidePanel === panel}
-                on:click={() => changeSidePanel(panel)}
+                selected={sidePanel === panelOption}
+                on:click={() => changeSidePanel(panelOption)}
+                tooltip={panelOption}
               >
-                <Icon name={panel} size="S" />
+                <Icon name={SidePanelIcons[panelOption]} size="S" />
               </ActionButton>
             {/each}
           </div>
@@ -377,6 +372,7 @@
               value={jsValue ? decodeJSBinding(jsValue) : jsValue}
               on:change={onChangeJSValue}
               {completions}
+              {validations}
               mode={EditorModes.JS}
               bind:getCaretPosition
               bind:insertAtPos
@@ -414,16 +410,19 @@
       </div>
     </div>
     <div class="side" class:visible={!!sidePanel}>
-      {#if sidePanel === SidePanel.Bindings}
+      {#if sidePanel === "Bindings"}
         <BindingSidePanel
           bindings={enrichedBindings}
           {allowHelpers}
+          {allowSnippets}
           {context}
           addHelper={onSelectHelper}
           addBinding={onSelectBinding}
-          mode={editorMode}
+          {addSnippet}
+          {mode}
+          {snippets}
         />
-      {:else if sidePanel === SidePanel.Evaluation}
+      {:else if sidePanel === "Evaluation"}
         <EvaluationSidePanel
           {expressionResult}
           {expressionError}
@@ -431,8 +430,6 @@
           {evaluating}
           expression={editorValue ? editorValue : ""}
         />
-      {:else if sidePanel === SidePanel.Snippets}
-        <SnippetSidePanel {addSnippet} {snippets} />
       {/if}
     </div>
   </div>

packages/builder/src/components/common/bindings/BindingSidePanel.svelte

@@ -1,34 +1,69 @@
-<script>
+<script lang="ts">
   import groupBy from "lodash/fp/groupBy"
   import { convertToJS } from "@budibase/string-templates"
-  import { Input, Layout, Icon, Popover } from "@budibase/bbui"
+  import { licensing } from "@/stores/portal"
+  import {
+    Input,
+    Layout,
+    Icon,
+    Popover,
+    Tags,
+    Tag,
+    Body,
+    Button,
+  } from "@budibase/bbui"
   import { handlebarsCompletions } from "@/constants/completions"
+  import type { EnrichedBinding, Helper, Snippet } from "@budibase/types"
+  import { BindingMode } from "@budibase/types"
+  import { EditorModes } from "../CodeEditor"
+  import CodeEditor from "../CodeEditor/CodeEditor.svelte"
 
-  export let addHelper
-  export let addBinding
-  export let bindings
-  export let mode
-  export let allowHelpers
+  import SnippetDrawer from "./SnippetDrawer.svelte"
+  import UpgradeButton from "@/pages/builder/portal/_components/UpgradeButton.svelte"
+
+  export let addHelper: (_helper: Helper, _js?: boolean) => void
+  export let addBinding: (_binding: EnrichedBinding) => void
+  export let addSnippet: (_snippet: Snippet) => void
+  export let bindings: EnrichedBinding[]
+  export let snippets: Snippet[] | null
+  export let mode: BindingMode
+  export let allowHelpers: boolean
+  export let allowSnippets: boolean
   export let context = null
 
   let search = ""
   let searching = false
-  let popover
-  let popoverAnchor
-  let hoverTarget
+  let popover: Popover
+  let popoverAnchor: HTMLElement | undefined
+  let hoverTarget: {
+    type: "binding" | "helper" | "snippet"
+    code: string
+    description?: string
+  } | null
   let helpers = handlebarsCompletions()
-  let selectedCategory
-  let hideTimeout
+  let selectedCategory: string | null
+  let hideTimeout: ReturnType<typeof setTimeout> | null
+  let snippetDrawer: SnippetDrawer
+  let editableSnippet: Snippet | null
 
-  $: bindingIcons = bindings?.reduce((acc, ele) => {
+  $: enableSnippets = !$licensing.isFreePlan
+  $: bindingIcons = bindings?.reduce<Record<string, string>>((acc, ele) => {
     if (ele.icon) {
       acc[ele.category] = acc[ele.category] || ele.icon
     }
     return acc
   }, {})
-  $: categoryIcons = { ...bindingIcons, Helpers: "MagicWand" }
+  $: categoryIcons = {
+    ...bindingIcons,
+    Helpers: "MagicWand",
+    Snippets: "Code",
+  } as Record<string, string>
   $: categories = Object.entries(groupBy("category", bindings))
-  $: categoryNames = getCategoryNames(categories)
+
+  $: categoryNames = getCategoryNames(
+    categories,
+    allowSnippets && mode === BindingMode.JavaScript
+  )
   $: searchRgx = new RegExp(search, "ig")
   $: filteredCategories = categories
     .map(([name, categoryBindings]) => ({
@@ -48,11 +83,22 @@
       (!search ||
         helper.label.match(searchRgx) ||
         helper.description.match(searchRgx)) &&
-      (mode.name !== "javascript" || helper.allowsJs)
+      (mode !== BindingMode.JavaScript || helper.allowsJs)
     )
   })
 
-  const getHelperExample = (helper, js) => {
+  $: filteredSnippets = getFilteredSnippets(
+    enableSnippets,
+    snippets || [],
+    search
+  )
+
+  function onModeChange(_mode: BindingMode) {
+    selectedCategory = null
+  }
+  $: onModeChange(mode)
+
+  const getHelperExample = (helper: Helper, js: boolean) => {
     let example = helper.example || ""
     if (js) {
       example = convertToJS(example).split("\n")[0].split("= ")[1]
@@ -63,37 +109,46 @@
     return example || ""
   }
 
-  const getCategoryNames = categories => {
-    let names = [...categories.map(cat => cat[0])]
+  const getCategoryNames = (
+    categories: [string, EnrichedBinding[]][],
+    showSnippets: boolean
+  ) => {
+    const names = [...categories.map(cat => cat[0])]
     if (allowHelpers) {
       names.push("Helpers")
     }
+    if (showSnippets) {
+      names.push("Snippets")
+    }
     return names
   }
 
-  const showBindingPopover = (binding, target) => {
+  const showBindingPopover = (
+    binding: EnrichedBinding,
+    target: HTMLElement
+  ) => {
     if (!context || !binding.value || binding.value === "") {
       return
     }
     stopHidingPopover()
     popoverAnchor = target
     hoverTarget = {
-      helper: false,
+      type: "binding",
       code: binding.valueHTML,
     }
     popover.show()
   }
 
-  const showHelperPopover = (helper, target) => {
+  const showHelperPopover = (helper: Helper, target: HTMLElement) => {
     stopHidingPopover()
     if (!helper.displayText && helper.description) {
       return
     }
     popoverAnchor = target
     hoverTarget = {
-      helper: true,
+      type: "helper",
       description: helper.description,
-      code: getHelperExample(helper, mode.name === "javascript"),
+      code: getHelperExample(helper, mode === BindingMode.JavaScript),
     }
     popover.show()
   }
@@ -101,7 +156,7 @@
   const hidePopover = () => {
     hideTimeout = setTimeout(() => {
       popover.hide()
-      popoverAnchor = null
+      popoverAnchor = undefined
       hoverTarget = null
       hideTimeout = null
     }, 100)
@@ -119,11 +174,53 @@
     search = ""
   }
 
-  const stopSearching = e => {
+  const stopSearching = (e: Event) => {
     e.stopPropagation()
     searching = false
     search = ""
   }
+
+  const getFilteredSnippets = (
+    enableSnippets: boolean,
+    snippets: Snippet[],
+    search: string
+  ) => {
+    if (!enableSnippets || !snippets.length) {
+      return []
+    }
+    if (!search?.length) {
+      return snippets
+    }
+    return snippets.filter(snippet =>
+      snippet.name.toLowerCase().includes(search.toLowerCase())
+    )
+  }
+
+  const showSnippet = (snippet: Snippet, target: HTMLElement) => {
+    stopHidingPopover()
+    if (!snippet.code) {
+      return
+    }
+    popoverAnchor = target
+    hoverTarget = {
+      type: "snippet",
+      code: snippet.code,
+    }
+
+    popover.show()
+  }
+
+  const createSnippet = () => {
+    editableSnippet = null
+    snippetDrawer.show()
+  }
+
+  const editSnippet = (e: Event, snippet: Snippet) => {
+    e.preventDefault()
+    e.stopPropagation()
+    editableSnippet = snippet
+    snippetDrawer.show()
+  }
 </script>
 
 <Popover
@@ -137,18 +234,31 @@
   on:mouseenter={stopHidingPopover}
   on:mouseleave={hidePopover}
 >
-  <div class="binding-popover" class:helper={hoverTarget.helper}>
-    {#if hoverTarget.description}
-      <div>
-        <!-- eslint-disable-next-line svelte/no-at-html-tags-->
-        {@html hoverTarget.description}
-      </div>
-    {/if}
-    {#if hoverTarget.code}
-      <!-- eslint-disable-next-line svelte/no-at-html-tags-->
-      <pre>{@html hoverTarget.code}</pre>
-    {/if}
-  </div>
+  {#if hoverTarget}
+    <div
+      class="binding-popover"
+      class:has-code={hoverTarget.type !== "binding"}
+    >
+      {#if hoverTarget.description}
+        <div>
+          <!-- eslint-disable-next-line svelte/no-at-html-tags-->
+          {@html hoverTarget.description}
+        </div>
+      {/if}
+      {#if hoverTarget.code}
+        {#if mode === BindingMode.JavaScript}
+          <CodeEditor
+            value={hoverTarget.code?.trim()}
+            mode={EditorModes.JS}
+            readonly
+          />
+        {:else if mode === BindingMode.Text}
+          <!-- eslint-disable-next-line svelte/no-at-html-tags-->
+          <pre>{@html hoverTarget.code}</pre>
+        {/if}
+      {/if}
+    </div>
+  {/if}
 </Popover>
 
 <!-- svelte-ignore a11y-click-events-have-key-events -->
@@ -164,6 +274,25 @@
           on:click={() => (selectedCategory = null)}
         />
         {selectedCategory}
+        {#if selectedCategory === "Snippets"}
+          {#if enableSnippets}
+            <div class="add-snippet-button">
+              <Icon
+                size="S"
+                name="Add"
+                hoverable
+                newStyles
+                on:click={createSnippet}
+              />
+            </div>
+          {:else}
+            <div class="title">
+              <Tags>
+                <Tag icon="LockClosed">Premium</Tag>
+              </Tags>
+            </div>
+          {/if}
+        {/if}
       </div>
     {/if}
 
@@ -173,7 +302,7 @@
           <div class="search-input">
             <Input
               placeholder="Search for bindings"
-              autocomplete="off"
+              autocomplete={false}
               bind:value={search}
               autofocus
             />
@@ -230,7 +359,8 @@
               {#each category.bindings as binding}
                 <li
                   class="binding"
-                  on:mouseenter={e => showBindingPopover(binding, e.target)}
+                  on:mouseenter={e =>
+                    showBindingPopover(binding, e.currentTarget)}
                   on:mouseleave={hidePopover}
                   on:click={() => addBinding(binding)}
                 >
@@ -264,9 +394,10 @@
               {#each filteredHelpers as helper}
                 <li
                   class="binding"
-                  on:mouseenter={e => showHelperPopover(helper, e.target)}
-                  on:mouseleave={hidePopover}
-                  on:click={() => addHelper(helper, mode.name === "javascript")}
+                  on:mouseenter={e =>
+                    showHelperPopover(helper, e.currentTarget)}
+                  on:click={() =>
+                    addHelper(helper, mode === BindingMode.JavaScript)}
                 >
                   <span class="binding__label">{helper.displayText}</span>
                   <span class="binding__typeWrap">
@@ -278,10 +409,48 @@
           </div>
         {/if}
       {/if}
+
+      {#if selectedCategory === "Snippets" || search}
+        <div class="snippet-list">
+          {#if enableSnippets && filteredSnippets.length}
+            {#each filteredSnippets as snippet}
+              <li
+                class="snippet"
+                on:mouseenter={e => showSnippet(snippet, e.currentTarget)}
+                on:mouseleave={hidePopover}
+                on:click={() => addSnippet(snippet)}
+              >
+                {snippet.name}
+                <Icon
+                  name="Edit"
+                  hoverable
+                  newStyles
+                  size="S"
+                  on:click={e => editSnippet(e, snippet)}
+                />
+              </li>
+            {/each}
+          {:else if !search}
+            <div class="upgrade">
+              <Body size="S">
+                Snippets let you create reusable JS functions and values that
+                can all be managed in one place
+              </Body>
+              {#if enableSnippets}
+                <Button cta on:click={createSnippet}>Create snippet</Button>
+              {:else}
+                <UpgradeButton />
+              {/if}
+            </div>
+          {/if}
+        </div>
+      {/if}
     {/if}
   </Layout>
 </div>
 
+<SnippetDrawer bind:this={snippetDrawer} snippet={editableSnippet} />
+
 <style>
   .binding-side-panel {
     border-left: var(--border-light);
@@ -346,6 +515,7 @@
     display: flex;
     align-items: center;
     gap: var(--spacing-m);
+    justify-content: space-between;
   }
   li.binding .binding__typeWrap {
     flex: 1;
@@ -421,7 +591,7 @@
     text-overflow: ellipsis;
     overflow: hidden;
   }
-  .binding-popover.helper pre {
+  .binding-popover.has-code pre {
     color: var(--spectrum-global-color-blue-700);
   }
   .binding-popover pre :global(span) {
@@ -433,7 +603,50 @@
     padding: 0;
     margin: 0;
   }
-  .binding-popover.helper :global(code) {
+  .binding-popover.has-code :global(code) {
     font-size: 12px;
   }
+  .binding-popover.has-code :global(.cm-line),
+  .binding-popover.has-code :global(.cm-content) {
+    padding: 0;
+  }
+
+  /* Snippets */
+  .add-snippet-button {
+    margin-left: auto;
+  }
+  .snippet-list {
+    padding: 0 var(--spacing-l);
+    padding-bottom: var(--spacing-l);
+    display: flex;
+    flex-direction: column;
+  }
+  .snippet {
+    font-size: var(--font-size-s);
+    padding: var(--spacing-m);
+    border-radius: 4px;
+    background-color: var(--spectrum-global-color-gray-200);
+    transition: background-color 130ms ease-out, color 130ms ease-out,
+      border-color 130ms ease-out;
+    word-wrap: break-word;
+    display: flex;
+    justify-content: space-between;
+  }
+  .snippet:hover {
+    color: var(--spectrum-global-color-gray-900);
+    background-color: var(--spectrum-global-color-gray-50);
+    cursor: pointer;
+  }
+
+  /* Upgrade */
+  .upgrade {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    gap: var(--spacing-l);
+  }
+  .upgrade :global(p) {
+    text-align: center;
+    align-self: center;
+  }
 </style>

packages/builder/src/components/common/bindings/SnippetDrawer.svelte

@@ -1,4 +1,4 @@
-<script>
+<script lang="ts">
   import {
     Button,
     Drawer,
@@ -14,19 +14,27 @@
   import { getSequentialName } from "@/helpers/duplicate"
   import ConfirmDialog from "@/components/common/ConfirmDialog.svelte"
   import { ValidSnippetNameRegex } from "@budibase/shared-core"
+  import type { Snippet } from "@budibase/types"
 
-  export let snippet
-
-  export const show = () => drawer.show()
+  export const show = () => {
+    if (!snippet) {
+      key = Math.random().toString()
+      // Reset state when creating multiple snippets
+      code = ""
+      name = defaultName
+    }
+    drawer.show()
+  }
   export const hide = () => drawer.hide()
+  export let snippet: Snippet | null
 
   const firstCharNumberRegex = /^[0-9].*$/
 
-  let drawer
+  let drawer: Drawer
   let name = ""
   let code = ""
   let loading = false
-  let deleteConfirmationDialog
+  let deleteConfirmationDialog: ConfirmDialog
 
   $: defaultName = getSequentialName($snippets, "MySnippet", {
     getName: x => x.name,
@@ -40,11 +48,11 @@
   const saveSnippet = async () => {
     loading = true
     try {
-      const newSnippet = { name, code: rawJS }
+      const newSnippet: Snippet = { name, code: rawJS || "" }
       await snippets.saveSnippet(newSnippet)
       drawer.hide()
       notifications.success(`Snippet ${newSnippet.name} saved`)
-    } catch (error) {
+    } catch (error: any) {
       notifications.error(error.message || "Error saving snippet")
     }
     loading = false
@@ -53,7 +61,9 @@
   const deleteSnippet = async () => {
     loading = true
     try {
-      await snippets.deleteSnippet(snippet.name)
+      if (snippet) {
+        await snippets.deleteSnippet(snippet.name)
+      }
       drawer.hide()
     } catch (error) {
       notifications.error("Error deleting snippet")
@@ -61,7 +71,7 @@
     loading = false
   }
 
-  const validateName = (name, snippets) => {
+  const validateName = (name: string, snippets: Snippet[]) => {
     if (!name?.length) {
       return "Name is required"
     }
@@ -108,7 +118,11 @@
         Delete
       </Button>
     {/if}
-    <Button cta on:click={saveSnippet} disabled={!code || loading || nameError}>
+    <Button
+      cta
+      on:click={saveSnippet}
+      disabled={!code || loading || !!nameError}
+    >
       Save
     </Button>
   </svelte:fragment>
@@ -124,9 +138,7 @@
         value={code}
         on:change={e => (code = e.detail)}
       >
-        <div slot="tabs">
-          <Input placeholder="Name" />
-        </div>
+        <Input placeholder="Name" />
       </BindingPanel>
     {/key}
   </svelte:fragment>

packages/builder/src/components/common/bindings/SnippetSidePanel.svelte

@@ -1,278 +0,0 @@
-<script>
-  import {
-    Input,
-    Layout,
-    Icon,
-    Popover,
-    Tags,
-    Tag,
-    Body,
-    Button,
-  } from "@budibase/bbui"
-  import CodeEditor from "@/components/common/CodeEditor/CodeEditor.svelte"
-  import { EditorModes } from "@/components/common/CodeEditor"
-  import SnippetDrawer from "./SnippetDrawer.svelte"
-  import { licensing } from "@/stores/portal"
-  import UpgradeButton from "@/pages/builder/portal/_components/UpgradeButton.svelte"
-
-  export let addSnippet
-  export let snippets
-
-  let search = ""
-  let searching = false
-  let popover
-  let popoverAnchor
-  let hoveredSnippet
-  let hideTimeout
-  let snippetDrawer
-  let editableSnippet
-
-  $: enableSnippets = !$licensing.isFreePlan
-  $: filteredSnippets = getFilteredSnippets(enableSnippets, snippets, search)
-
-  const getFilteredSnippets = (enableSnippets, snippets, search) => {
-    if (!enableSnippets || !snippets?.length) {
-      return []
-    }
-    if (!search?.length) {
-      return snippets
-    }
-    return snippets.filter(snippet =>
-      snippet.name.toLowerCase().includes(search.toLowerCase())
-    )
-  }
-
-  const showSnippet = (snippet, target) => {
-    stopHidingPopover()
-    popoverAnchor = target
-    hoveredSnippet = snippet
-    popover.show()
-  }
-
-  const hidePopover = () => {
-    hideTimeout = setTimeout(() => {
-      popover.hide()
-      popoverAnchor = null
-      hoveredSnippet = null
-      hideTimeout = null
-    }, 100)
-  }
-
-  const stopHidingPopover = () => {
-    if (hideTimeout) {
-      clearTimeout(hideTimeout)
-      hideTimeout = null
-    }
-  }
-
-  const startSearching = () => {
-    searching = true
-    search = ""
-  }
-
-  const stopSearching = () => {
-    searching = false
-    search = ""
-  }
-
-  const createSnippet = () => {
-    editableSnippet = null
-    snippetDrawer.show()
-  }
-
-  const editSnippet = (e, snippet) => {
-    e.preventDefault()
-    e.stopPropagation()
-    editableSnippet = snippet
-    snippetDrawer.show()
-  }
-</script>
-
-<!-- svelte-ignore a11y-no-static-element-interactions -->
-<!-- svelte-ignore a11y-click-events-have-key-events -->
-<div class="snippet-side-panel">
-  <Layout noPadding gap="S">
-    <div class="header">
-      {#if enableSnippets}
-        {#if searching}
-          <div class="search-input">
-            <Input
-              placeholder="Search for snippets"
-              autocomplete="off"
-              bind:value={search}
-              autofocus
-            />
-          </div>
-          <Icon
-            size="S"
-            name="Close"
-            hoverable
-            newStyles
-            on:click={stopSearching}
-          />
-        {:else}
-          <div class="title">Snippets</div>
-          <Icon
-            size="S"
-            name="Search"
-            hoverable
-            newStyles
-            on:click={startSearching}
-          />
-          <Icon
-            size="S"
-            name="Add"
-            hoverable
-            newStyles
-            on:click={createSnippet}
-          />
-        {/if}
-      {:else}
-        <div class="title">
-          Snippets
-          <Tags>
-            <Tag icon="LockClosed">Premium</Tag>
-          </Tags>
-        </div>
-      {/if}
-    </div>
-    <div class="snippet-list">
-      {#if enableSnippets && filteredSnippets?.length}
-        {#each filteredSnippets as snippet}
-          <div
-            class="snippet"
-            on:mouseenter={e => showSnippet(snippet, e.target)}
-            on:mouseleave={hidePopover}
-            on:click={() => addSnippet(snippet)}
-          >
-            {snippet.name}
-            <Icon
-              name="Edit"
-              hoverable
-              newStyles
-              size="S"
-              on:click={e => editSnippet(e, snippet)}
-            />
-          </div>
-        {/each}
-      {:else}
-        <div class="upgrade">
-          <Body size="S">
-            Snippets let you create reusable JS functions and values that can
-            all be managed in one place
-          </Body>
-          {#if enableSnippets}
-            <Button cta on:click={createSnippet}>Create snippet</Button>
-          {:else}
-            <UpgradeButton />
-          {/if}
-        </div>
-      {/if}
-    </div>
-  </Layout>
-</div>
-
-<Popover
-  align="left-outside"
-  bind:this={popover}
-  anchor={popoverAnchor}
-  minWidth={0}
-  maxWidth={480}
-  maxHeight={480}
-  dismissible={false}
-  on:mouseenter={stopHidingPopover}
-  on:mouseleave={hidePopover}
->
-  <div class="snippet-popover">
-    {#key hoveredSnippet}
-      <CodeEditor
-        value={hoveredSnippet.code?.trim()}
-        mode={EditorModes.JS}
-        readonly
-      />
-    {/key}
-  </div>
-</Popover>
-
-<SnippetDrawer bind:this={snippetDrawer} snippet={editableSnippet} />
-
-<style>
-  .snippet-side-panel {
-    border-left: var(--border-light);
-    height: 100%;
-    overflow: auto;
-  }
-
-  /* Header */
-  .header {
-    height: 53px;
-    padding: 0 var(--spacing-l);
-    display: flex;
-    align-items: center;
-    border-bottom: var(--border-light);
-    position: sticky;
-    top: 0;
-    gap: var(--spacing-m);
-    background: var(--background);
-    z-index: 1;
-  }
-  .header :global(input) {
-    border: none;
-    border-radius: 0;
-    background: none;
-    padding: 0;
-  }
-  .search-input,
-  .title {
-    flex: 1 1 auto;
-  }
-  .title {
-    display: flex;
-    flex-direction: row;
-    justify-content: flex-start;
-    align-items: center;
-    gap: var(--spacing-m);
-  }
-
-  /* Upgrade */
-  .upgrade {
-    display: flex;
-    flex-direction: column;
-    align-items: center;
-    gap: var(--spacing-l);
-  }
-  .upgrade :global(p) {
-    text-align: center;
-    align-self: center;
-  }
-
-  /* List */
-  .snippet-list {
-    padding: 0 var(--spacing-l);
-    padding-bottom: var(--spacing-l);
-    display: flex;
-    flex-direction: column;
-    gap: var(--spacing-s);
-  }
-  .snippet {
-    font-size: var(--font-size-s);
-    padding: var(--spacing-m);
-    border-radius: 4px;
-    background-color: var(--spectrum-global-color-gray-200);
-    transition: background-color 130ms ease-out, color 130ms ease-out,
-      border-color 130ms ease-out;
-    word-wrap: break-word;
-    display: flex;
-    justify-content: space-between;
-  }
-  .snippet:hover {
-    color: var(--spectrum-global-color-gray-900);
-    background-color: var(--spectrum-global-color-gray-50);
-    cursor: pointer;
-  }
-
-  /* Popover */
-  .snippet-popover {
-    width: 400px;
-  }
-</style>

packages/builder/src/components/common/bindings/index.js

@@ -8,5 +8,3 @@ export { default as DrawerBindableSlot } from "./DrawerBindableSlot.svelte"
 export { default as EvaluationSidePanel } from "./EvaluationSidePanel.svelte"
 export { default as ModalBindableInput } from "./ModalBindableInput.svelte"
 export { default as ServerBindingPanel } from "./ServerBindingPanel.svelte"
-export { default as SnippetDrawer } from "./SnippetDrawer.svelte"
-export { default as SnippetSidePanel } from "./SnippetSidePanel.svelte"

packages/builder/src/components/design/settings/controls/URLVariableTestInput.svelte

@@ -2,7 +2,7 @@
   import { onMount } from "svelte"
   import { Input, Label } from "@budibase/bbui"
   import { previewStore, selectedScreen } from "@/stores/builder"
-  import type { ComponentContext } from "@budibase/types"
+  import type { AppContext } from "@budibase/types"
 
   export let baseRoute = ""
 
@@ -31,7 +31,7 @@
 
   // This function is needed to repopulate the test value from componentContext
   // when a user navigates to another component and then back again
-  const updateTestValueFromContext = (context: ComponentContext | null) => {
+  const updateTestValueFromContext = (context: AppContext | null) => {
     if (context?.url && !testValue) {
       const { wild, ...urlParams } = context.url
       const queryParams = context.query

packages/builder/src/constants/completions.ts

@@ -1,10 +1,10 @@
 import { getManifest, helpersToRemoveForJs } from "@budibase/string-templates"
+import { Helper } from "@budibase/types"
 
-export function handlebarsCompletions() {
+export function handlebarsCompletions(): Helper[] {
   const manifest = getManifest()
-
-  return Object.keys(manifest).flatMap(key =>
-    Object.entries(manifest[key]).map(([helperName, helperConfig]) => ({
+  return Object.values(manifest).flatMap(helpersObj =>
+    Object.entries(helpersObj).map<Helper>(([helperName, helperConfig]) => ({
       text: helperName,
       path: helperName,
       example: helperConfig.example,
@@ -14,6 +14,7 @@ export function handlebarsCompletions() {
       allowsJs:
         !helperConfig.requiresBlock &&
         !helpersToRemoveForJs.includes(helperName),
+      args: helperConfig.args,
     }))
   )
 }

packages/builder/src/dataBinding.js

@@ -373,6 +373,18 @@ const getContextBindings = (asset, componentId) => {
     .flat()
 }
 
+export const makeReadableKeyPropSafe = key => {
+  if (!key.includes(" ")) {
+    return key
+  }
+
+  if (new RegExp(/^\[(.+)\]$/).test(key.test)) {
+    return key
+  }
+
+  return `[${key}]`
+}
+
 /**
  * Generates a set of bindings for a given component context
  */
@@ -457,11 +469,11 @@ const generateComponentContextBindings = (asset, componentContext) => {
       const runtimeBinding = `${safeComponentId}.${safeKey}`
 
       // Optionally use a prefix with readable bindings
-      let readableBinding = component._instanceName
+      let readableBinding = makeReadableKeyPropSafe(component._instanceName)
       if (readablePrefix) {
         readableBinding += `.${readablePrefix}`
       }
-      readableBinding += `.${fieldSchema.name || key}`
+      readableBinding += `.${makeReadableKeyPropSafe(fieldSchema.name || key)}`
 
       // Determine which category this binding belongs in
       const bindingCategory = getComponentBindingCategory(
@@ -473,7 +485,7 @@ const generateComponentContextBindings = (asset, componentContext) => {
       bindings.push({
         type: "context",
         runtimeBinding,
-        readableBinding: `${readableBinding}`,
+        readableBinding,
         // Field schema and provider are required to construct relationship
         // datasource options, based on bindable properties
         fieldSchema,
@@ -1354,13 +1366,14 @@ const bindingReplacement = (
   }
   // work from longest to shortest
   const convertFromProps = bindableProperties
+    // TODO check whitespaces
     .map(el => el[convertFrom])
     .sort((a, b) => {
       return b.length - a.length
     })
   const boundValues = textWithBindings.match(regex) || []
   let result = textWithBindings
-  for (let boundValue of boundValues) {
+  for (const boundValue of boundValues) {
     let newBoundValue = boundValue
     // we use a search string, where any time we replace something we blank it out
     // in the search, working from longest to shortest so always use best match first

packages/builder/src/helpers/components.ts

@@ -8,27 +8,32 @@ import {
 } from "@budibase/string-templates"
 import { capitalise } from "@/helpers"
 import { Constants } from "@budibase/frontend-core"
+import { Component, ComponentContext } from "@budibase/types"
 
 const { ContextScopes } = Constants
 
 /**
  * Recursively searches for a specific component ID
  */
-export const findComponent = (rootComponent, id) => {
+export const findComponent = (rootComponent: Component, id: string) => {
   return searchComponentTree(rootComponent, comp => comp._id === id)
 }
 
 /**
  * Recursively searches for a specific component type
  */
-export const findComponentType = (rootComponent, type) => {
+export const findComponentType = (rootComponent: Component, type: string) => {
   return searchComponentTree(rootComponent, comp => comp._component === type)
 }
 
 /**
  * Recursively searches for the parent component of a specific component ID
  */
-export const findComponentParent = (rootComponent, id, parentComponent) => {
+export const findComponentParent = (
+  rootComponent: Component | undefined,
+  id: string | undefined,
+  parentComponent: Component | null = null
+): Component | null => {
   if (!rootComponent || !id) {
     return null
   }
@@ -51,7 +56,11 @@ export const findComponentParent = (rootComponent, id, parentComponent) => {
  * Recursively searches for a specific component ID and records the component
  * path to this component
  */
-export const findComponentPath = (rootComponent, id, path = []) => {
+export const findComponentPath = (
+  rootComponent: Component,
+  id: string | undefined,
+  path: Component[] = []
+): Component[] => {
   if (!rootComponent || !id) {
     return []
   }
@@ -75,11 +84,14 @@ export const findComponentPath = (rootComponent, id, path = []) => {
  * Recurses through the component tree and finds all components which match
  * a certain selector
  */
-export const findAllMatchingComponents = (rootComponent, selector) => {
+export const findAllMatchingComponents = (
+  rootComponent: Component | null,
+  selector: (component: Component) => boolean
+) => {
   if (!rootComponent || !selector) {
     return []
   }
-  let components = []
+  let components: Component[] = []
   if (rootComponent._children) {
     rootComponent._children.forEach(child => {
       components = [
@@ -97,7 +109,7 @@ export const findAllMatchingComponents = (rootComponent, selector) => {
 /**
  * Recurses through the component tree and finds all components.
  */
-export const findAllComponents = rootComponent => {
+export const findAllComponents = (rootComponent: Component) => {
   return findAllMatchingComponents(rootComponent, () => true)
 }
 
@@ -105,9 +117,9 @@ export const findAllComponents = rootComponent => {
  * Finds the closest parent component which matches certain criteria
  */
 export const findClosestMatchingComponent = (
-  rootComponent,
-  componentId,
-  selector
+  rootComponent: Component,
+  componentId: string | undefined,
+  selector: (component: Component) => boolean
 ) => {
   if (!selector) {
     return null
@@ -125,7 +137,10 @@ export const findClosestMatchingComponent = (
  * Recurses through a component tree evaluating a matching function against
  * components until a match is found
  */
-const searchComponentTree = (rootComponent, matchComponent) => {
+const searchComponentTree = (
+  rootComponent: Component,
+  matchComponent: (component: Component) => boolean
+): Component | null => {
   if (!rootComponent || !matchComponent) {
     return null
   }
@@ -150,15 +165,18 @@ const searchComponentTree = (rootComponent, matchComponent) => {
  * This mutates the object in place.
  * @param component the component to randomise
  */
-export const makeComponentUnique = component => {
+export const makeComponentUnique = (component: Component) => {
   if (!component) {
     return
   }
 
   // Generate a full set of component ID replacements in this tree
-  const idReplacements = []
-  const generateIdReplacements = (component, replacements) => {
-    const oldId = component._id
+  const idReplacements: [string, string][] = []
+  const generateIdReplacements = (
+    component: Component,
+    replacements: [string, string][]
+  ) => {
+    const oldId = component._id!
     const newId = Helpers.uuid()
     replacements.push([oldId, newId])
     component._children?.forEach(x => generateIdReplacements(x, replacements))
@@ -182,9 +200,9 @@ export const makeComponentUnique = component => {
     let js = decodeJSBinding(sanitizedBinding)
     if (js != null) {
       // Replace ID inside JS binding
-      idReplacements.forEach(([oldId, newId]) => {
+      for (const [oldId, newId] of idReplacements) {
         js = js.replace(new RegExp(oldId, "g"), newId)
-      })
+      }
 
       // Create new valid JS binding
       let newBinding = encodeJSBinding(js)
@@ -204,7 +222,7 @@ export const makeComponentUnique = component => {
   return JSON.parse(definition)
 }
 
-export const getComponentText = component => {
+export const getComponentText = (component: Component) => {
   if (component == null) {
     return ""
   }
@@ -218,7 +236,7 @@ export const getComponentText = component => {
   return capitalise(type)
 }
 
-export const getComponentName = component => {
+export const getComponentName = (component: Component) => {
   if (component == null) {
     return ""
   }
@@ -229,9 +247,9 @@ export const getComponentName = component => {
 }
 
 // Gets all contexts exposed by a certain component type, including actions
-export const getComponentContexts = component => {
+export const getComponentContexts = (component: string) => {
   const def = componentStore.getDefinition(component)
-  let contexts = []
+  let contexts: ComponentContext[] = []
   if (def?.context) {
     contexts = Array.isArray(def.context) ? [...def.context] : [def.context]
   }
@@ -251,9 +269,9 @@ export const getComponentContexts = component => {
  * Recurses through the component tree and builds a tree of contexts provided
  * by components.
  */
-export const buildContextTree = (
-  rootComponent,
-  tree = { root: [] },
+const buildContextTree = (
+  rootComponent: Component,
+  tree: Record<string, string[]> = { root: [] },
   currentBranch = "root"
 ) => {
   // Sanity check
@@ -264,12 +282,12 @@ export const buildContextTree = (
   // Process this component's contexts
   const contexts = getComponentContexts(rootComponent._component)
   if (contexts.length) {
-    tree[currentBranch].push(rootComponent._id)
+    tree[currentBranch].push(rootComponent._id!)
 
     // If we provide local context, start a new branch for our children
     if (contexts.some(context => context.scope === ContextScopes.Local)) {
-      currentBranch = rootComponent._id
-      tree[rootComponent._id] = []
+      currentBranch = rootComponent._id!
+      tree[rootComponent._id!] = []
     }
   }
 
@@ -287,9 +305,9 @@ export const buildContextTree = (
  * Generates a lookup map of which context branch all components in a component
  * tree are inside.
  */
-export const buildContextTreeLookupMap = rootComponent => {
+export const buildContextTreeLookupMap = (rootComponent: Component) => {
   const tree = buildContextTree(rootComponent)
-  let map = {}
+  const map: Record<string, string> = {}
   Object.entries(tree).forEach(([branch, ids]) => {
     ids.forEach(id => {
       map[id] = branch
@@ -299,9 +317,9 @@ export const buildContextTreeLookupMap = rootComponent => {
 }
 
 // Get a flat list of ids for all descendants of a component
-export const getChildIdsForComponent = component => {
+export const getChildIdsForComponent = (component: Component): string[] => {
   return [
-    component._id,
+    component._id!,
     ...(component?._children ?? []).map(getChildIdsForComponent).flat(1),
   ]
 }

packages/builder/src/pages/builder/app/[application]/_components/BuilderGroupPopover.svelte

@@ -0,0 +1,31 @@
+<script>
+  import { Icon } from "@budibase/bbui"
+  import { helpers } from "@budibase/shared-core"
+
+  export let groups = []
+  function tooltip(groups) {
+    const sortedNames = groups
+      .sort((a, b) => a.name.localeCompare(b.name))
+      .map(group => group.name)
+    return `Member of ${helpers.lists.punctuateList(sortedNames)}`
+  }
+</script>
+
+<div class="icon">
+  <Icon
+    name="Info"
+    size="XS"
+    color="grey"
+    hoverable
+    tooltip={tooltip(groups)}
+    tooltipPosition="top"
+  />
+</div>
+
+<style>
+  .icon {
+    height: auto;
+    display: flex;
+    justify-content: center;
+  }
+</style>

packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte

@@ -37,6 +37,7 @@
   import { emailValidator } from "@/helpers/validation"
   import { fly } from "svelte/transition"
   import InfoDisplay from "../design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte"
+  import BuilderGroupPopover from "./BuilderGroupPopover.svelte"
 
   let query = null
   let loaded = false
@@ -197,12 +198,19 @@
       return
     }
     const update = await users.get(user._id)
+    const newRoles = {
+      ...update.roles,
+      [prodAppId]: role,
+    }
+    // make sure no undefined/null roles (during removal)
+    for (let [appId, role] of Object.entries(newRoles)) {
+      if (!role) {
+        delete newRoles[appId]
+      }
+    }
     await users.save({
       ...update,
-      roles: {
-        ...update.roles,
-        [prodAppId]: role,
-      },
+      roles: newRoles,
     })
     await searchUsers(query, $builderStore.builderSidePanel, loaded)
   }
@@ -539,6 +547,10 @@
       creationAccessType = Constants.Roles.CREATOR
     }
   }
+
+  const itemCountText = (word, count) => {
+    return `${count} ${word}${count !== 1 ? "s" : ""}`
+  }
 </script>
 
 <svelte:window on:keydown={handleKeyDown} />
@@ -701,13 +713,11 @@
                 >
                   <div class="details">
                     <GroupIcon {group} size="S" />
-                    <div>
+                    <div class="group-name">
                       {group.name}
                     </div>
                     <div class="auth-entity-meta">
-                      {`${group.users?.length} user${
-                        group.users?.length != 1 ? "s" : ""
-                      }`}
+                      {itemCountText("user", group.users?.length)}
                     </div>
                   </div>
                   <div class="auth-entity-access">
@@ -741,16 +751,33 @@
                 <div class="auth-entity-access-title">Access</div>
               </div>
               {#each allUsers as user}
+                {@const userGroups = sdk.users.getUserAppGroups(
+                  $appStore.appId,
+                  user,
+                  $groups
+                )}
                 <div class="auth-entity">
                   <div class="details">
-                    <div class="user-email" title={user.email}>
-                      {user.email}
+                    <div class="user-groups">
+                      <div class="user-email" title={user.email}>
+                        {user.email}
+                      </div>
+                      {#if userGroups.length}
+                        <div class="group-info">
+                          <div class="auth-entity-meta">
+                            {itemCountText("group", userGroups.length)}
+                          </div>
+                          <BuilderGroupPopover groups={userGroups} />
+                        </div>
+                      {/if}
                     </div>
                   </div>
                   <div class="auth-entity-access" class:muted={user.group}>
                     <RoleSelect
                       footer={getRoleFooter(user)}
-                      placeholder={false}
+                      placeholder={userGroups?.length
+                        ? "Controlled by group"
+                        : false}
                       value={parseRole(user)}
                       allowRemove={user.role && !user.group}
                       allowPublic={false}
@@ -915,6 +942,7 @@
     color: var(--spectrum-global-color-gray-600);
     font-size: 12px;
     white-space: nowrap;
+    text-align: end;
   }
 
   .auth-entity-access {
@@ -931,7 +959,7 @@
 
   .auth-entity,
   .auth-entity-header {
-    padding: 0px var(--spacing-xl);
+    padding: 0 var(--spacing-xl);
   }
 
   .auth-entity,
@@ -946,15 +974,17 @@
     display: flex;
     align-items: center;
     gap: var(--spacing-m);
-    color: var(--spectrum-global-color-gray-900);
     overflow: hidden;
+    width: 100%;
   }
 
-  .auth-entity .user-email {
-    text-overflow: ellipsis;
-    white-space: nowrap;
+  .auth-entity .user-email,
+  .group-name {
+    flex: 1 1 0;
+    min-width: 0;
     overflow: hidden;
-    color: var(--spectrum-global-color-gray-900);
+    white-space: nowrap;
+    text-overflow: ellipsis;
   }
 
   #builder-side-panel-container {
@@ -1048,4 +1078,23 @@
   .alert {
     padding: 0 var(--spacing-xl);
   }
+
+  .user-groups {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+    gap: var(--spacing-m);
+    width: 100%;
+    min-width: 0;
+  }
+
+  .group-info {
+    display: flex;
+    flex-direction: row;
+    gap: var(--spacing-xs);
+    justify-content: end;
+    width: 60px;
+    flex: 0 0 auto;
+  }
 </style>

packages/builder/src/pages/builder/portal/users/users/[userId].svelte

@@ -98,7 +98,9 @@
   $: privileged = sdk.users.isAdminOrGlobalBuilder(user)
   $: nameLabel = getNameLabel(user)
   $: filteredGroups = getFilteredGroups(internalGroups, searchTerm)
-  $: availableApps = getAvailableApps($appsStore.apps, privileged, user?.roles)
+  $: availableApps = user
+    ? getApps(user, sdk.users.userAppAccessList(user, $groups || []))
+    : []
   $: userGroups = $groups.filter(x => {
     return x.users?.find(y => {
       return y._id === userId
@@ -107,23 +109,19 @@
   $: globalRole = users.getUserRole(user)
   $: isTenantOwner = tenantOwner?.email && tenantOwner.email === user?.email
 
-  const getAvailableApps = (appList, privileged, roles) => {
-    let availableApps = appList.slice()
-    if (!privileged) {
-      availableApps = availableApps.filter(x => {
-        let roleKeys = Object.keys(roles || {})
-        return roleKeys.concat(user?.builder?.apps).find(y => {
-          return x.appId === appsStore.extractAppId(y)
-        })
-      })
-    }
+  const getApps = (user, appIds) => {
+    let availableApps = $appsStore.apps
+      .slice()
+      .filter(app =>
+        appIds.find(id => id === appsStore.getProdAppID(app.devId))
+      )
     return availableApps.map(app => {
       const prodAppId = appsStore.getProdAppID(app.devId)
       return {
         name: app.name,
         devId: app.devId,
         icon: app.icon,
-        role: getRole(prodAppId, roles),
+        role: getRole(prodAppId, user),
       }
     })
   }
@@ -136,7 +134,7 @@
     return groups.filter(group => group.name?.toLowerCase().includes(search))
   }
 
-  const getRole = (prodAppId, roles) => {
+  const getRole = (prodAppId, user) => {
     if (privileged) {
       return Constants.Roles.ADMIN
     }
@@ -145,7 +143,21 @@
       return Constants.Roles.CREATOR
     }
 
-    return roles[prodAppId]
+    if (user?.roles[prodAppId]) {
+      return user.roles[prodAppId]
+    }
+
+    // check if access via group for creator
+    const foundGroup = $groups?.find(
+      group => group.roles[prodAppId] || group.builder?.apps[prodAppId]
+    )
+    if (foundGroup.builder?.apps[prodAppId]) {
+      return Constants.Roles.CREATOR
+    }
+    // can't tell how groups will control role
+    if (foundGroup.roles[prodAppId]) {
+      return Constants.Roles.GROUP
+    }
   }
 
   const getNameLabel = user => {

packages/builder/src/pages/builder/portal/users/users/_components/AppRoleTableRenderer.svelte

@@ -15,7 +15,9 @@
   }
 </script>
 
-{#if value === Constants.Roles.CREATOR}
+{#if value === Constants.Roles.GROUP}
+  Controlled by group
+{:else if value === Constants.Roles.CREATOR}
   Can edit
 {:else}
   <StatusLight

packages/builder/src/pages/builder/portal/users/users/index.svelte

@@ -128,7 +128,7 @@
           $auth.user?.email === user.email
             ? false
             : true,
-        apps: [...new Set(Object.keys(user.roles))],
+        apps: sdk.users.userAppAccessList(user, $groups),
         access: role.sortOrder,
       }
     })

packages/builder/src/stores/builder/automations.ts

@@ -15,7 +15,6 @@ import {
 import {
   AutomationTriggerStepId,
   AutomationEventType,
-  AutomationStepType,
   AutomationActionStepId,
   Automation,
   AutomationStep,
@@ -26,10 +25,14 @@ import {
   UILogicalOperator,
   EmptyFilterOption,
   AutomationIOType,
-  AutomationStepSchema,
-  AutomationTriggerSchema,
   BranchPath,
   BlockDefinitions,
+  isBranchStep,
+  isTrigger,
+  isRowUpdateTrigger,
+  isRowSaveTrigger,
+  isAppTrigger,
+  BranchStep,
 } from "@budibase/types"
 import { ActionStepID } from "@/constants/backend/automations"
 import { FIELDS } from "@/constants/backend"
@@ -291,16 +294,16 @@ const automationActions = (store: AutomationStore) => ({
     let result: (AutomationStep | AutomationTrigger)[] = []
     pathWay.forEach(path => {
       const { stepIdx, branchIdx } = path
-      let last = result.length ? result[result.length - 1] : []
       if (!result.length) {
         // Preceeding steps.
         result = steps.slice(0, stepIdx + 1)
         return
       }
-      if (last && "inputs" in last) {
+      let last = result[result.length - 1]
+      if (isBranchStep(last)) {
         if (Number.isInteger(branchIdx)) {
           const branchId = last.inputs.branches[branchIdx].id
-          const children = last.inputs.children[branchId]
+          const children = last.inputs.children?.[branchId] || []
           const stepChildren = children.slice(0, stepIdx + 1)
           // Preceeding steps.
           result = result.concat(stepChildren)
@@ -473,23 +476,28 @@ const automationActions = (store: AutomationStore) => ({
           id: block.id,
         },
       ]
-      const branches: Branch[] = block.inputs?.branches || []
 
-      branches.forEach((branch, bIdx) => {
-        block.inputs?.children[branch.id].forEach(
-          (bBlock: AutomationStep, sIdx: number, array: AutomationStep[]) => {
-            const ended =
-              array.length - 1 === sIdx && !bBlock.inputs?.branches?.length
-            treeTraverse(bBlock, pathToCurrentNode, sIdx, bIdx, ended)
-          }
-        )
-      })
+      if (isBranchStep(block)) {
+        const branches = block.inputs?.branches || []
+        const children = block.inputs?.children || {}
+
+        branches.forEach((branch, bIdx) => {
+          children[branch.id].forEach(
+            (bBlock: AutomationStep, sIdx: number, array: AutomationStep[]) => {
+              const ended = array.length - 1 === sIdx
+              treeTraverse(bBlock, pathToCurrentNode, sIdx, bIdx, ended)
+            }
+          )
+        })
+
+        terminating = terminating && !branches.length
+      }
 
       store.actions.registerBlock(
         blockRefs,
         block,
         pathToCurrentNode,
-        terminating && !branches.length
+        terminating
       )
     }
 
@@ -497,7 +505,6 @@ const automationActions = (store: AutomationStore) => ({
     blocks.forEach((block, idx, array) => {
       treeTraverse(block, null, idx, null, array.length - 1 === idx)
     })
-
     return blockRefs
   },
 
@@ -575,7 +582,6 @@ const automationActions = (store: AutomationStore) => ({
           pathBlock.stepId === ActionStepID.LOOP &&
           pathBlock.blockToLoop in blocks
       }
-      const isTrigger = pathBlock.type === AutomationStepType.TRIGGER
 
       if (isLoopBlock && loopBlockCount == 0) {
         schema = {
@@ -586,17 +592,14 @@ const automationActions = (store: AutomationStore) => ({
         }
       }
 
-      const icon = isTrigger
+      const icon = isTrigger(pathBlock)
         ? pathBlock.icon
         : isLoopBlock
         ? "Reuse"
         : pathBlock.icon
 
-      if (blockIdx === 0 && isTrigger) {
-        if (
-          pathBlock.event === AutomationEventType.ROW_UPDATE ||
-          pathBlock.event === AutomationEventType.ROW_SAVE
-        ) {
+      if (blockIdx === 0 && isTrigger(pathBlock)) {
+        if (isRowUpdateTrigger(pathBlock) || isRowSaveTrigger(pathBlock)) {
           let table: any = get(tables).list.find(
             (table: Table) => table._id === pathBlock.inputs.tableId
           )
@@ -608,7 +611,7 @@ const automationActions = (store: AutomationStore) => ({
             }
           }
           delete schema.row
-        } else if (pathBlock.event === AutomationEventType.APP_TRIGGER) {
+        } else if (isAppTrigger(pathBlock)) {
           schema = Object.fromEntries(
             Object.keys(pathBlock.inputs.fields || []).map(key => [
               key,
@@ -915,8 +918,10 @@ const automationActions = (store: AutomationStore) => ({
     ]
 
     let cache:
-      | AutomationStepSchema<AutomationActionStepId>
-      | AutomationTriggerSchema<AutomationTriggerStepId>
+      | AutomationStep
+      | AutomationTrigger
+      | AutomationStep[]
+      | undefined = undefined
 
     pathWay.forEach((path, pathIdx, array) => {
       const { stepIdx, branchIdx } = path
@@ -938,9 +943,13 @@ const automationActions = (store: AutomationStore) => ({
         }
         return
       }
-      if (Number.isInteger(branchIdx)) {
+      if (
+        Number.isInteger(branchIdx) &&
+        !Array.isArray(cache) &&
+        isBranchStep(cache)
+      ) {
         const branchId = cache.inputs.branches[branchIdx].id
-        const children = cache.inputs.children[branchId]
+        const children = cache.inputs.children?.[branchId] || []
 
         if (final) {
           insertBlock(children, stepIdx)
@@ -1090,7 +1099,7 @@ const automationActions = (store: AutomationStore) => ({
   branchLeft: async (
     pathTo: Array<any>,
     automation: Automation,
-    block: AutomationStep
+    block: BranchStep
   ) => {
     const update = store.actions.shiftBranch(pathTo, block)
     if (update) {
@@ -1113,7 +1122,7 @@ const automationActions = (store: AutomationStore) => ({
   branchRight: async (
     pathTo: Array<BranchPath>,
     automation: Automation,
-    block: AutomationStep
+    block: BranchStep
   ) => {
     const update = store.actions.shiftBranch(pathTo, block, 1)
     if (update) {
@@ -1133,7 +1142,7 @@ const automationActions = (store: AutomationStore) => ({
    * @param {Number} direction - the direction of the swap. Defaults to -1 for left, add 1 for right
    * @returns
    */
-  shiftBranch: (pathTo: Array<any>, block: AutomationStep, direction = -1) => {
+  shiftBranch: (pathTo: Array<any>, block: BranchStep, direction = -1) => {
     let newBlock = cloneDeep(block)
     const branchPath = pathTo.at(-1)
     const targetIdx = branchPath.branchIdx

packages/builder/src/stores/builder/componentTreeNodes.ts

@@ -58,7 +58,7 @@ export class ComponentTreeNodesStore extends BudiStore<OpenNodesState> {
 
     const path = findComponentPath(selectedScreen.props, componentId)
 
-    const componentIds = path.map((component: Component) => component._id)
+    const componentIds = path.map((component: Component) => component._id!)
 
     this.update((openNodes: OpenNodesState) => {
       const newNodes = Object.fromEntries(

packages/builder/src/stores/builder/components.ts

@@ -1,3 +1,5 @@
+// TODO: analise and fix all the undefined ! and ?
+
 import { get, derived } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
 import { API } from "@/api"
@@ -36,7 +38,7 @@ import { Utils } from "@budibase/frontend-core"
 import {
   ComponentDefinition,
   ComponentSetting,
-  Component as ComponentType,
+  Component,
   ComponentCondition,
   FieldType,
   Screen,
@@ -45,10 +47,6 @@ import {
 import { utils } from "@budibase/shared-core"
 import { getSequentialName } from "@/helpers/duplicate"
 
-interface Component extends ComponentType {
-  _id: string
-}
-
 export interface ComponentState {
   components: Record<string, ComponentDefinition>
   customComponents: string[]
@@ -182,7 +180,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
    * Takes an enriched component instance and applies any required migration
    * logic
    */
-  migrateSettings(enrichedComponent: Component) {
+  migrateSettings(enrichedComponent: Component | null) {
     const componentPrefix = "@budibase/standard-components"
     let migrated = false
 
@@ -232,7 +230,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
 
   enrichEmptySettings(
     component: Component,
-    opts: { screen?: Screen; parent?: Component; useDefaultValues?: boolean }
+    opts: { screen?: Screen; parent?: string; useDefaultValues?: boolean }
   ) {
     if (!component?._component) {
       return
@@ -240,7 +238,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
     const defaultDS = this.getDefaultDatasource()
     const settings = this.getComponentSettings(component._component)
     const { parent, screen, useDefaultValues } = opts || {}
-    const treeId = parent?._id || component._id
+    const treeId = parent || component._id
     if (!screen) {
       return
     }
@@ -425,7 +423,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
   createInstance(
     componentType: string,
     presetProps?: Record<string, any>,
-    parent?: Component
+    parent?: string
   ): Component | null {
     const screen = get(selectedScreen)
     if (!screen) {
@@ -503,7 +501,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
   async create(
     componentType: string,
     presetProps?: Record<string, any>,
-    parent?: Component,
+    parent?: string,
     index?: number
   ) {
     const state = get(this.store)
@@ -519,7 +517,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
     // Insert in position if specified
     if (parent && index != null) {
       await screenStore.patch((screen: Screen) => {
-        let parentComponent = findComponent(screen.props, parent)
+        let parentComponent = findComponent(screen.props, parent)!
         if (!parentComponent._children?.length) {
           parentComponent._children = [componentInstance]
         } else {
@@ -538,7 +536,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
         }
         const currentComponent = findComponent(
           screen.props,
-          selectedComponentId
+          selectedComponentId!
         )
         if (!currentComponent) {
           return false
@@ -581,7 +579,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
       return state
     })
 
-    componentTreeNodesStore.makeNodeVisible(componentInstance._id)
+    componentTreeNodesStore.makeNodeVisible(componentInstance._id!)
 
     // Log event
     analytics.captureEvent(Events.COMPONENT_CREATED, {
@@ -633,7 +631,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
     // Determine the next component to select, and select it before deletion
     // to avoid an intermediate state of no component selection
     const state = get(this.store)
-    let nextId = ""
+    let nextId: string | null = ""
     if (state.selectedComponentId === component._id) {
       nextId = this.getNext()
       if (!nextId) {
@@ -646,7 +644,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
         nextId = nextId.replace("-navigation", "-screen")
       }
       this.update(state => {
-        state.selectedComponentId = nextId
+        state.selectedComponentId = nextId ?? undefined
         return state
       })
     }
@@ -654,18 +652,18 @@ export class ComponentStore extends BudiStore<ComponentState> {
     // Patch screen
     await screenStore.patch((screen: Screen) => {
       // Check component exists
-      component = findComponent(screen.props, component._id)
-      if (!component) {
+      const updatedComponent = findComponent(screen.props, component._id!)
+      if (!updatedComponent) {
         return false
       }
 
       // Check component has a valid parent
-      const parent = findComponentParent(screen.props, component._id)
+      const parent = findComponentParent(screen.props, updatedComponent._id)
       if (!parent) {
         return false
       }
-      parent._children = parent._children.filter(
-        (child: Component) => child._id !== component._id
+      parent._children = parent._children!.filter(
+        (child: Component) => child._id !== updatedComponent._id
       )
     }, null)
   }
@@ -729,7 +727,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
     // Patch screen
     const patch = (screen: Screen) => {
       // Get up to date ref to target
-      targetComponent = findComponent(screen.props, targetComponent._id)
+      targetComponent = findComponent(screen.props, targetComponent!._id!)!
       if (!targetComponent) {
         return false
       }
@@ -743,7 +741,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
       if (!cut) {
         componentToPaste = makeComponentUnique(componentToPaste)
       }
-      newComponentId = componentToPaste._id
+      newComponentId = componentToPaste._id!
 
       // Strip grid position metadata if pasting into a new screen, but keep
       // alignment metadata
@@ -820,8 +818,8 @@ export class ComponentStore extends BudiStore<ComponentState> {
     if (!screen) {
       throw "A valid screen must be selected"
     }
-    const parent = findComponentParent(screen.props, componentId)
-    const index = parent?._children.findIndex(
+    const parent = findComponentParent(screen.props, componentId)!
+    const index = parent?._children?.findIndex(
       (x: Component) => x._id === componentId
     )
 
@@ -839,29 +837,29 @@ export class ComponentStore extends BudiStore<ComponentState> {
     }
 
     // If we have siblings above us, choose the sibling or a descendant
-    if (index > 0) {
+    if (index !== undefined && index > 0) {
       // If sibling before us accepts children, and is not collapsed, select a descendant
-      const previousSibling = parent._children[index - 1]
+      const previousSibling = parent._children![index - 1]
       if (
         previousSibling._children?.length &&
-        componentTreeNodesStore.isNodeExpanded(previousSibling._id)
+        componentTreeNodesStore.isNodeExpanded(previousSibling._id!)
       ) {
         let target = previousSibling
         while (
           target._children?.length &&
-          componentTreeNodesStore.isNodeExpanded(target._id)
+          componentTreeNodesStore.isNodeExpanded(target._id!)
         ) {
           target = target._children[target._children.length - 1]
         }
-        return target._id
+        return target._id!
       }
 
       // Otherwise just select sibling
-      return previousSibling._id
+      return previousSibling._id!
     }
 
     // If no siblings above us, select the parent
-    return parent._id
+    return parent._id!
   }
 
   getNext() {
@@ -873,9 +871,9 @@ export class ComponentStore extends BudiStore<ComponentState> {
       throw "A valid screen must be selected"
     }
     const parent = findComponentParent(screen.props, componentId)
-    const index = parent?._children.findIndex(
+    const index = parent?._children?.findIndex(
       (x: Component) => x._id === componentId
-    )
+    )!
 
     // Check for screen and navigation component edge cases
     const screenComponentId = `${screen._id}-screen`
@@ -888,37 +886,38 @@ export class ComponentStore extends BudiStore<ComponentState> {
     if (
       component?._children?.length &&
       (state.selectedComponentId === navComponentId ||
-        componentTreeNodesStore.isNodeExpanded(component._id))
+        componentTreeNodesStore.isNodeExpanded(component._id!))
     ) {
-      return component._children[0]._id
+      return component._children[0]._id!
     } else if (!parent) {
       return null
     }
 
     // Otherwise select the next sibling if we have one
-    if (index < parent._children.length - 1) {
-      const nextSibling = parent._children[index + 1]
-      return nextSibling._id
+    if (index < parent._children!.length - 1) {
+      const nextSibling = parent._children![index + 1]
+      return nextSibling._id!
     }
 
     // Last child, select our parents next sibling
     let target = parent
     let targetParent = findComponentParent(screen.props, target._id)
-    let targetIndex = targetParent?._children.findIndex(
+    let targetIndex = targetParent?._children?.findIndex(
       (child: Component) => child._id === target._id
-    )
+    )!
     while (
       targetParent != null &&
-      targetIndex === targetParent._children?.length - 1
+      targetParent._children &&
+      targetIndex === targetParent._children.length - 1
     ) {
       target = targetParent
       targetParent = findComponentParent(screen.props, target._id)
-      targetIndex = targetParent?._children.findIndex(
+      targetIndex = targetParent?._children!.findIndex(
         (child: Component) => child._id === target._id
-      )
+      )!
     }
     if (targetParent) {
-      return targetParent._children[targetIndex + 1]._id
+      return targetParent._children![targetIndex + 1]._id!
     } else {
       return null
     }
@@ -950,16 +949,16 @@ export class ComponentStore extends BudiStore<ComponentState> {
       const parent = findComponentParent(screen.props, componentId)
 
       // Check we aren't right at the top of the tree
-      const index = parent?._children.findIndex(
+      const index = parent?._children?.findIndex(
         (x: Component) => x._id === componentId
-      )
+      )!
       if (!parent || (index === 0 && parent._id === screen.props._id)) {
         return
       }
 
       // Copy original component and remove it from the parent
-      const originalComponent = cloneDeep(parent._children[index])
-      parent._children = parent._children.filter(
+      const originalComponent = cloneDeep(parent._children![index])
+      parent._children = parent._children!.filter(
         (component: Component) => component._id !== componentId
       )
 
@@ -971,9 +970,9 @@ export class ComponentStore extends BudiStore<ComponentState> {
         const definition = this.getDefinition(previousSibling._component)
         if (
           definition?.hasChildren &&
-          componentTreeNodesStore.isNodeExpanded(previousSibling._id)
+          componentTreeNodesStore.isNodeExpanded(previousSibling._id!)
         ) {
-          previousSibling._children.push(originalComponent)
+          previousSibling._children!.push(originalComponent)
         }
 
         // Otherwise just move component above sibling
@@ -985,11 +984,11 @@ export class ComponentStore extends BudiStore<ComponentState> {
       // If no siblings above us, go above the parent as long as it isn't
       // the screen
       else if (parent._id !== screen.props._id) {
-        const grandParent = findComponentParent(screen.props, parent._id)
-        const parentIndex = grandParent._children.findIndex(
+        const grandParent = findComponentParent(screen.props, parent._id)!
+        const parentIndex = grandParent._children!.findIndex(
           (child: Component) => child._id === parent._id
         )
-        grandParent._children.splice(parentIndex, 0, originalComponent)
+        grandParent._children!.splice(parentIndex, 0, originalComponent)
       }
     }, null)
   }
@@ -1028,9 +1027,9 @@ export class ComponentStore extends BudiStore<ComponentState> {
         const definition = this.getDefinition(nextSibling._component)
         if (
           definition?.hasChildren &&
-          componentTreeNodesStore.isNodeExpanded(nextSibling._id)
+          componentTreeNodesStore.isNodeExpanded(nextSibling._id!)
         ) {
-          nextSibling._children.splice(0, 0, originalComponent)
+          nextSibling._children!.splice(0, 0, originalComponent)
         }
 
         // Otherwise move below next sibling
@@ -1041,11 +1040,11 @@ export class ComponentStore extends BudiStore<ComponentState> {
 
       // Last child, so move below our parent
       else {
-        const grandParent = findComponentParent(screen.props, parent._id)
-        const parentIndex = grandParent._children.findIndex(
+        const grandParent = findComponentParent(screen.props, parent._id)!
+        const parentIndex = grandParent._children!.findIndex(
           (child: Component) => child._id === parent._id
         )
-        grandParent._children.splice(parentIndex + 1, 0, originalComponent)
+        grandParent._children!.splice(parentIndex + 1, 0, originalComponent)
       }
     }, null)
   }
@@ -1208,13 +1207,13 @@ export class ComponentStore extends BudiStore<ComponentState> {
       }
 
       // Replace component with parent
-      const index = oldParentDefinition._children.findIndex(
+      const index = oldParentDefinition._children!.findIndex(
         (component: Component) => component._id === componentId
       )
       if (index === -1) {
         return false
       }
-      oldParentDefinition._children[index] = {
+      oldParentDefinition._children![index] = {
         ...newParentDefinition,
         _children: [definition],
       }

packages/builder/src/stores/builder/preview.ts

@@ -1,6 +1,6 @@
 import { get } from "svelte/store"
 import { BudiStore } from "../BudiStore"
-import { PreviewDevice, ComponentContext } from "@budibase/types"
+import { PreviewDevice, ComponentContext, AppContext } from "@budibase/types"
 
 type PreviewEventHandler = (name: string, payload?: any) => void
 
@@ -8,7 +8,7 @@ interface PreviewState {
   previewDevice: PreviewDevice
   previewEventHandler: PreviewEventHandler | null
   showPreview: boolean
-  selectedComponentContext: ComponentContext | null
+  selectedComponentContext: AppContext | null
 }
 
 const INITIAL_PREVIEW_STATE: PreviewState = {

packages/builder/src/stores/builder/screenComponent.ts

@@ -8,6 +8,7 @@ import {
   UIComponentError,
   ComponentDefinition,
   DependsOnComponentSetting,
+  Screen,
 } from "@budibase/types"
 import { queries } from "./queries"
 import { views } from "./views"
@@ -66,6 +67,7 @@ export const screenComponentErrorList = derived(
     if (!$selectedScreen) {
       return []
     }
+    const screen = $selectedScreen
 
     const datasources = {
       ...reduceBy("_id", $tables.list),
@@ -79,9 +81,11 @@ export const screenComponentErrorList = derived(
     const errors: UIComponentError[] = []
 
     function checkComponentErrors(component: Component, ancestors: string[]) {
-      errors.push(...getInvalidDatasources(component, datasources, definitions))
-      errors.push(...getMissingRequiredSettings(component, definitions))
       errors.push(...getMissingAncestors(component, definitions, ancestors))
+      errors.push(
+        ...getInvalidDatasources(screen, component, datasources, definitions)
+      )
+      errors.push(...getMissingRequiredSettings(component, definitions))
 
       for (const child of component._children || []) {
         checkComponentErrors(child, [...ancestors, component._component])
@@ -95,6 +99,7 @@ export const screenComponentErrorList = derived(
 )
 
 function getInvalidDatasources(
+  screen: Screen,
   component: Component,
   datasources: Record<string, any>,
   definitions: Record<string, ComponentDefinition>
@@ -234,7 +239,10 @@ function getMissingAncestors(
   ancestors: string[]
 ): UIComponentError[] {
   const definition = definitions[component._component]
-
+  if (ancestors.some(a => !a.startsWith(BudibasePrefix))) {
+    // We don't have a way to know what components are used within a plugin component
+    return []
+  }
   if (!definition?.requiredAncestors?.length) {
     return []
   }

packages/builder/src/stores/builder/screens.ts

@@ -490,7 +490,7 @@ export class ScreenStore extends BudiStore<ScreenState> {
     // Flatten the recursive component tree
     const components = findAllMatchingComponents(
       screen.props,
-      (x: Component) => x
+      (x: Component) => !!x
     )
 
     // Iterate over all components and run checks

packages/client/manifest.json

@@ -4492,6 +4492,12 @@
           }
         ]
       },
+      {
+        "type": "text",
+        "label": "Zoom level",
+        "key": "defaultZoom",
+        "defaultValue": "1"
+      },
       {
         "type": "event",
         "label": "On change",

packages/client/package.json

@@ -28,7 +28,7 @@
     "apexcharts": "^3.48.0",
     "dayjs": "^1.10.8",
     "downloadjs": "1.4.7",
-    "html5-qrcode": "^2.2.1",
+    "html5-qrcode": "^2.3.8",
     "leaflet": "^1.7.1",
     "sanitize-html": "^2.13.0",
     "screenfull": "^6.0.1",

packages/client/src/components/app/DataProvider.svelte

@@ -2,9 +2,7 @@
   import { getContext } from "svelte"
   import { Pagination, ProgressCircle } from "@budibase/bbui"
   import { fetchData, QueryUtils } from "@budibase/frontend-core"
-  import {
-    LogicalOperator,
-    EmptyFilterOption,
+  import type {
     TableSchema,
     SortOrder,
     SearchFilters,
@@ -14,6 +12,7 @@
     GroupUserDatasource,
     DataFetchOptions,
   } from "@budibase/types"
+  import { LogicalOperator, EmptyFilterOption } from "@budibase/types"
 
   type ProviderDatasource = Exclude<
     DataFetchDatasource,

packages/client/src/components/app/blocks/form/FormBlock.svelte

@@ -4,7 +4,7 @@
   import { Utils } from "@budibase/frontend-core"
   import FormBlockWrapper from "./FormBlockWrapper.svelte"
   import { get } from "svelte/store"
-  import { TableSchema, UIDatasource } from "@budibase/types"
+  import type { TableSchema, UIDatasource } from "@budibase/types"
 
   type Field = { name: string; active: boolean }
 

packages/client/src/components/app/forms/CodeScanner.svelte

@@ -20,6 +20,7 @@
   export let beepFrequency = 2637
   export let customFrequency = 1046
   export let preferredCamera = "environment"
+  export let defaultZoom = 1
   export let validator
 
   const dispatch = createEventDispatcher()
@@ -58,6 +59,14 @@
       html5QrCode
         .start(cameraSetting, cameraConfig, onScanSuccess)
         .then(() => {
+          if (defaultZoom > 1) {
+            const cameraOptions =
+              html5QrCode.getRunningTrackCameraCapabilities()
+            const zoom = cameraOptions.zoomFeature()
+            if (zoom.isSupported()) {
+              zoom.apply(defaultZoom)
+            }
+          }
           resolve({ initialised: true })
         })
         .catch(err => {

packages/client/src/components/app/forms/CodeScannerField.svelte

@@ -17,6 +17,7 @@
   export let beepFrequency
   export let customFrequency
   export let preferredCamera
+  export let defaultZoom
   export let helpText = null
 
   let fieldState
@@ -56,6 +57,7 @@
       {beepFrequency}
       {customFrequency}
       {preferredCamera}
+      {defaultZoom}
       validator={fieldState.validator}
     />
   {/if}

packages/client/src/components/app/forms/RelationshipField.svelte

@@ -1,18 +1,15 @@
 <script lang="ts">
   import { CoreSelect, CoreMultiselect } from "@budibase/bbui"
-  import { FieldType } from "@budibase/types"
+  import { FieldType, InternalTable } from "@budibase/types"
   import { fetchData, Utils } from "@budibase/frontend-core"
   import { getContext } from "svelte"
   import Field from "./Field.svelte"
   import type {
     SearchFilter,
     RelationshipFieldMetadata,
-    Table,
     Row,
   } from "@budibase/types"
 
-  const { API } = getContext("sdk")
-
   export let field: string | undefined = undefined
   export let label: string | undefined = undefined
   export let placeholder: any = undefined
@@ -20,10 +17,10 @@
   export let readonly: boolean = false
   export let validation: any
   export let autocomplete: boolean = true
-  export let defaultValue: string | undefined = undefined
+  export let defaultValue: string | string[] | undefined = undefined
   export let onChange: any
   export let filter: SearchFilter[]
-  export let datasourceType: "table" | "user" | "groupUser" = "table"
+  export let datasourceType: "table" | "user" = "table"
   export let primaryDisplay: string | undefined = undefined
   export let span: number | undefined = undefined
   export let helpText: string | undefined = undefined
@@ -32,191 +29,305 @@
     | FieldType.BB_REFERENCE
     | FieldType.BB_REFERENCE_SINGLE = FieldType.LINK
 
-  type RelationshipValue = { _id: string; [key: string]: any }
-  type OptionObj = Record<string, RelationshipValue>
-  type OptionsObjType = Record<string, OptionObj>
+  type BasicRelatedRow = { _id: string; primaryDisplay: string }
+  type OptionsMap = Record<string, BasicRelatedRow>
 
+  const { API } = getContext("sdk")
+
+  // Field state
   let fieldState: any
   let fieldApi: any
   let fieldSchema: RelationshipFieldMetadata | undefined
-  let tableDefinition: Table | null | undefined
-  let searchTerm: any
-  let open: boolean
-  let selectedValue: string[] | string
 
-  // need a cast version of this for reactivity, components below aren't typed
-  $: castSelectedValue = selectedValue as any
+  // Local UI state
+  let searchTerm: any
+  let open: boolean = false
+
+  // Options state
+  let options: BasicRelatedRow[] = []
+  let optionsMap: OptionsMap = {}
+  let loadingMissingOptions: boolean = false
+
+  // Determine if we can select multiple rows or not
   $: multiselect =
     [FieldType.LINK, FieldType.BB_REFERENCE].includes(type) &&
     fieldSchema?.relationshipType !== "one-to-many"
-  $: linkedTableId = fieldSchema?.tableId!
-  $: fetch = fetchData({
-    API,
-    datasource: {
-      // typing here doesn't seem correct - we have the correct datasourceType options
-      // but when we configure the fetchData, it seems to think only "table" is valid
-      type: datasourceType as any,
-      tableId: linkedTableId,
-    },
-    options: {
-      filter,
-      limit: 100,
-    },
-  })
 
-  $: tableDefinition = $fetch.definition
-  $: selectedValue = multiselect
-    ? flatten(fieldState?.value) ?? []
-    : flatten(fieldState?.value)?.[0]
-  $: component = multiselect ? CoreMultiselect : CoreSelect
-  $: primaryDisplay = primaryDisplay || tableDefinition?.primaryDisplay
+  // Get the proper string representation of the value
+  $: realValue = fieldState?.value
+  $: selectedValue = parseSelectedValue(realValue, multiselect)
+  $: selectedIDs = getSelectedIDs(selectedValue)
 
-  let optionsObj: OptionsObjType = {}
-  const debouncedFetchRows = Utils.debounce(fetchRows, 250)
+  // If writable, we use a fetch to load options
+  $: linkedTableId = fieldSchema?.tableId
+  $: writable = !disabled && !readonly
+  $: fetch = createFetch(writable, datasourceType, filter, linkedTableId)
 
-  $: {
-    if (primaryDisplay && fieldState && !optionsObj) {
-      // Persist the initial values as options, allowing them to be present in the dropdown,
-      // even if they are not in the inital fetch results
-      let valueAsSafeArray = fieldState.value || []
-      if (!Array.isArray(valueAsSafeArray)) {
-        valueAsSafeArray = [fieldState.value]
-      }
-      optionsObj = valueAsSafeArray.reduce(
-        (
-          accumulator: OptionObj,
-          value: { _id: string; primaryDisplay: any }
-        ) => {
-          // fieldState has to be an array of strings to be valid for an update
-          // therefore we cannot guarantee value will be an object
-          // https://linear.app/budibase/issue/BUDI-7577/refactor-the-relationshipfield-component-to-have-better-support-for
-          if (!value._id) {
-            return accumulator
+  // Attempt to determine the primary display field to use
+  $: tableDefinition = $fetch?.definition
+  $: primaryDisplayField = primaryDisplay || tableDefinition?.primaryDisplay
+
+  // Build our options map
+  $: rows = $fetch?.rows || []
+  $: processOptions(realValue, rows, primaryDisplayField)
+
+  // If we ever have a value selected for which we don't have an option, we must
+  // fetch those rows to ensure we can render them as options
+  $: missingIDs = selectedIDs.filter(id => !optionsMap[id])
+  $: loadMissingOptions(missingIDs, linkedTableId, primaryDisplayField)
+
+  // Convert our options map into an array for display
+  $: updateOptions(optionsMap)
+  $: !open && sortOptions()
+
+  // Search for new options when search term changes
+  $: debouncedSearchOptions(searchTerm || "", primaryDisplayField)
+
+  // Ensure backwards compatibility
+  $: enrichedDefaultValue = enrichDefaultValue(defaultValue)
+
+  // We need to cast value to pass it down, as those components aren't typed
+  $: emptyValue = multiselect ? [] : null
+  $: displayValue = missingIDs.length ? emptyValue : (selectedValue as any)
+
+  // Ensures that we flatten any objects so that only the IDs of the selected
+  // rows are passed down. Not sure how this can be an object to begin with?
+  const parseSelectedValue = (
+    value: any,
+    multiselect: boolean
+  ): undefined | string | string[] => {
+    return multiselect ? flatten(value) : flatten(value)[0]
+  }
+
+  // Where applicable, creates the fetch instance to load row options
+  const createFetch = (
+    writable: boolean,
+    dsType: typeof datasourceType,
+    filter: SearchFilter[],
+    linkedTableId?: string
+  ) => {
+    if (!linkedTableId) {
+      return undefined
+    }
+    const datasource =
+      datasourceType === "table"
+        ? {
+            type: datasourceType,
+            tableId: fieldSchema?.tableId!,
           }
-          accumulator[value._id] = {
-            _id: value._id,
-            [primaryDisplay]: value.primaryDisplay,
+        : {
+            type: datasourceType,
+            tableId: InternalTable.USER_METADATA,
           }
-          return accumulator
-        },
-        {}
-      )
-    }
-  }
-
-  $: enrichedOptions = enrichOptions(optionsObj, $fetch.rows)
-  const enrichOptions = (optionsObj: OptionsObjType, fetchResults: Row[]) => {
-    const result = (fetchResults || [])?.reduce((accumulator, row) => {
-      if (!accumulator[row._id!]) {
-        accumulator[row._id!] = row
-      }
-      return accumulator
-    }, optionsObj || {})
-
-    return Object.values(result)
-  }
-  $: {
-    // We don't want to reorder while the dropdown is open, to avoid UX jumps
-    if (!open && primaryDisplay) {
-      enrichedOptions = enrichedOptions.sort((a: OptionObj, b: OptionObj) => {
-        const selectedValues = flatten(fieldState?.value) || []
-
-        const aIsSelected = selectedValues.find(
-          (v: RelationshipValue) => v === a._id
-        )
-        const bIsSelected = selectedValues.find(
-          (v: RelationshipValue) => v === b._id
-        )
-        if (aIsSelected && !bIsSelected) {
-          return -1
-        } else if (!aIsSelected && bIsSelected) {
-          return 1
-        }
-
-        return (a[primaryDisplay] > b[primaryDisplay]) as unknown as number
-      })
-    }
-  }
-
-  $: {
-    if (filter || defaultValue) {
-      forceFetchRows()
-    }
-  }
-  $: debouncedFetchRows(searchTerm, primaryDisplay, defaultValue)
-
-  const forceFetchRows = async () => {
-    // if the filter has changed, then we need to reset the options, clear the selection, and re-fetch
-    optionsObj = {}
-    fieldApi?.setValue([])
-    selectedValue = []
-    debouncedFetchRows(searchTerm, primaryDisplay, defaultValue)
-  }
-  async function fetchRows(
-    searchTerm: any,
-    primaryDisplay: string,
-    defaultVal: string | string[]
-  ) {
-    const allRowsFetched =
-      $fetch.loaded &&
-      !Object.keys($fetch.query?.string || {}).length &&
-      !$fetch.hasNextPage
-    // Don't request until we have the primary display or default value has been fetched
-    if (allRowsFetched || !primaryDisplay) {
-      return
-    }
-    // must be an array
-    const defaultValArray: string[] = !defaultVal
-      ? []
-      : !Array.isArray(defaultVal)
-      ? defaultVal.split(",")
-      : defaultVal
-
-    if (
-      defaultVal &&
-      optionsObj &&
-      defaultValArray.some(val => !optionsObj[val])
-    ) {
-      await fetch.update({
-        query: { oneOf: { _id: defaultValArray } },
-      })
-    }
-
-    if (
-      (Array.isArray(selectedValue) &&
-        selectedValue.some(val => !optionsObj[val])) ||
-      (selectedValue && !optionsObj[selectedValue as string])
-    ) {
-      await fetch.update({
-        query: {
-          oneOf: {
-            _id: Array.isArray(selectedValue) ? selectedValue : [selectedValue],
-          },
-        },
-      })
-    }
-
-    // Ensure we match all filters, rather than any
-    // @ts-expect-error this doesn't fit types, but don't want to change it yet
-    const baseFilter: any = (filter || []).filter(x => x.operator !== "allOr")
-    await fetch.update({
-      filter: [
-        ...baseFilter,
-        {
-          // Use a big numeric prefix to avoid clashing with an existing filter
-          field: `999:${primaryDisplay}`,
-          operator: "string",
-          value: searchTerm,
-        },
-      ],
+    return fetchData({
+      API,
+      datasource,
+      options: {
+        filter,
+        limit: writable ? 100 : 1,
+      },
     })
   }
 
-  const flatten = (values: any | any[]) => {
+  // Small helper to represent the selected value as an array
+  const getSelectedIDs = (
+    selectedValue: undefined | string | string[]
+  ): string[] => {
+    if (!selectedValue) {
+      return []
+    }
+    return Array.isArray(selectedValue) ? selectedValue : [selectedValue]
+  }
+
+  // Builds a map of all available options, in a consistent structure
+  const processOptions = (
+    realValue: any | any[],
+    rows: Row[],
+    primaryDisplay?: string
+  ) => {
+    // First ensure that all options included in the value are present as valid
+    // options. These can be basic related row shapes which already include
+    // a value for primary display
+    if (realValue) {
+      const valueArray = Array.isArray(realValue) ? realValue : [realValue]
+      for (let val of valueArray) {
+        const option = parseOption(val, primaryDisplay)
+        if (option) {
+          optionsMap[option._id] = option
+        }
+      }
+    }
+
+    // Process all rows loaded from our fetch
+    for (let row of rows) {
+      const option = parseOption(row, primaryDisplay)
+      if (option) {
+        optionsMap[option._id] = option
+      }
+    }
+
+    // Reassign to trigger reactivity
+    optionsMap = optionsMap
+  }
+
+  // Parses a row-like structure into a properly shaped option
+  const parseOption = (
+    option: any | BasicRelatedRow | Row,
+    primaryDisplay?: string
+  ): BasicRelatedRow | null => {
+    if (!option || typeof option !== "object" || !option?._id) {
+      return null
+    }
+    // If this is a basic related row shape (_id and PD only) then just use
+    // that
+    if (Object.keys(option).length === 2 && "primaryDisplay" in option) {
+      return {
+        _id: option._id,
+        primaryDisplay: ensureString(option.primaryDisplay),
+      }
+    }
+    // Otherwise use the primary display field specified
+    if (primaryDisplay) {
+      return {
+        _id: option._id,
+        primaryDisplay: ensureString(
+          option[primaryDisplay as keyof typeof option]
+        ),
+      }
+    } else {
+      return {
+        _id: option._id,
+        primaryDisplay: option._id,
+      }
+    }
+  }
+
+  // Loads any rows which are selected and aren't present in the currently
+  // available option set. This is typically only IDs specified as default
+  // values.
+  const loadMissingOptions = async (
+    missingIDs: string[],
+    linkedTableId?: string,
+    primaryDisplay?: string
+  ) => {
+    if (
+      loadingMissingOptions ||
+      !missingIDs.length ||
+      !linkedTableId ||
+      !primaryDisplay
+    ) {
+      return
+    }
+    loadingMissingOptions = true
+    try {
+      const res = await API.searchTable(linkedTableId, {
+        query: {
+          oneOf: {
+            _id: missingIDs,
+          },
+        },
+      })
+      for (let row of res.rows) {
+        const option = parseOption(row, primaryDisplay)
+        if (option) {
+          optionsMap[option._id] = option
+        }
+      }
+
+      // Reassign to trigger reactivity
+      optionsMap = optionsMap
+      updateOptions(optionsMap)
+    } catch (error) {
+      console.error("Error loading missing row IDs", error)
+    } finally {
+      // Ensure we have some sort of option for all IDs
+      for (let id of missingIDs) {
+        if (!optionsMap[id]) {
+          optionsMap[id] = {
+            _id: id,
+            primaryDisplay: id,
+          }
+        }
+      }
+      loadingMissingOptions = false
+    }
+  }
+
+  // Updates the options list to reflect the currently available options
+  const updateOptions = (optionsMap: OptionsMap) => {
+    let newOptions = Object.values(optionsMap)
+
+    // Only override options if the quantity of options changes
+    if (newOptions.length !== options.length) {
+      options = newOptions
+      sortOptions()
+    }
+  }
+
+  // Sorts the options list by selected state, then by primary display
+  const sortOptions = () => {
+    // Create a quick lookup map so we can test whether options are selected
+    const selectedMap: Record<string, boolean> = selectedIDs.reduce(
+      (map, id) => ({ ...map, [id]: true }),
+      {}
+    )
+    options.sort((a, b) => {
+      const aSelected = !!selectedMap[a._id]
+      const bSelected = !!selectedMap[b._id]
+      if (aSelected === bSelected) {
+        return a.primaryDisplay < b.primaryDisplay ? -1 : 1
+      } else {
+        return aSelected ? -1 : 1
+      }
+    })
+  }
+
+  // Util to ensure a value is stringified
+  const ensureString = (val: any): string => {
+    return typeof val === "string" ? val : JSON.stringify(val)
+  }
+
+  // We previously included logic to manually process default value, which
+  // should not be done as it is handled by the core form logic.
+  // This logic included handling a comma separated list of IDs, so for
+  // backwards compatibility we must now unfortunately continue to handle that
+  // at this level.
+  const enrichDefaultValue = (val: any) => {
+    if (!val || typeof val !== "string") {
+      return val
+    }
+    return val.includes(",") ? val.split(",") : val
+  }
+
+  // Searches for new options matching the given term
+  async function searchOptions(searchTerm: string, primaryDisplay?: string) {
+    if (!primaryDisplay) {
+      return
+    }
+
+    // Ensure we match all filters, rather than any
+    let newFilter: any = filter
+    if (searchTerm) {
+      // @ts-expect-error this doesn't fit types, but don't want to change it yet
+      newFilter = (newFilter || []).filter(x => x.operator !== "allOr")
+      newFilter.push({
+        // Use a big numeric prefix to avoid clashing with an existing filter
+        field: `999:${primaryDisplay}`,
+        operator: "string",
+        value: searchTerm,
+      })
+    }
+    await fetch?.update({
+      filter: newFilter,
+    })
+  }
+  const debouncedSearchOptions = Utils.debounce(searchOptions, 250)
+
+  // Flattens an array of row-like objects into a simple array of row IDs
+  const flatten = (values: any | any[]): string[] => {
     if (!values) {
       return []
     }
-
     if (!Array.isArray(values)) {
       values = [values]
     }
@@ -226,16 +337,11 @@
     return values
   }
 
-  const getDisplayName = (row: Row) => {
-    return row?.[primaryDisplay!] || "-"
-  }
-
   const handleChange = (e: any) => {
     let value = e.detail
     if (!multiselect) {
       value = value == null ? [] : [value]
     }
-
     if (
       type === FieldType.BB_REFERENCE_SINGLE &&
       value &&
@@ -243,7 +349,6 @@
     ) {
       value = value[0] || null
     }
-
     const changed = fieldApi.setValue(value)
     if (onChange && changed) {
       onChange({
@@ -251,12 +356,6 @@
       })
     }
   }
-
-  const loadMore = () => {
-    if (!$fetch.loading) {
-      fetch.nextPage()
-    }
-  }
 </script>
 
 <Field
@@ -265,31 +364,31 @@
   {disabled}
   {readonly}
   {validation}
-  {defaultValue}
   {type}
   {span}
   {helpText}
+  defaultValue={enrichedDefaultValue}
   bind:fieldState
   bind:fieldApi
   bind:fieldSchema
 >
   {#if fieldState}
     <svelte:component
-      this={component}
-      options={enrichedOptions}
-      {autocomplete}
-      value={castSelectedValue}
-      on:change={handleChange}
-      on:loadMore={loadMore}
-      id={fieldState.fieldId}
-      disabled={fieldState.disabled}
-      readonly={fieldState.readonly}
-      getOptionLabel={getDisplayName}
+      this={multiselect ? CoreMultiselect : CoreSelect}
+      value={displayValue}
+      id={fieldState?.fieldId}
+      disabled={fieldState?.disabled}
+      readonly={fieldState?.readonly}
+      loading={!!$fetch?.loading}
+      getOptionLabel={option => option.primaryDisplay}
       getOptionValue={option => option._id}
+      {options}
       {placeholder}
+      {autocomplete}
       bind:searchTerm
-      loading={$fetch.loading}
       bind:open
+      on:change={handleChange}
+      on:loadMore={() => fetch?.nextPage()}
     />
   {/if}
 </Field>

packages/client/src/components/error-states/ComponentErrorState.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
   import { getContext } from "svelte"
   import { Icon } from "@budibase/bbui"
-  import { UIComponentError } from "@budibase/types"
+  import type { UIComponentError } from "@budibase/types"
   import ComponentErrorStateCta from "./ComponentErrorStateCTA.svelte"
 
   export let componentErrors: UIComponentError[] | undefined

packages/client/src/components/error-states/ComponentErrorStateCTA.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
   import { getContext } from "svelte"
-  import { UIComponentError } from "@budibase/types"
+  import type { UIComponentError } from "@budibase/types"
 
   export let error: UIComponentError | undefined
 

packages/client/src/components/preview/DNDHandler.svelte

@@ -6,7 +6,7 @@
   import { findComponentById } from "@/utils/components.js"
   import { isGridEvent } from "@/utils/grid"
   import { DNDPlaceholderID } from "@/constants"
-  import { Component } from "@budibase/types"
+  import type { Component } from "@budibase/types"
 
   type ChildCoords = {
     placeholder: boolean

packages/client/src/index.js

@@ -1,142 +0,0 @@
-import ClientApp from "./components/ClientApp.svelte"
-import UpdatingApp from "./components/UpdatingApp.svelte"
-import {
-  builderStore,
-  appStore,
-  blockStore,
-  componentStore,
-  environmentStore,
-  dndStore,
-  eventStore,
-  hoverStore,
-  stateStore,
-  routeStore,
-} from "./stores"
-import loadSpectrumIcons from "@budibase/bbui/spectrum-icons-vite.js"
-import { get } from "svelte/store"
-import { initWebsocket } from "./websocket.js"
-
-// Provide svelte and svelte/internal as globals for custom components
-import * as svelte from "svelte"
-import * as internal from "svelte/internal"
-
-window.svelte_internal = internal
-window.svelte = svelte
-
-// Initialise spectrum icons
-loadSpectrumIcons()
-
-let app
-
-const loadBudibase = async () => {
-  // Update builder store with any builder flags
-  builderStore.set({
-    ...get(builderStore),
-    inBuilder: !!window["##BUDIBASE_IN_BUILDER##"],
-    layout: window["##BUDIBASE_PREVIEW_LAYOUT##"],
-    screen: window["##BUDIBASE_PREVIEW_SCREEN##"],
-    selectedComponentId: window["##BUDIBASE_SELECTED_COMPONENT_ID##"],
-    previewId: window["##BUDIBASE_PREVIEW_ID##"],
-    theme: window["##BUDIBASE_PREVIEW_THEME##"],
-    customTheme: window["##BUDIBASE_PREVIEW_CUSTOM_THEME##"],
-    previewDevice: window["##BUDIBASE_PREVIEW_DEVICE##"],
-    navigation: window["##BUDIBASE_PREVIEW_NAVIGATION##"],
-    hiddenComponentIds: window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"],
-    usedPlugins: window["##BUDIBASE_USED_PLUGINS##"],
-    location: window["##BUDIBASE_LOCATION##"],
-    snippets: window["##BUDIBASE_SNIPPETS##"],
-    componentErrors: window["##BUDIBASE_COMPONENT_ERRORS##"],
-  })
-
-  // Set app ID - this window flag is set by both the preview and the real
-  // server rendered app HTML
-  appStore.actions.setAppId(window["##BUDIBASE_APP_ID##"])
-
-  // Set the flag used to determine if the app is being loaded via an iframe
-  appStore.actions.setAppEmbedded(
-    window["##BUDIBASE_APP_EMBEDDED##"] === "true"
-  )
-
-  if (window.MIGRATING_APP) {
-    new UpdatingApp({
-      target: window.document.body,
-    })
-    return
-  }
-
-  // Fetch environment info
-  if (!get(environmentStore)?.loaded) {
-    await environmentStore.actions.fetchEnvironment()
-  }
-
-  // Register handler for runtime events from the builder
-  window.handleBuilderRuntimeEvent = (type, data) => {
-    if (!window["##BUDIBASE_IN_BUILDER##"]) {
-      return
-    }
-    if (type === "event-completed") {
-      eventStore.actions.resolveEvent(data)
-    } else if (type === "eject-block") {
-      const block = blockStore.actions.getBlock(data)
-      block?.eject()
-    } else if (type === "dragging-new-component") {
-      const { dragging, component } = data
-      if (dragging) {
-        const definition =
-          componentStore.actions.getComponentDefinition(component)
-        dndStore.actions.startDraggingNewComponent({ component, definition })
-      } else {
-        dndStore.actions.reset()
-      }
-    } else if (type === "request-context") {
-      const { selectedComponentInstance, screenslotInstance } =
-        get(componentStore)
-      const instance = selectedComponentInstance || screenslotInstance
-      const context = instance?.getDataContext()
-      let stringifiedContext = null
-      try {
-        stringifiedContext = JSON.stringify(context)
-      } catch (error) {
-        // Ignore - invalid context
-      }
-      eventStore.actions.dispatchEvent("provide-context", {
-        context: stringifiedContext,
-      })
-    } else if (type === "hover-component") {
-      hoverStore.actions.hoverComponent(data, false)
-    } else if (type === "builder-meta") {
-      builderStore.actions.setMetadata(data)
-    } else if (type === "builder-state") {
-      const [[key, value]] = Object.entries(data)
-      stateStore.actions.setValue(key, value)
-    } else if (type === "builder-url-test-data") {
-      const { route, testValue } = data
-      routeStore.actions.setTestUrlParams(route, testValue)
-    }
-  }
-
-  // Register any custom components
-  if (window["##BUDIBASE_CUSTOM_COMPONENTS##"]) {
-    window["##BUDIBASE_CUSTOM_COMPONENTS##"].forEach(component => {
-      componentStore.actions.registerCustomComponent(component)
-    })
-  }
-
-  // Make a callback available for custom component bundles to register
-  // themselves at runtime
-  window.registerCustomComponent =
-    componentStore.actions.registerCustomComponent
-
-  // Initialise websocket
-  initWebsocket()
-
-  // Create app if one hasn't been created yet
-  if (!app) {
-    app = new ClientApp({
-      target: window.document.body,
-    })
-  }
-}
-
-// Attach to window so the HTML template can call this when it loads
-window.loadBudibase = loadBudibase

packages/frontend-core/src/constants.ts

@@ -106,16 +106,14 @@ export const Roles = {
   PUBLIC: "PUBLIC",
   BUILDER: "BUILDER",
   CREATOR: "CREATOR",
+  GROUP: "GROUP",
 }
 
 export const EventPublishType = {
   ENV_VAR_UPGRADE_PANEL_OPENED: "environment_variable_upgrade_panel_opened",
 }
 
-export const ContextScopes = {
-  Local: "local",
-  Global: "global",
-}
+export { ComponentContextScopes as ContextScopes } from "@budibase/types"
 
 export const TypeIconMap = {
   [FieldType.STRING]: "Text",

packages/frontend-core/src/fetch/GroupUserFetch.ts

@@ -4,7 +4,7 @@ import { GroupUserDatasource, InternalTable } from "@budibase/types"
 
 interface GroupUserQuery {
   groupId: string
-  emailSearch: string
+  emailSearch?: string
 }
 
 interface GroupUserDefinition {

packages/frontend-core/src/fetch/UserFetch.ts

@@ -9,8 +9,8 @@ import {
 } from "@budibase/types"
 
 interface UserFetchQuery {
-  appId: string
-  paginated: boolean
+  appId?: string
+  paginated?: boolean
 }
 
 interface UserDefinition {

packages/pro

@@ -1 +1 @@
-Subproject commit eb96d8b2f2029033b0f758078ed30c888e8fb249
+Subproject commit b28dbd549284cf450be7f25ad85aadf614d08f0b

packages/server/__mocks__/@elastic/elasticsearch.ts

@@ -1,24 +0,0 @@
-const elastic: any = {}
-
-elastic.Client = function () {
-  this.index = jest.fn().mockResolvedValue({ body: [] })
-  this.search = jest.fn().mockResolvedValue({
-    body: {
-      hits: {
-        hits: [
-          {
-            _source: {
-              name: "test",
-            },
-          },
-        ],
-      },
-    },
-  })
-  this.update = jest.fn().mockResolvedValue({ body: [] })
-  this.delete = jest.fn().mockResolvedValue({ body: [] })
-
-  this.close = jest.fn()
-}
-
-module.exports = elastic

packages/server/datasource-sha.env

@@ -1,5 +1,6 @@
-MSSQL_SHA=sha256:3b913841850a4d57fcfcb798be06acc88ea0f2acc5418bc0c140a43e91c4a545
+MSSQL_SHA=sha256:d252932ef839c24c61c1139cc98f69c85ca774fa7c6bfaaa0015b7eb02b9dc87
 MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588ebe
 POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e
 MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d
 MARIADB_SHA=sha256:e59ba8783bf7bc02a4779f103bb0d8751ac0e10f9471089709608377eded7aa8
+ELASTICSEARCH_SHA=sha256:9a6443f55243f6acbfeb4a112d15eb3b9aac74bf25e0e39fa19b3ddd3a6879d0

packages/server/package.json

@@ -156,6 +156,7 @@
     "@types/pouchdb": "6.4.2",
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.14",
+    "@types/swagger-jsdoc": "^6.0.4",
     "@types/tar": "6.1.5",
     "@types/tmp": "0.2.6",
     "@types/uuid": "8.3.4",

packages/server/specs/generate.ts

@@ -4,11 +4,11 @@ import { examples, schemas } from "./resources"
 import * as parameters from "./parameters"
 import * as security from "./security"
 
-const swaggerJsdoc = require("swagger-jsdoc")
+import swaggerJsdoc from "swagger-jsdoc"
 
 const VARIABLES = {}
 
-const options = {
+const opts: swaggerJsdoc.Options = {
   definition: {
     openapi: "3.0.0",
     info: {
@@ -58,30 +58,27 @@ const options = {
 }
 
 function writeFile(output: any, filename: string) {
-  try {
-    const path = join(__dirname, filename)
-    let spec = output
-    if (filename.endsWith("json")) {
-      spec = JSON.stringify(output, null, 2)
-    }
-    // input the static variables
-    for (let [key, replacement] of Object.entries(VARIABLES)) {
-      spec = spec.replace(new RegExp(`{${key}}`, "g"), replacement)
-    }
-    writeFileSync(path, spec)
-    console.log(`Wrote spec to ${path}`)
-    return path
-  } catch (err) {
-    console.error("Error writing spec file", err)
+  const path = join(__dirname, filename)
+  let spec = output
+  if (filename.endsWith("json")) {
+    spec = JSON.stringify(output, null, 2)
   }
+  // input the static variables
+  for (let [key, replacement] of Object.entries(VARIABLES)) {
+    spec = spec.replace(new RegExp(`{${key}}`, "g"), replacement)
+  }
+  writeFileSync(path, spec)
+  console.log(`Wrote spec to ${path}`)
+  return path
+}
+
+export function spec() {
+  return swaggerJsdoc({ ...opts, format: ".json" })
 }
 
 export function run() {
-  const outputJSON = swaggerJsdoc(options)
-  options.format = ".yaml"
-  const outputYAML = swaggerJsdoc(options)
-  writeFile(outputJSON, "openapi.json")
-  return writeFile(outputYAML, "openapi.yaml")
+  writeFile(swaggerJsdoc({ ...opts, format: ".json" }), "openapi.json")
+  return writeFile(swaggerJsdoc({ ...opts, format: ".yaml" }), "openapi.yaml")
 }
 
 if (require.main === module) {

packages/server/specs/resources/error.ts

@@ -0,0 +1,21 @@
+import { object } from "./utils"
+import Resource from "./utils/Resource"
+
+const errorSchema = object({
+  status: {
+    type: "number",
+    description: "The HTTP status code of the error.",
+  },
+  message: {
+    type: "string",
+    description: "A descriptive message about the error.",
+  },
+})
+
+export default new Resource()
+  .setExamples({
+    error: {},
+  })
+  .setSchemas({
+    error: errorSchema,
+  })

packages/server/src/api/controllers/application.ts

@@ -635,6 +635,11 @@ async function unpublishApp(ctx: UserCtx) {
   return result
 }
 
+async function invalidateAppCache(appId: string) {
+  await cache.app.invalidateAppMetadata(dbCore.getDevAppID(appId))
+  await cache.app.invalidateAppMetadata(dbCore.getProdAppID(appId))
+}
+
 async function destroyApp(ctx: UserCtx) {
   let appId = ctx.params.appId
   appId = dbCore.getProdAppID(appId)
@@ -654,17 +659,21 @@ async function destroyApp(ctx: UserCtx) {
   await quotas.removeApp()
   await events.app.deleted(app)
 
-  if (!env.isTest()) {
+  if (!env.USE_LOCAL_COMPONENT_LIBS) {
     await deleteAppFiles(appId)
   }
 
   await removeAppFromUserRoles(ctx, appId)
-  await cache.app.invalidateAppMetadata(devAppId)
+  await invalidateAppCache(appId)
   return result
 }
 
 async function preDestroyApp(ctx: UserCtx) {
-  const { rows } = await getUniqueRows([ctx.params.appId])
+  // invalidate the cache immediately in-case they are leading to
+  // zombie appearing apps
+  const appId = ctx.params.appId
+  await invalidateAppCache(appId)
+  const { rows } = await getUniqueRows([appId])
   ctx.rowCount = rows.length
 }
 

packages/server/src/api/controllers/integration.ts

@@ -9,6 +9,7 @@ import {
 const DISABLED_EXTERNAL_INTEGRATIONS = [
   SourceName.AIRTABLE,
   SourceName.BUDIBASE,
+  SourceName.ARANGODB,
 ]
 
 export async function fetch(ctx: UserCtx<void, FetchIntegrationsResponse>) {