Merge branch 'master' into BUDI-9011

.gitattributes

@@ -0,0 +1 @@
+scripts/resources/minio filter=lfs diff=lfs merge=lfs -text

.github/workflows/budibase_ci.yml

@@ -30,7 +30,7 @@ env:
 
 jobs:
   lint:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -47,7 +47,7 @@ jobs:
       - run: yarn lint
 
   build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -76,7 +76,7 @@ jobs:
           fi
 
   helm-lint:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -88,7 +88,7 @@ jobs:
       - run: cd charts/budibase && helm lint .
 
   test-libraries:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -122,7 +122,7 @@ jobs:
           fi
 
   test-worker:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -151,11 +151,22 @@ jobs:
           yarn test --verbose --reporters=default --reporters=github-actions
 
   test-server:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         datasource:
-          [mssql, mysql, postgres, postgres_legacy, mongodb, mariadb, oracle, sqs, none]
+          [
+            mssql,
+            mysql,
+            postgres,
+            postgres_legacy,
+            mongodb,
+            mariadb,
+            oracle,
+            sqs,
+            elasticsearch,
+            none,
+          ]
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -192,6 +203,8 @@ jobs:
             docker pull budibase/oracle-database:23.2-slim-faststart
           elif [ "${{ matrix.datasource }}" == "postgres_legacy" ]; then
             docker pull postgres:9.5.25
+          elif [ "${{ matrix.datasource }}" == "elasticsearch" ]; then
+            docker pull elasticsearch@${{ steps.dotenv.outputs.ELASTICSEARCH_SHA }}
           fi
           docker pull minio/minio &
           docker pull redis &
@@ -240,7 +253,7 @@ jobs:
           yarn test --filter $FILTER --verbose --reporters=default --reporters=github-actions
 
   check-pro-submodule:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
     steps:
       - name: Checkout repo and submodules
@@ -299,7 +312,7 @@ jobs:
           fi
 
   check-lockfile:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     if: inputs.run_as_oss != true  && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase')
     steps:
       - name: Checkout repo

.github/workflows/readme-openapi.yml

@@ -20,7 +20,7 @@ jobs:
       - run: yarn --frozen-lockfile
 
       - name: Install OpenAPI pkg
-        run: yarn global add openapi
+        run: yarn global add rdme@8.6.6
 
       - name: update specs
-        run: cd packages/server && yarn specs && openapi specs/openapi.yaml --key=${{ secrets.README_API_KEY }} --id=6728a74f5918b50036c61841
+        run: cd packages/server && yarn specs && rdme openapi specs/openapi.yaml --key=${{ secrets.README_API_KEY }} --id=67c16880add6da002352069a

docs/CONTRIBUTING.md

@@ -108,7 +108,7 @@ You can install them following any of the steps described below:
   - Installation steps: https://asdf-vm.com/guide/getting-started.html
   - asdf plugin add nodejs
   - asdf plugin add python
-  - npm install -g yarn
+  - asdf plugin add yarn
 
 ### Using NVM and pyenv
 

eslint.config.mjs

@@ -94,6 +94,15 @@ export default [
         allowImportExportEverywhere: true,
       },
     },
+
+    plugins: {
+      ...config.plugins,
+      "@typescript-eslint": tseslint.plugin,
+    },
+    rules: {
+      ...config.rules,
+      "@typescript-eslint/consistent-type-imports": "error",
+    },
   })),
   ...tseslint.configs.strict.map(config => ({
     ...config,

hosting/nginx.dev.conf

@@ -63,6 +63,11 @@ http {
       proxy_send_timeout 120s;
       proxy_http_version 1.1;
 
+      # Enable buffering for potentially large OIDC configs
+      proxy_buffering on;
+      proxy_buffer_size 16k;
+      proxy_buffers 4 32k;
+
       proxy_set_header Host $host;
       proxy_set_header Connection "";
 

hosting/single/Dockerfile

@@ -1,5 +1,5 @@
 ARG BASEIMG=budibase/couchdb:v3.3.3-sqs-v2.1.1
-FROM node:20-slim as build
+FROM node:20-slim AS build
 
 # install node-gyp dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends g++ make python3 jq
@@ -34,13 +34,13 @@ COPY packages/worker/dist packages/worker/dist
 COPY packages/worker/pm2.config.js packages/worker/pm2.config.js
 
 
-FROM $BASEIMG as runner
+FROM $BASEIMG AS runner
 ARG TARGETARCH
-ENV TARGETARCH $TARGETARCH
+ENV TARGETARCH=$TARGETARCH
 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service)
 # e.g. docker build --build-arg TARGETBUILD=aas ....
 ARG TARGETBUILD=single
-ENV TARGETBUILD $TARGETBUILD
+ENV TARGETBUILD=$TARGETBUILD
 
 # install base dependencies
 RUN apt-get update && \
@@ -67,6 +67,12 @@ RUN mkdir -p /var/log/nginx && \
 
 # setup minio
 WORKDIR /minio
+
+# a 2022 version of minio that supports gateway mode
+COPY scripts/resources/minio /minio
+RUN chmod +x minio
+
+# handles the installation of minio in non-aas environments
 COPY scripts/install-minio.sh ./install.sh
 RUN chmod +x install.sh && ./install.sh
 
@@ -86,7 +92,7 @@ COPY hosting/single/ssh/sshd_config /etc/
 COPY hosting/single/ssh/ssh_setup.sh /tmp
 
 #  setup letsencrypt certificate
-RUN apt-get install -y certbot python3-certbot-nginx
+RUN apt-get update && apt-get install -y certbot python3-certbot-nginx
 COPY hosting/letsencrypt /app/letsencrypt
 RUN chmod +x /app/letsencrypt/certificate-request.sh /app/letsencrypt/certificate-renew.sh
 

hosting/single/runner.sh

@@ -1,53 +1,61 @@
 #!/bin/bash
-declare -a ENV_VARS=("COUCHDB_USER" "COUCHDB_PASSWORD" "DATA_DIR" "MINIO_ACCESS_KEY" "MINIO_SECRET_KEY" "INTERNAL_API_KEY" "JWT_SECRET" "REDIS_PASSWORD")
-declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONMENT" "CLUSTER_PORT" "DEPLOYMENT_ENVIRONMENT" "MINIO_URL" "NODE_ENV" "POSTHOG_TOKEN" "REDIS_URL" "SELF_HOSTED" "WORKER_PORT" "WORKER_URL" "TENANT_FEATURE_FLAGS" "ACCOUNT_PORTAL_URL")
-# Check the env vars set in Dockerfile have come through, AAS seems to drop them
-[[ -z "${APP_PORT}" ]] && export APP_PORT=4001
-[[ -z "${ARCHITECTURE}" ]] && export ARCHITECTURE=amd
-[[ -z "${BUDIBASE_ENVIRONMENT}" ]] && export BUDIBASE_ENVIRONMENT=PRODUCTION
-[[ -z "${CLUSTER_PORT}" ]] && export CLUSTER_PORT=80
-[[ -z "${DEPLOYMENT_ENVIRONMENT}" ]] && export DEPLOYMENT_ENVIRONMENT=docker
-[[ -z "${MINIO_URL}" ]] && [[ -z "${USE_S3}" ]] && export MINIO_URL=http://127.0.0.1:9000
-[[ -z "${NODE_ENV}" ]] && export NODE_ENV=production
-[[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
-[[ -z "${ACCOUNT_PORTAL_URL}" ]] && export ACCOUNT_PORTAL_URL=https://account.budibase.app
-[[ -z "${REDIS_URL}" ]] && export REDIS_URL=127.0.0.1:6379
-[[ -z "${SELF_HOSTED}" ]] && export SELF_HOSTED=1
-[[ -z "${WORKER_PORT}" ]] && export WORKER_PORT=4002
-[[ -z "${WORKER_URL}" ]] && export WORKER_URL=http://127.0.0.1:4002
-[[ -z "${APPS_URL}" ]] && export APPS_URL=http://127.0.0.1:4001
-[[ -z "${SERVER_TOP_LEVEL_PATH}" ]] && export SERVER_TOP_LEVEL_PATH=/app
-#  export CUSTOM_DOMAIN=budi001.custom.com
 
-# Azure App Service customisations
+echo "Starting runner.sh..."
-if [[ "${TARGETBUILD}" = "aas" ]]; then
+
-    export DATA_DIR="${DATA_DIR:-/home}"
+# set defaults for Docker-related variables
-    WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
+export APP_PORT="${APP_PORT:-4001}"
-    /etc/init.d/ssh start
+export ARCHITECTURE="${ARCHITECTURE:-amd}"
-else
+export BUDIBASE_ENVIRONMENT="${BUDIBASE_ENVIRONMENT:-PRODUCTION}"
-    export DATA_DIR=${DATA_DIR:-/data}
+export CLUSTER_PORT="${CLUSTER_PORT:-80}"
+export DEPLOYMENT_ENVIRONMENT="${DEPLOYMENT_ENVIRONMENT:-docker}"
+
+# only set MINIO_URL if neither MINIO_URL nor USE_S3 is set
+if [[ -z "${MINIO_URL}" && -z "${USE_S3}" ]]; then
+  export MINIO_URL="http://127.0.0.1:9000"
 fi
-mkdir -p ${DATA_DIR}
+
-# Mount NFS or GCP Filestore if env vars exist for it
+export NODE_ENV="${NODE_ENV:-production}"
-if [[ ! -z ${FILESHARE_IP} && ! -z ${FILESHARE_NAME} ]]; then
+export POSTHOG_TOKEN="${POSTHOG_TOKEN:-phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU}"
+export ACCOUNT_PORTAL_URL="${ACCOUNT_PORTAL_URL:-https://account.budibase.app}"
+export REDIS_URL="${REDIS_URL:-127.0.0.1:6379}"
+export SELF_HOSTED="${SELF_HOSTED:-1}"
+export WORKER_PORT="${WORKER_PORT:-4002}"
+export WORKER_URL="${WORKER_URL:-http://127.0.0.1:4002}"
+export APPS_URL="${APPS_URL:-http://127.0.0.1:4001}"
+export SERVER_TOP_LEVEL_PATH="${SERVER_TOP_LEVEL_PATH:-/app}"
+
+# set DATA_DIR and ensure the directory exists
+if [[ ${TARGETBUILD} == "aas" ]]; then
+    export DATA_DIR="/home"
+else
+    export DATA_DIR="${DATA_DIR:-/data}"
+fi
+mkdir -p "${DATA_DIR}"
+
+# mount NFS or GCP Filestore if FILESHARE_IP and FILESHARE_NAME are set
+if [[ -n "${FILESHARE_IP}" && -n "${FILESHARE_NAME}" ]]; then
     echo "Mounting NFS share"
     apt update && apt install -y nfs-common nfs-kernel-server
     echo "Mount file share ${FILESHARE_IP}:/${FILESHARE_NAME} to ${DATA_DIR}"
-    mount -o nolock ${FILESHARE_IP}:/${FILESHARE_NAME} ${DATA_DIR}
+    mount -o nolock "${FILESHARE_IP}:/${FILESHARE_NAME}" "${DATA_DIR}"
     echo "Mounting result: $?"
 fi
 
-if [ -f "${DATA_DIR}/.env" ]; then
+# source environment variables from a .env file if it exists in DATA_DIR
-    # Read in the .env file and export the variables
+if [[ -f "${DATA_DIR}/.env" ]]; then
-    for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done
+    set -a  # Automatically export all variables loaded from .env
+    source "${DATA_DIR}/.env"
+    set +a
 fi
-# randomise any unset environment variables
+
-for ENV_VAR in "${ENV_VARS[@]}"
+# randomize any unset sensitive environment variables using uuidgen
-do
+env_vars=(COUCHDB_USER COUCHDB_PASSWORD MINIO_ACCESS_KEY MINIO_SECRET_KEY INTERNAL_API_KEY JWT_SECRET REDIS_PASSWORD)
-    if [[ -z "${!ENV_VAR}" ]]; then
+for var in "${env_vars[@]}"; do
-        eval "export $ENV_VAR=$(uuidgen | sed -e 's/-//g')"
+    if [[ -z "${!var}" ]]; then
+        export "$var"="$(uuidgen | tr -d '-')"
     fi
 done
+
 if [[ -z "${COUCH_DB_URL}" ]]; then
     export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@127.0.0.1:5984
 fi
@@ -58,13 +66,11 @@ fi
 
 if [ ! -f "${DATA_DIR}/.env" ]; then
     touch ${DATA_DIR}/.env
-    for ENV_VAR in "${ENV_VARS[@]}"
+    for ENV_VAR in "${ENV_VARS[@]}"; do
-    do
         temp=$(eval "echo \$$ENV_VAR")
         echo "$ENV_VAR=$temp" >>${DATA_DIR}/.env
     done
-    for ENV_VAR in "${DOCKER_VARS[@]}"
+    for ENV_VAR in "${DOCKER_VARS[@]}"; do
-    do
         temp=$(eval "echo \$$ENV_VAR")
         echo "$ENV_VAR=$temp" >>${DATA_DIR}/.env
     done
@@ -79,6 +85,7 @@ ln -s ${DATA_DIR}/.env /worker/.env
 # make these directories in runner, incase of mount
 mkdir -p ${DATA_DIR}/minio
 mkdir -p ${DATA_DIR}/redis
+mkdir -p ${DATA_DIR}/couch
 chown -R couchdb:couchdb ${DATA_DIR}/couch
 
 REDIS_CONFIG="/etc/redis/redis.conf"
@@ -93,12 +100,24 @@ if [[ -n "${REDIS_PASSWORD}" ]]; then
 else
     redis-server "${REDIS_CONFIG}" >/dev/stdout 2>&1 &
 fi
-/bbcouch-runner.sh &
+
+echo "Starting callback CouchDB runner..."
+./bbcouch-runner.sh &
 
 # only start minio if use s3 isn't passed
 if [[ -z "${USE_S3}" ]]; then
+    if [[ ${TARGETBUILD} == aas ]]; then
+        echo "Starting MinIO in Azure Gateway mode"
+        if [[ -z "${AZURE_STORAGE_ACCOUNT}" || -z "${AZURE_STORAGE_KEY}" || -z "${MINIO_ACCESS_KEY}" || -z "${MINIO_SECRET_KEY}" ]]; then
+            echo "The following environment variables must be set: AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_KEY, MINIO_ACCESS_KEY, MINIO_SECRET_KEY"
+            exit 1
+        fi
+        /minio/minio gateway azure --console-address ":9001" >/dev/stdout 2>&1 &
+    else
+        echo "Starting MinIO in standalone mode"
         /minio/minio server --console-address ":9001" ${DATA_DIR}/minio >/dev/stdout 2>&1 &
     fi
+fi
 
 /etc/init.d/nginx restart
 if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.4.12",
+  "version": "3.4.22",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/backend-core/src/context/mainContext.ts

@@ -123,7 +123,7 @@ export async function doInAutomationContext<T>(params: {
   task: () => T
 }): Promise<T> {
   await ensureSnippetContext()
-  return newContext(
+  return await newContext(
     {
       tenantId: getTenantIDFromAppID(params.appId),
       appId: params.appId,
@@ -266,9 +266,9 @@ export const getProdAppId = () => {
   return conversions.getProdAppID(appId)
 }
 
-export function doInEnvironmentContext(
+export function doInEnvironmentContext<T>(
   values: Record<string, string>,
-  task: any
+  task: () => T
 ) {
   if (!values) {
     throw new Error("Must supply environment variables.")

packages/backend-core/src/objectStore/objectStore.ts

@@ -478,7 +478,7 @@ export async function deleteFolder(
   if (existingObjectsResponse.Contents?.length === 0) {
     return
   }
-  const deleteParams: any = {
+  const deleteParams: { Bucket: string; Delete: { Objects: any[] } } = {
     Bucket: bucketName,
     Delete: {
       Objects: [],
@@ -489,12 +489,14 @@ export async function deleteFolder(
     deleteParams.Delete.Objects.push({ Key: content.Key })
   })
 
+  if (deleteParams.Delete.Objects.length) {
     const deleteResponse = await client.deleteObjects(deleteParams)
     // can only empty 1000 items at once
     if (deleteResponse.Deleted?.length === 1000) {
       return deleteFolder(bucketName, folder)
     }
   }
+}
 
 export async function uploadDirectory(
   bucketName: string,

packages/backend-core/src/sql/utils.ts

@@ -5,10 +5,10 @@ import {
   SqlQuery,
   Table,
   TableSourceType,
+  SEPARATOR,
 } from "@budibase/types"
 import { DEFAULT_BB_DATASOURCE_ID } from "../constants"
 import { Knex } from "knex"
-import { SEPARATOR } from "../db"
 import environment from "../environment"
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`

packages/backend-core/src/utils/Duration.ts

@@ -15,23 +15,27 @@ const conversion: Record<DurationType, number> = {
 }
 
 export class Duration {
+  constructor(public ms: number) {}
+
+  to(type: DurationType) {
+    return this.ms / conversion[type]
+  }
+
+  toMs() {
+    return this.ms
+  }
+
+  toSeconds() {
+    return this.to(DurationType.SECONDS)
+  }
+
   static convert(from: DurationType, to: DurationType, duration: number) {
     const milliseconds = duration * conversion[from]
     return milliseconds / conversion[to]
   }
 
   static from(from: DurationType, duration: number) {
-    return {
+    return new Duration(duration * conversion[from])
-      to: (to: DurationType) => {
-        return Duration.convert(from, to, duration)
-      },
-      toMs: () => {
-        return Duration.convert(from, DurationType.MILLISECONDS, duration)
-      },
-      toSeconds: () => {
-        return Duration.convert(from, DurationType.SECONDS, duration)
-      },
-    }
   }
 
   static fromSeconds(duration: number) {

packages/backend-core/src/utils/index.ts

@@ -2,3 +2,4 @@ export * from "./hashing"
 export * from "./utils"
 export * from "./stringUtils"
 export * from "./Duration"
+export * from "./time"

packages/backend-core/src/utils/time.ts

@@ -0,0 +1,7 @@
+import { Duration } from "./Duration"
+
+export async function time<T>(f: () => Promise<T>): Promise<[T, Duration]> {
+  const start = performance.now()
+  const result = await f()
+  return [result, Duration.fromMilliseconds(performance.now() - start)]
+}

packages/bbui/src/Actions/position_dropdown.ts

@@ -27,7 +27,7 @@ export type UpdateHandler = (
 
 interface Opts {
   anchor?: HTMLElement
-  align: PopoverAlignment
+  align: PopoverAlignment | `${PopoverAlignment}`
   maxHeight?: number
   maxWidth?: number
   minWidth?: number

packages/bbui/src/Badge/Badge.svelte

@@ -11,6 +11,7 @@
   export let active = false
   export let inactive = false
   export let hoverable = false
+  export let outlineColor = null
 </script>
 
 <!-- svelte-ignore a11y-no-static-element-interactions -->
@@ -29,6 +30,7 @@
   class:spectrum-Label--seafoam={seafoam}
   class:spectrum-Label--active={active}
   class:spectrum-Label--inactive={inactive}
+  style={outlineColor ? `border: 2px solid ${outlineColor}` : ""}
 >
   <slot />
 </span>

packages/bbui/src/ClearButton/ClearButton.svelte

@@ -1,6 +1,6 @@
-<script>
+<script lang="ts">
-  export let small = false
+  export let small: boolean = false
-  export let disabled
+  export let disabled: boolean = false
 </script>
 
 <button

packages/bbui/src/Drawer/Drawer.svelte

@@ -64,7 +64,7 @@
   import { setContext, createEventDispatcher, onDestroy } from "svelte"
   import { generate } from "shortid"
 
-  export let title
+  export let title = ""
   export let forceModal = false
 
   const dispatch = createEventDispatcher()

packages/bbui/src/Form/Checkbox.svelte

@@ -1,24 +1,24 @@
-<script>
+<script lang="ts">
   import Field from "./Field.svelte"
   import Checkbox from "./Core/Checkbox.svelte"
   import { createEventDispatcher } from "svelte"
 
-  export let value = null
+  export let value: boolean | undefined = undefined
-  export let label = null
+  export let label: string | undefined = undefined
-  export let labelPosition = "above"
+  export let labelPosition: "above" | "below" = "above"
-  export let text = null
+  export let text: string | undefined = undefined
-  export let disabled = false
+  export let disabled: boolean = false
-  export let error = null
+  export let error: string | undefined = undefined
-  export let size = "M"
+  export let size: "S" | "M" | "L" | "XL" = "M"
-  export let helpText = null
+  export let helpText: string | undefined = undefined
 
   const dispatch = createEventDispatcher()
-  const onChange = e => {
+  const onChange = (e: CustomEvent<boolean>) => {
     value = e.detail
     dispatch("change", e.detail)
   }
 </script>
 
 <Field {helpText} {label} {labelPosition} {error}>
-  <Checkbox {error} {disabled} {text} {value} {size} on:change={onChange} />
+  <Checkbox {disabled} {text} {value} {size} on:change={onChange} />
 </Field>

packages/bbui/src/Icon/Icon.svelte

@@ -28,6 +28,9 @@
     <svg
       on:contextmenu
       on:click
+      on:mouseover
+      on:mouseleave
+      on:focus
       class:hoverable
       class:disabled
       class="spectrum-Icon spectrum-Icon--size{size}"

packages/bbui/src/Layout/Page.svelte

@@ -47,7 +47,7 @@
     overflow-x: hidden;
   }
   .main {
-    overflow-y: scroll;
+    overflow-y: auto;
   }
   .content {
     display: flex;

packages/bbui/src/Link/Link.svelte

@@ -10,7 +10,7 @@
   export let secondary: boolean = false
   export let overBackground: boolean = false
   export let target: string | undefined = undefined
-  export let download: boolean | undefined = undefined
+  export let download: string | undefined = undefined
   export let disabled: boolean = false
   export let tooltip: string | null = null
 

packages/bbui/src/Popover/Popover.svelte

@@ -18,8 +18,9 @@
   import type { KeyboardEventHandler } from "svelte/elements"
   import { PopoverAlignment } from "../constants"
 
-  export let anchor: HTMLElement
+  export let anchor: HTMLElement | undefined
-  export let align: PopoverAlignment = PopoverAlignment.Right
+  export let align: PopoverAlignment | `${PopoverAlignment}` =
+    PopoverAlignment.Right
   export let portalTarget: string | undefined = undefined
   export let minWidth: number | undefined = undefined
   export let maxWidth: number | undefined = undefined

packages/bbui/src/SideNavigation/Item.svelte

@@ -1,63 +0,0 @@
-<script>
-  import { getContext } from "svelte"
-
-  const multilevel = getContext("sidenav-type")
-  import Badge from "../Badge/Badge.svelte"
-
-  export let href = ""
-  export let external = false
-  export let heading = ""
-  export let icon = ""
-  export let selected = false
-  export let disabled = false
-  export let badge = ""
-</script>
-
-<!-- svelte-ignore a11y-click-events-have-key-events -->
-<!-- svelte-ignore a11y-no-noninteractive-element-interactions -->
-<li
-  class="spectrum-SideNav-item"
-  class:is-selected={selected}
-  class:is-disabled={disabled}
-  on:click
->
-  {#if heading}
-    <h2 class="spectrum-SideNav-heading" id="nav-heading-{heading}">
-      {heading}
-    </h2>
-  {/if}
-  <a
-    target={external ? "_blank" : ""}
-    {href}
-    class="spectrum-SideNav-itemLink"
-    aria-current="page"
-  >
-    {#if icon}
-      <svg
-        class="spectrum-Icon spectrum-Icon--sizeM spectrum-SideNav-itemIcon"
-        focusable="false"
-        aria-hidden="true"
-      >
-        <use xlink:href="#spectrum-icon-18-{icon}" />
-      </svg>
-    {/if}
-    <slot />
-    {#if badge}
-      <div class="badge">
-        <Badge active size="S">{badge}</Badge>
-      </div>
-    {/if}
-  </a>
-
-  {#if multilevel && $$slots.subnav}
-    <ul class="spectrum-SideNav">
-      <slot name="subnav" />
-    </ul>
-  {/if}
-</li>
-
-<style>
-  .badge {
-    margin-left: 10px;
-  }
-</style>

packages/bbui/src/SideNavigation/Navigation.svelte

@@ -1,13 +0,0 @@
-<script>
-  import { setContext } from "svelte"
-  import "@spectrum-css/sidenav/dist/index-vars.css"
-
-  export let multilevel = false
-  setContext("sidenav-type", multilevel)
-</script>
-
-<nav>
-  <ul class="spectrum-SideNav" class:spectrum-SideNav--multiLevel={multilevel}>
-    <slot />
-  </ul>
-</nav>

packages/bbui/src/Stores/banner.ts

@@ -7,17 +7,26 @@ export const BANNER_TYPES = {
   WARNING: "warning",
 }
 
+interface BannerConfig {
+  message?: string
+  type?: string
+  extraButtonText?: string
+  extraButtonAction?: () => void
+  onChange?: () => void
+}
+
+interface DefaultConfig {
+  messages: BannerConfig[]
+}
+
 export function createBannerStore() {
-  const DEFAULT_CONFIG = {
+  const DEFAULT_CONFIG: DefaultConfig = {
     messages: [],
   }
 
-  const banner = writable(DEFAULT_CONFIG)
+  const banner = writable<DefaultConfig>(DEFAULT_CONFIG)
 
-  const show = async (
+  const show = async (config: BannerConfig = {}) => {
-    // eslint-disable-next-line
-    config = { message, type, extraButtonText, extraButtonAction, onChange }
-  ) => {
     banner.update(store => {
       return {
         ...store,
@@ -27,7 +36,7 @@ export function createBannerStore() {
   }
 
   const showStatus = async () => {
-    const config = {
+    const config: BannerConfig = {
       message: "Some systems are experiencing issues",
       type: BANNER_TYPES.NEGATIVE,
       extraButtonText: "View Status",
@@ -37,18 +46,24 @@ export function createBannerStore() {
     await queue([config])
   }
 
-  const queue = async entries => {
+  const queue = async (entries: Array<BannerConfig>) => {
-    const priority = {
+    const priority: Record<string, number> = {
       [BANNER_TYPES.NEGATIVE]: 0,
       [BANNER_TYPES.WARNING]: 1,
       [BANNER_TYPES.INFO]: 2,
     }
     banner.update(store => {
       const sorted = [...store.messages, ...entries].sort((a, b) => {
-        if (priority[a.type] == priority[b.type]) {
+        if (
+          priority[a.type as keyof typeof priority] ===
+          priority[b.type as keyof typeof priority]
+        ) {
           return 0
         }
-        return priority[a.type] < priority[b.type] ? -1 : 1
+        return priority[a.type as keyof typeof priority] <
+          priority[b.type as keyof typeof priority]
+          ? -1
+          : 1
       })
       return {
         ...store,

packages/bbui/src/Stores/notifications.ts

@@ -2,9 +2,21 @@ import { writable } from "svelte/store"
 
 const NOTIFICATION_TIMEOUT = 3000
 
+interface Notification {
+  id: string
+  type: string
+  message: string
+  icon: string
+  dismissable: boolean
+  action: (() => void) | null
+  actionMessage: string | null
+  wide: boolean
+  dismissTimeout: number
+}
+
 export const createNotificationStore = () => {
-  const timeoutIds = new Set()
+  const timeoutIds = new Set<ReturnType<typeof setTimeout>>()
-  const _notifications = writable([], () => {
+  const _notifications = writable<Notification[]>([], () => {
     return () => {
       // clear all the timers
       timeoutIds.forEach(timeoutId => {
@@ -21,7 +33,7 @@ export const createNotificationStore = () => {
   }
 
   const send = (
-    message,
+    message: string,
     {
       type = "default",
       icon = "",
@@ -30,7 +42,15 @@ export const createNotificationStore = () => {
       actionMessage = null,
       wide = false,
       dismissTimeout = NOTIFICATION_TIMEOUT,
-    }
+    }: {
+      type?: string
+      icon?: string
+      autoDismiss?: boolean
+      action?: (() => void) | null
+      actionMessage?: string | null
+      wide?: boolean
+      dismissTimeout?: number
+    } = {}
   ) => {
     if (block) {
       return
@@ -60,7 +80,7 @@ export const createNotificationStore = () => {
     }
   }
 
-  const dismissNotification = id => {
+  const dismissNotification = (id: string) => {
     _notifications.update(state => {
       return state.filter(n => n.id !== id)
     })
@@ -71,17 +91,18 @@ export const createNotificationStore = () => {
   return {
     subscribe,
     send,
-    info: msg => send(msg, { type: "info", icon: "Info" }),
+    info: (msg: string) => send(msg, { type: "info", icon: "Info" }),
-    error: msg =>
+    error: (msg: string) =>
       send(msg, { type: "error", icon: "Alert", autoDismiss: false }),
-    warning: msg => send(msg, { type: "warning", icon: "Alert" }),
+    warning: (msg: string) => send(msg, { type: "warning", icon: "Alert" }),
-    success: msg => send(msg, { type: "success", icon: "CheckmarkCircle" }),
+    success: (msg: string) =>
+      send(msg, { type: "success", icon: "CheckmarkCircle" }),
     blockNotifications,
     dismiss: dismissNotification,
   }
 }
 
-function id() {
+function id(): string {
   return "_" + Math.random().toString(36).slice(2, 9)
 }
 

packages/bbui/src/Table/ArrayRenderer.svelte

@@ -1,8 +1,8 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/label/dist/index-vars.css"
   import Badge from "../Badge/Badge.svelte"
 
-  export let value
+  export let value: string | string[]
 
   const displayLimit = 5
 

packages/bbui/src/Table/AttachmentRenderer.svelte

@@ -1,15 +1,15 @@
-<script>
+<script lang="ts">
   import Link from "../Link/Link.svelte"
 
-  export let value
+  export let value: { name: string; url: string; extension: string }[]
 
   const displayLimit = 5
   $: attachments = value?.slice(0, displayLimit) ?? []
   $: leftover = (value?.length ?? 0) - attachments.length
 
-  const imageExtensions = ["png", "tiff", "gif", "raw", "jpg", "jpeg"]
+  const imageExtensions: string[] = ["png", "tiff", "gif", "raw", "jpg", "jpeg"]
-  const isImage = extension => {
+  const isImage = (extension: string | undefined): boolean => {
-    return imageExtensions.includes(extension?.toLowerCase())
+    return imageExtensions.includes(extension?.toLowerCase() ?? "")
   }
 </script>
 

packages/bbui/src/Table/BoldRenderer.svelte

@@ -1,5 +1,5 @@
-<script>
+<script lang="ts">
-  export let value
+  export let value: string
 </script>
 
 <div class="bold">{value}</div>

packages/bbui/src/Table/BooleanRenderer.svelte

@@ -1,7 +1,7 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/checkbox/dist/index-vars.css"
 
-  export let value
+  export let value: boolean
 </script>
 
 <label

packages/bbui/src/Table/CellRenderer.svelte

@@ -1,4 +1,4 @@
-<script>
+<script lang="ts">
   import StringRenderer from "./StringRenderer.svelte"
   import BooleanRenderer from "./BooleanRenderer.svelte"
   import DateTimeRenderer from "./DateTimeRenderer.svelte"
@@ -8,14 +8,14 @@
   import InternalRenderer from "./InternalRenderer.svelte"
   import { processStringSync } from "@budibase/string-templates"
 
-  export let row
+  export let row: Record<string, any>
-  export let schema
+  export let schema: Record<string, any>
-  export let value
+  export let value: Record<string, any>
-  export let customRenderers = []
+  export let customRenderers: { column: string; component: any }[] = []
-  export let snippets
+  export let snippets: any
 
-  let renderer
+  let renderer: any
-  const typeMap = {
+  const typeMap: Record<string, any> = {
     boolean: BooleanRenderer,
     datetime: DateTimeRenderer,
     link: RelationshipRenderer,
@@ -33,7 +33,7 @@
   $: renderer = customRenderer?.component ?? typeMap[type] ?? StringRenderer
   $: cellValue = getCellValue(value, schema.template)
 
-  const getType = schema => {
+  const getType = (schema: any): string => {
     // Use a string renderer for dates if we use a custom template
     if (schema?.type === "datetime" && schema?.template) {
       return "string"
@@ -41,7 +41,7 @@
     return schema?.type || "string"
   }
 
-  const getCellValue = (value, template) => {
+  const getCellValue = (value: any, template: string | undefined): any => {
     if (!template) {
       return value
     }

packages/bbui/src/Table/CodeRenderer.svelte

@@ -1,5 +1,5 @@
-<script>
+<script lang="ts">
-  export let value
+  export let value: string
 </script>
 
 <code>{value}</code>

packages/bbui/src/Table/DateTimeRenderer.svelte

@@ -1,13 +1,13 @@
-<script>
+<script lang="ts">
   import dayjs from "dayjs"
 
-  export let value
+  export let value: string | Date
-  export let schema
+  export let schema: { timeOnly?: boolean; dateOnly?: boolean }
 
   // adding the 0- will turn a string like 00:00:00 into a valid ISO
   // date, but will make actual ISO dates invalid
   $: time = new Date(`0-${value}`)
-  $: isTimeOnly = !isNaN(time) || schema?.timeOnly
+  $: isTimeOnly = !isNaN(time as any) || schema?.timeOnly
   $: isDateOnly = schema?.dateOnly
   $: format = isTimeOnly
     ? "HH:mm:ss"

packages/bbui/src/Table/InternalRenderer.svelte

@@ -1,11 +1,11 @@
-<script>
+<script lang="ts">
   import Icon from "../Icon/Icon.svelte"
   import { copyToClipboard } from "../helpers"
   import { notifications } from "../Stores/notifications"
 
-  export let value
+  export let value: string
 
-  const onClick = async e => {
+  const onClick = async (e: MouseEvent): Promise<void> => {
     e.stopPropagation()
     try {
       await copyToClipboard(value)

packages/bbui/src/Table/RelationshipRenderer.svelte

@@ -1,11 +1,11 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/label/dist/index-vars.css"
   import { createEventDispatcher } from "svelte"
   import Badge from "../Badge/Badge.svelte"
 
-  export let row
+  export let row: { tableId: string; _id: string }
-  export let value
+  export let value: { primaryDisplay?: string }[] | undefined
-  export let schema
+  export let schema: { name?: string }
 
   const dispatch = createEventDispatcher()
   const displayLimit = 5
@@ -13,7 +13,7 @@
   $: relationships = value?.slice(0, displayLimit) ?? []
   $: leftover = (value?.length ?? 0) - relationships.length
 
-  const onClick = e => {
+  const onClick = (e: MouseEvent) => {
     e.stopPropagation()
     dispatch("clickrelationship", {
       tableId: row.tableId,

packages/bbui/src/Table/SelectEditRenderer.svelte

@@ -1,12 +1,12 @@
-<script>
+<script lang="ts">
   import Checkbox from "../Form/Checkbox.svelte"
   import ActionButton from "../ActionButton/ActionButton.svelte"
 
-  export let selected
+  export let selected: boolean | undefined
-  export let onEdit
+  export let onEdit: (_e: Event) => void
-  export let allowSelectRows = false
+  export let allowSelectRows: boolean = false
-  export let allowEditRows = false
+  export let allowEditRows: boolean = false
-  export let data
+  export let data: Record<string, any>
 </script>
 
 <div>

packages/bbui/src/Table/StringRenderer.svelte

@@ -1,6 +1,6 @@
-<script>
+<script lang="ts">
-  export let value
+  export let value: string | object
-  export let schema
+  export let schema: { capitalise?: boolean }
 </script>
 
 <div class:capitalise={schema?.capitalise}>

packages/bbui/src/Table/Table.svelte

@@ -1,4 +1,4 @@
-<script>
+<script lang="ts">
   import { createEventDispatcher } from "svelte"
   import "@spectrum-css/table/dist/index-vars.css"
   import CellRenderer from "./CellRenderer.svelte"
@@ -7,6 +7,7 @@
   import ProgressCircle from "../ProgressCircle/ProgressCircle.svelte"
   import Checkbox from "../Form/Checkbox.svelte"
 
+  /**
    /**
    * The expected schema is our normal couch schemas for our tables.
    * Each field schema can be enriched with a few extra properties to customise
@@ -24,42 +25,42 @@
    * borderLeft: show a left border
    * borderRight: show a right border
    */
-  export let data = []
+  export let data: any[] = []
-  export let schema = {}
+  export let schema: Record<string, any> = {}
-  export let showAutoColumns = false
+  export let showAutoColumns: boolean = false
-  export let rowCount = 0
+  export let rowCount: number = 0
-  export let quiet = false
+  export let quiet: boolean = false
-  export let loading = false
+  export let loading: boolean = false
-  export let allowSelectRows
+  export let allowSelectRows: boolean = false
-  export let allowEditRows = true
+  export let allowEditRows: boolean = true
-  export let allowEditColumns = true
+  export let allowEditColumns: boolean = true
-  export let allowClickRows = true
+  export let allowClickRows: boolean = true
-  export let selectedRows = []
+  export let selectedRows: any[] = []
-  export let customRenderers = []
+  export let customRenderers: any[] = []
-  export let disableSorting = false
+  export let disableSorting: boolean = false
-  export let autoSortColumns = true
+  export let autoSortColumns: boolean = true
-  export let compact = false
+  export let compact: boolean = false
-  export let customPlaceholder = false
+  export let customPlaceholder: boolean = false
-  export let showHeaderBorder = true
+  export let showHeaderBorder: boolean = true
-  export let placeholderText = "No rows found"
+  export let placeholderText: string = "No rows found"
-  export let snippets = []
+  export let snippets: any[] = []
-  export let defaultSortColumn = undefined
+  export let defaultSortColumn: string | undefined = undefined
-  export let defaultSortOrder = "Ascending"
+  export let defaultSortOrder: "Ascending" | "Descending" = "Ascending"
 
   const dispatch = createEventDispatcher()
 
   // Config
-  const headerHeight = 36
+  const headerHeight: number = 36
   $: rowHeight = compact ? 46 : 55
 
   // Sorting state
-  let sortColumn
+  let sortColumn: string | undefined
-  let sortOrder
+  let sortOrder: "Ascending" | "Descending" | undefined
 
   // Table state
-  let height = 0
+  let height: number = 0
-  let loaded = false
+  let loaded: boolean = false
-  let checkboxStatus = false
+  let checkboxStatus: boolean = false
 
   $: schema = fixSchema(schema)
   $: if (!loading) loaded = true
@@ -95,8 +96,8 @@
     }
   }
 
-  const fixSchema = schema => {
+  const fixSchema = (schema: Record<string, any>): Record<string, any> => {
-    let fixedSchema = {}
+    let fixedSchema: Record<string, any> = {}
     Object.entries(schema || {}).forEach(([fieldName, fieldSchema]) => {
       if (typeof fieldSchema === "string") {
         fixedSchema[fieldName] = {
@@ -120,7 +121,13 @@
     return fixedSchema
   }
 
-  const getVisibleRowCount = (loaded, height, allRows, rowCount, rowHeight) => {
+  const getVisibleRowCount = (
+    loaded: boolean,
+    height: number,
+    allRows: number,
+    rowCount: number,
+    rowHeight: number
+  ): number => {
     if (!loaded) {
       return rowCount || 0
     }
@@ -131,12 +138,12 @@
   }
 
   const getHeightStyle = (
-    visibleRowCount,
+    visibleRowCount: number,
-    rowCount,
+    rowCount: number,
-    totalRowCount,
+    totalRowCount: number,
-    rowHeight,
+    rowHeight: number,
-    loading
+    loading: boolean
-  ) => {
+  ): string => {
     if (loading) {
       return `height: ${headerHeight + visibleRowCount * rowHeight}px;`
     }
@@ -146,7 +153,11 @@
     return `height: ${headerHeight + visibleRowCount * rowHeight}px;`
   }
 
-  const getGridStyle = (fields, schema, showEditColumn) => {
+  const getGridStyle = (
+    fields: string[],
+    schema: Record<string, any>,
+    showEditColumn: boolean
+  ): string => {
     let style = "grid-template-columns:"
     if (showEditColumn) {
       style += " auto"
@@ -163,7 +174,11 @@
     return style
   }
 
-  const sortRows = (rows, sortColumn, sortOrder) => {
+  const sortRows = (
+    rows: any[],
+    sortColumn: string | undefined,
+    sortOrder: string | undefined
+  ): any[] => {
     sortColumn = sortColumn ?? defaultSortColumn
     sortOrder = sortOrder ?? defaultSortOrder
     if (!sortColumn || !sortOrder || disableSorting) {
@@ -180,7 +195,7 @@
     })
   }
 
-  const sortBy = fieldSchema => {
+  const sortBy = (fieldSchema: Record<string, any>): void => {
     if (fieldSchema.sortable === false) {
       return
     }
@@ -193,7 +208,7 @@
     dispatch("sort", { column: sortColumn, order: sortOrder })
   }
 
-  const getDisplayName = schema => {
+  const getDisplayName = (schema: Record<string, any>): string => {
     let name = schema?.displayName
     if (schema && name === undefined) {
       name = schema.name
@@ -201,9 +216,13 @@
     return name || ""
   }
 
-  const getFields = (schema, showAutoColumns, autoSortColumns) => {
+  const getFields = (
-    let columns = []
+    schema: Record<string, any>,
-    let autoColumns = []
+    showAutoColumns: boolean,
+    autoSortColumns: boolean
+  ): string[] => {
+    let columns: any[] = []
+    let autoColumns: any[] = []
     Object.entries(schema || {}).forEach(([field, fieldSchema]) => {
       if (!field || !fieldSchema) {
         return
@@ -235,17 +254,17 @@
       .map(column => column.name)
   }
 
-  const editColumn = (e, field) => {
+  const editColumn = (e: Event, field: any): void => {
     e.stopPropagation()
     dispatch("editcolumn", field)
   }
 
-  const editRow = (e, row) => {
+  const editRow = (e: Event, row: any): void => {
     e.stopPropagation()
     dispatch("editrow", cloneDeep(row))
   }
 
-  const toggleSelectRow = row => {
+  const toggleSelectRow = (row: any): void => {
     if (!allowSelectRows) {
       return
     }
@@ -258,7 +277,7 @@
     }
   }
 
-  const toggleSelectAll = e => {
+  const toggleSelectAll = (e: CustomEvent): void => {
     const select = !!e.detail
     if (select) {
       // Add any rows which are not already in selected rows
@@ -278,8 +297,10 @@
     }
   }
 
-  const computeCellStyles = schema => {
+  const computeCellStyles = (
-    let styles = {}
+    schema: Record<string, any>
+  ): Record<string, string> => {
+    let styles: Record<string, string> = {}
     Object.keys(schema || {}).forEach(field => {
       styles[field] = ""
       if (schema[field].color) {

packages/bbui/src/Tabs/Tabs.svelte

@@ -1,28 +1,48 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/tabs/dist/index-vars.css"
   import { writable } from "svelte/store"
   import { onMount, setContext, createEventDispatcher } from "svelte"
 
-  export let selected
+  interface TabInfo {
-  export let vertical = false
+    width?: number
-  export let noPadding = false
+    height?: number
-  // added as a separate option as noPadding is used for vertical padding
+    left?: number
-  export let noHorizPadding = false
+    top?: number
-  export let quiet = false
+  }
-  export let emphasized = false
-  export let onTop = false
-  export let size = "M"
-  export let beforeSwitch = null
 
-  let thisSelected = undefined
+  interface TabState {
+    title: string
+    id: string
+    emphasized: boolean
+    info?: TabInfo
+  }
+
+  export let selected: string
+  export let vertical: boolean = false
+  export let noPadding: boolean = false
+  // added as a separate option as noPadding is used for vertical padding
+  export let noHorizPadding: boolean = false
+  export let quiet: boolean = false
+  export let emphasized: boolean = false
+  export let onTop: boolean = false
+  export let size: "S" | "M" | "L" = "M"
+  export let beforeSwitch: ((_title: string) => boolean) | null = null
+
+  let thisSelected: string | undefined = undefined
+  let container: HTMLElement | undefined
 
   let _id = id()
-  const tab = writable({ title: selected, id: _id, emphasized })
+  const tab = writable<TabState>({ title: selected, id: _id, emphasized })
   setContext("tab", tab)
 
-  let container
+  let top: string | undefined
+  let left: string | undefined
+  let width: string | undefined
+  let height: string | undefined
 
-  const dispatch = createEventDispatcher()
+  const dispatch = createEventDispatcher<{
+    select: string
+  }>()
 
   $: {
     if (thisSelected !== selected) {
@@ -44,29 +64,34 @@
     }
     if ($tab.title !== thisSelected) {
       tab.update(state => {
-        state.title = thisSelected
+        state.title = thisSelected as string
         return state
       })
     }
   }
 
-  let top, left, width, height
+  $: $tab && calculateIndicatorLength()
-  $: calculateIndicatorLength($tab)
+  $: $tab && calculateIndicatorOffset()
-  $: calculateIndicatorOffset($tab)
 
   function calculateIndicatorLength() {
     if (!vertical) {
-      width = $tab.info?.width + "px"
+      width = ($tab.info?.width ?? 0) + "px"
     } else {
-      height = $tab.info?.height + 4 + "px"
+      height = ($tab.info?.height ?? 0) + 4 + "px"
     }
   }
 
   function calculateIndicatorOffset() {
     if (!vertical) {
-      left = $tab.info?.left - container?.getBoundingClientRect().left + "px"
+      left =
+        ($tab.info?.left ?? 0) -
+        (container?.getBoundingClientRect().left ?? 0) +
+        "px"
     } else {
-      top = $tab.info?.top - container?.getBoundingClientRect().top + "px"
+      top =
+        ($tab.info?.top ?? 0) -
+        (container?.getBoundingClientRect().top ?? 0) +
+        "px"
     }
   }
 
@@ -75,7 +100,7 @@
     calculateIndicatorOffset()
   })
 
-  function id() {
+  function id(): string {
     return "_" + Math.random().toString(36).slice(2, 9)
   }
 </script>

packages/bbui/src/Tags/Tag.svelte

@@ -1,13 +1,13 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/tags/dist/index-vars.css"
   import Avatar from "../Avatar/Avatar.svelte"
   import ClearButton from "../ClearButton/ClearButton.svelte"
 
-  export let icon = ""
+  export let icon: string = ""
-  export let avatar = ""
+  export let avatar: string = ""
-  export let invalid = false
+  export let invalid: boolean = false
-  export let disabled = false
+  export let disabled: boolean = false
-  export let closable = false
+  export let closable: boolean = false
 </script>
 
 <div

packages/bbui/src/Tags/Tags.svelte

@@ -1,4 +1,4 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/tags/dist/index-vars.css"
 </script>
 

packages/bbui/src/Tooltip/TooltipWrapper.svelte

@@ -1,10 +1,10 @@
-<script>
+<script lang="ts">
   import Icon from "../Icon/Icon.svelte"
   import AbsTooltip from "./AbsTooltip.svelte"
 
-  export let tooltip = ""
+  export let tooltip: string = ""
-  export let size = "M"
+  export let size: "S" | "M" = "M"
-  export let disabled = true
+  export let disabled: boolean = true
 </script>
 
 <div class:container={!!tooltip}>

packages/bbui/src/TreeView/Item.svelte

@@ -1,9 +1,9 @@
-<script>
+<script lang="ts">
-  export let selected = false
+  export let selected: boolean = false
-  export let open = false
+  export let open: boolean = false
-  export let href = false
+  export let href: string | null = null
-  export let title
+  export let title: string
-  export let icon
+  export let icon: string | undefined
 </script>
 
 <li

packages/bbui/src/TreeView/Tree.svelte

@@ -1,9 +1,9 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/treeview/dist/index-vars.css"
 
-  export let quiet = false
+  export let quiet: boolean = false
-  export let standalone = true
+  export let standalone: boolean = true
-  export let width = "250px"
+  export let width: string = "250px"
 </script>
 
 <ul

packages/bbui/src/Typography/Code.svelte

@@ -1,8 +1,8 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/typography/dist/index-vars.css"
 
   // Sizes
-  export let size = "M"
+  export let size: "S" | "M" | "L" = "M"
 </script>
 
 <code class="spectrum-Code spectrum-Code--size{size}">

packages/bbui/src/Typography/Detail.svelte

@@ -1,9 +1,9 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/typography/dist/index-vars.css"
 
-  export let size = "M"
+  export let size: "S" | "M" | "L" = "M"
-  export let serif = false
+  export let serif: boolean = false
-  export let weight = 600
+  export let weight: number | null = null
 </script>
 
 <p

packages/bbui/src/index.ts

@@ -65,8 +65,6 @@ export { default as Modal } from "./Modal/Modal.svelte"
 export { default as ModalContent, keepOpen } from "./Modal/ModalContent.svelte"
 export { default as NotificationDisplay } from "./Notification/NotificationDisplay.svelte"
 export { default as Notification } from "./Notification/Notification.svelte"
-export { default as SideNavigation } from "./SideNavigation/Navigation.svelte"
-export { default as SideNavigationItem } from "./SideNavigation/Item.svelte"
 export { default as Context } from "./context"
 export { default as Table } from "./Table/Table.svelte"
 export { default as Tabs } from "./Tabs/Tabs.svelte"

packages/builder/src/components/backend/DataTable/formula.js

@@ -2,6 +2,7 @@ import { FieldType } from "@budibase/types"
 import { FIELDS } from "@/constants/backend"
 import { tables } from "@/stores/builder"
 import { get as svelteGet } from "svelte/store"
+import { makeReadableKeyPropSafe } from "@/dataBinding"
 
 // currently supported level of relationship depth (server side)
 const MAX_DEPTH = 1
@@ -26,7 +27,7 @@ export function getBindings({
   if (!table) {
     return bindings
   }
-  for (let [column, schema] of Object.entries(table.schema)) {
+  for (const [column, schema] of Object.entries(table.schema)) {
     const isRelationship = schema.type === FieldType.LINK
     // skip relationships after a certain depth and types which
     // can't bind to
@@ -62,6 +63,10 @@ export function getBindings({
 
     const label = path == null ? column : `${path}.0.${column}`
     const binding = path == null ? `[${column}]` : `[${path}].0.[${column}]`
+    const readableBinding = (path == null ? [column] : [path, "0", column])
+      .map(makeReadableKeyPropSafe)
+      .join(".")
+
     // only supply a description for relationship paths
     const description =
       path == null
@@ -75,7 +80,7 @@ export function getBindings({
       description,
       // don't include path, it messes things up, relationship path
       // will be replaced by the main array binding
-      readableBinding: label,
+      readableBinding,
       runtimeBinding: binding,
       display: {
         name: label,

packages/builder/src/components/common/CodeEditor/CodeEditor.svelte

@@ -49,6 +49,7 @@
   import type { EditorMode } from "@budibase/types"
   import type { BindingCompletion, CodeValidator } from "@/types"
   import { validateHbsTemplate } from "./validator/hbs"
+  import { validateJsTemplate } from "./validator/js"
 
   export let label: string | undefined = undefined
   export let completions: BindingCompletion[] = []
@@ -356,6 +357,9 @@
     if (mode === EditorModes.Handlebars) {
       const diagnostics = validateHbsTemplate(value, validations)
       editor.dispatch(setDiagnostics(editor.state, diagnostics))
+    } else if (mode === EditorModes.JS) {
+      const diagnostics = validateJsTemplate(value, validations)
+      editor.dispatch(setDiagnostics(editor.state, diagnostics))
     }
   }
 

packages/builder/src/components/common/CodeEditor/validator/hbs.ts

@@ -45,7 +45,10 @@ export function validateHbsTemplate(
     ) {
       const ignoreMissing = options?.ignoreMissing || false
       nodes.forEach(node => {
-        if (isMustacheStatement(node) && isPathExpression(node.path)) {
+        if (
+          (isMustacheStatement(node) || isBlockStatement(node)) &&
+          isPathExpression(node.path)
+        ) {
           const helperName = node.path.original
 
           const from =
@@ -75,21 +78,64 @@ export function validateHbsTemplate(
               message: `Helper "${helperName}" requires a body:\n{{#${helperName} ...}} [body] {{/${helperName}}}`,
             })
             return
+          } else if (!requiresBlock && isBlockStatement(node)) {
+            diagnostics.push({
+              from,
+              to,
+              severity: "error",
+              message: `Helper "${helperName}" should not contain a body.`,
+            })
+            return
           }
 
-          const providedParams = node.params
+          let providedParamsCount = node.params.length
+          if (isBlockStatement(node)) {
+            // Block body counts as a parameter
+            providedParamsCount++
+          }
 
-          if (providedParams.length !== expectedArguments.length) {
+          const optionalArgMatcher = new RegExp(/^\[(.+)\]$/)
+          const optionalArgs = expectedArguments.filter(a =>
+            optionalArgMatcher.test(a)
+          )
+
+          if (
+            !optionalArgs.length &&
+            providedParamsCount !== expectedArguments.length
+          ) {
             diagnostics.push({
               from,
               to,
               severity: "error",
               message: `Helper "${helperName}" expects ${
                 expectedArguments.length
-              } parameters (${expectedArguments.join(", ")}), but got ${
+              } parameters (${expectedArguments.join(
-                providedParams.length
+                ", "
-              }.`,
+              )}), but got ${providedParamsCount}.`,
             })
+          } else if (optionalArgs.length) {
+            const maxArgs = expectedArguments.length
+            const minArgs = maxArgs - optionalArgs.length
+            if (
+              minArgs > providedParamsCount ||
+              maxArgs < providedParamsCount
+            ) {
+              const parameters = expectedArguments
+                .map(a => {
+                  const test = optionalArgMatcher.exec(a)
+                  if (!test?.[1]) {
+                    return a
+                  }
+                  return `${test[1]} (optional)`
+                })
+                .join(", ")
+              diagnostics.push({
+                from,
+                to,
+                severity: "error",
+                message: `Helper "${helperName}" expects between ${minArgs} to ${expectedArguments.length} parameters (${parameters}), but got ${providedParamsCount}.`,
+              })
+            }
           }
         }
 

packages/builder/src/components/common/CodeEditor/validator/js.ts

@@ -0,0 +1,101 @@
+import { Parser } from "acorn"
+import * as walk from "acorn-walk"
+
+import type { Diagnostic } from "@codemirror/lint"
+import { CodeValidator } from "@/types"
+
+export function validateJsTemplate(
+  code: string,
+  validations: CodeValidator
+): Diagnostic[] {
+  const diagnostics: Diagnostic[] = []
+
+  try {
+    const ast = Parser.parse(code, {
+      ecmaVersion: "latest",
+      locations: true,
+      allowReturnOutsideFunction: true,
+    })
+
+    const lineOffsets: number[] = []
+    let offset = 0
+    for (const line of code.split("\n")) {
+      lineOffsets.push(offset)
+      offset += line.length + 1 // +1 for newline character
+    }
+
+    let hasReturnStatement = false
+    walk.ancestor(ast, {
+      ReturnStatement(node, _state, ancestors) {
+        if (
+          // it returns a value
+          node.argument &&
+          // and it is top level
+          ancestors.length === 2 &&
+          ancestors[0].type === "Program" &&
+          ancestors[1].type === "ReturnStatement"
+        ) {
+          hasReturnStatement = true
+        }
+      },
+      CallExpression(node) {
+        const callee: any = node.callee
+        if (
+          node.type === "CallExpression" &&
+          callee.object?.name === "helpers" &&
+          node.loc
+        ) {
+          const functionName = callee.property.name
+          const from =
+            lineOffsets[node.loc.start.line - 1] + node.loc.start.column
+          const to = lineOffsets[node.loc.end.line - 1] + node.loc.end.column
+
+          if (!(functionName in validations)) {
+            diagnostics.push({
+              from,
+              to,
+              severity: "warning",
+              message: `"${functionName}" function does not exist.`,
+            })
+            return
+          }
+
+          const { arguments: expectedArguments } = validations[functionName]
+          if (
+            expectedArguments &&
+            node.arguments.length !== expectedArguments.length
+          ) {
+            diagnostics.push({
+              from,
+              to,
+              severity: "error",
+              message: `Function "${functionName}" expects ${
+                expectedArguments.length
+              } parameters (${expectedArguments.join(", ")}), but got ${
+                node.arguments.length
+              }.`,
+            })
+          }
+        }
+      },
+    })
+
+    if (!hasReturnStatement) {
+      diagnostics.push({
+        from: 0,
+        to: code.length,
+        severity: "error",
+        message: "Your code must return a value.",
+      })
+    }
+  } catch (e: any) {
+    diagnostics.push({
+      from: 0,
+      to: code.length,
+      severity: "error",
+      message: `Syntax error: ${e.message}`,
+    })
+  }
+
+  return diagnostics
+}

packages/builder/src/components/common/CodeEditor/validator/tests/hbs.spec.ts

@@ -2,7 +2,7 @@ import { validateHbsTemplate } from "../hbs"
 import { CodeValidator } from "@/types"
 
 describe("hbs validator", () => {
-  it("validate empty strings", () => {
+  it("validates empty strings", () => {
     const text = ""
     const validators = {}
 
@@ -10,7 +10,7 @@ describe("hbs validator", () => {
     expect(result).toHaveLength(0)
   })
 
-  it("validate strings without hbs expressions", () => {
+  it("validates strings without hbs expressions", () => {
     const text = "first line\nand another one"
     const validators = {}
 
@@ -23,7 +23,7 @@ describe("hbs validator", () => {
       fieldName: {},
     }
 
-    it("validate valid expressions", () => {
+    it("validates valid expressions", () => {
       const text = "{{ fieldName }}"
 
       const result = validateHbsTemplate(text, validators)
@@ -98,13 +98,14 @@ describe("hbs validator", () => {
   })
 
   describe("expressions with parameters", () => {
+    describe("basic expression", () => {
       const validators: CodeValidator = {
         helperFunction: {
           arguments: ["a", "b", "c"],
         },
       }
 
-    it("validate valid params", () => {
+      it("validates valid params", () => {
         const text = "{{ helperFunction 1 99 'a' }}"
 
         const result = validateHbsTemplate(text, validators)
@@ -139,4 +140,151 @@ describe("hbs validator", () => {
         ])
       })
     })
+
+    describe("body expressions", () => {
+      const validators: CodeValidator = {
+        bodyFunction: {
+          arguments: ["a", "b", "c"],
+          requiresBlock: true,
+        },
+        nonBodyFunction: {
+          arguments: ["a", "b"],
+        },
+      }
+
+      it("validates valid params", () => {
+        const text = "{{#bodyFunction 1 99  }}body{{/bodyFunction}}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toHaveLength(0)
+      })
+
+      it("validates empty bodies", () => {
+        const text = "{{#bodyFunction 1 99  }}{{/bodyFunction}}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toHaveLength(0)
+      })
+
+      it("validates too little parameters", () => {
+        const text = "{{#bodyFunction 1 }}{{/bodyFunction}}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toEqual([
+          {
+            from: 0,
+            message: `Helper "bodyFunction" expects 3 parameters (a, b, c), but got 2.`,
+            severity: "error",
+            to: 37,
+          },
+        ])
+      })
+
+      it("validates too many parameters", () => {
+        const text = "{{#bodyFunction 1 99 'a' 0 }}{{/bodyFunction}}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toEqual([
+          {
+            from: 0,
+            message: `Helper "bodyFunction" expects 3 parameters (a, b, c), but got 5.`,
+            severity: "error",
+            to: 46,
+          },
+        ])
+      })
+
+      it("validates non-supported body usages", () => {
+        const text = "{{#nonBodyFunction 1 99}}{{/nonBodyFunction}}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toEqual([
+          {
+            from: 0,
+            message: `Helper "nonBodyFunction" should not contain a body.`,
+            severity: "error",
+            to: 45,
+          },
+        ])
+      })
+    })
+
+    describe("optional parameters", () => {
+      it("supports empty parameters", () => {
+        const validators: CodeValidator = {
+          helperFunction: {
+            arguments: ["a", "b", "[c]"],
+          },
+        }
+        const text = "{{ helperFunction 1 99 }}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toHaveLength(0)
+      })
+
+      it("supports valid parameters", () => {
+        const validators: CodeValidator = {
+          helperFunction: {
+            arguments: ["a", "b", "[c]"],
+          },
+        }
+        const text = "{{ helperFunction 1 99 'a' }}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toHaveLength(0)
+      })
+
+      it("returns a valid message on missing parameters", () => {
+        const validators: CodeValidator = {
+          helperFunction: {
+            arguments: ["a", "b", "[c]"],
+          },
+        }
+        const text = "{{ helperFunction 1 }}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toEqual([
+          {
+            from: 0,
+            message: `Helper "helperFunction" expects between 2 to 3 parameters (a, b, c (optional)), but got 1.`,
+            severity: "error",
+            to: 22,
+          },
+        ])
+      })
+
+      it("returns a valid message on too many parameters", () => {
+        const validators: CodeValidator = {
+          helperFunction: {
+            arguments: ["a", "b", "[c]"],
+          },
+        }
+        const text = "{{ helperFunction 1 2 3 4 }}"
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toEqual([
+          {
+            from: 0,
+            message: `Helper "helperFunction" expects between 2 to 3 parameters (a, b, c (optional)), but got 4.`,
+            severity: "error",
+            to: 28,
+          },
+        ])
+      })
+    })
+  })
+
+  it("validates wrong hbs code", () => {
+    const text = "{{#fieldName}}{{/wrong}}"
+
+    const result = validateHbsTemplate(text, {})
+    expect(result).toEqual([
+      {
+        from: 0,
+        message: `The handlebars code is not valid:\nfieldName doesn't match wrong - 1:3`,
+        severity: "error",
+        to: text.length,
+      },
+    ])
+  })
 })

packages/builder/src/components/common/CodeEditor/validator/tests/js.spec.ts

@@ -0,0 +1,156 @@
+import { validateJsTemplate } from "../js"
+import { CodeValidator } from "@/types"
+
+describe("js validator", () => {
+  it("validates valid code", () => {
+    const text = "return 7"
+    const validators = {}
+
+    const result = validateJsTemplate(text, validators)
+    expect(result).toEqual([])
+  })
+
+  it("does not validate runtime errors", () => {
+    const text = "return a"
+    const validators = {}
+
+    const result = validateJsTemplate(text, validators)
+    expect(result).toEqual([])
+  })
+
+  it("validates multiline code", () => {
+    const text = "const foo='bar'\nreturn 123"
+    const validators = {}
+
+    const result = validateJsTemplate(text, validators)
+    expect(result).toEqual([])
+  })
+
+  it("allows return not being on the last line", () => {
+    const text = "const foo='bar'\nreturn 123\nconsole.log(foo)"
+    const validators = {}
+
+    const result = validateJsTemplate(text, validators)
+    expect(result).toEqual([])
+  })
+
+  it("throws on missing return", () => {
+    const text = "const foo='bar'\nbar='foo'"
+    const validators = {}
+
+    const result = validateJsTemplate(text, validators)
+    expect(result).toEqual([
+      {
+        from: 0,
+        message: "Your code must return a value.",
+        severity: "error",
+        to: 25,
+      },
+    ])
+  })
+
+  it("checks that returns are at top level", () => {
+    const text = `
+    function call(){
+      return 1
+    }`
+    const validators = {}
+
+    const result = validateJsTemplate(text, validators)
+    expect(result).toEqual([
+      {
+        from: 0,
+        message: "Your code must return a value.",
+        severity: "error",
+        to: text.length,
+      },
+    ])
+  })
+
+  describe("helpers", () => {
+    const validators: CodeValidator = {
+      helperFunction: {
+        arguments: ["a", "b", "c"],
+      },
+    }
+
+    it("validates helpers with valid params", () => {
+      const text = "return helpers.helperFunction(1, 99, 'a')"
+
+      const result = validateJsTemplate(text, validators)
+      expect(result).toEqual([])
+    })
+
+    it("throws on too few params", () => {
+      const text = "return helpers.helperFunction(100)"
+
+      const result = validateJsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 7,
+          message: `Function "helperFunction" expects 3 parameters (a, b, c), but got 1.`,
+          severity: "error",
+          to: 34,
+        },
+      ])
+    })
+
+    it("throws on too many params", () => {
+      const text = "return helpers.helperFunction( 1, 99, 'a', 100)"
+
+      const result = validateJsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 7,
+          message: `Function "helperFunction" expects 3 parameters (a, b, c), but got 4.`,
+          severity: "error",
+          to: 47,
+        },
+      ])
+    })
+
+    it("validates helpers on inner functions", () => {
+      const text = `function call(){
+                      return helpers.helperFunction(1, 99)
+                    }
+                    return call()`
+
+      const result = validateJsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 46,
+          message: `Function "helperFunction" expects 3 parameters (a, b, c), but got 2.`,
+          severity: "error",
+          to: 75,
+        },
+      ])
+    })
+
+    it("validates multiple helpers", () => {
+      const text =
+        "return helpers.helperFunction(1, 99, 'a') + helpers.helperFunction(1) + helpers.another(1) + helpers.another()"
+      const validators: CodeValidator = {
+        helperFunction: {
+          arguments: ["a", "b", "c"],
+        },
+        another: { arguments: [] },
+      }
+
+      const result = validateJsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 44,
+          message: `Function "helperFunction" expects 3 parameters (a, b, c), but got 1.`,
+          severity: "error",
+          to: 69,
+        },
+        {
+          from: 72,
+          message: `Function "another" expects 0 parameters (), but got 1.`,
+          severity: "error",
+          to: 90,
+        },
+      ])
+    })
+  })
+})

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -27,12 +27,11 @@
   } from "../CodeEditor"
   import BindingSidePanel from "./BindingSidePanel.svelte"
   import EvaluationSidePanel from "./EvaluationSidePanel.svelte"
-  import SnippetSidePanel from "./SnippetSidePanel.svelte"
   import { BindingHelpers } from "./utils"
   import { capitalise } from "@/helpers"
   import { Utils, JsonFormatter } from "@budibase/frontend-core"
   import { licensing } from "@/stores/portal"
-  import { BindingMode, SidePanel } from "@budibase/types"
+  import { BindingMode } from "@budibase/types"
   import type {
     EnrichedBinding,
     Snippet,
@@ -44,6 +43,8 @@
   import type { Log } from "@budibase/string-templates"
   import type { CodeValidator } from "@/types"
 
+  type SidePanel = "Bindings" | "Evaluation"
+
   const dispatch = createEventDispatcher()
 
   export let bindings: EnrichedBinding[] = []
@@ -55,7 +56,7 @@
   export let context = null
   export let snippets: Snippet[] | null = null
   export let autofocusEditor = false
-  export let placeholder = null
+  export let placeholder: string | null = null
   export let showTabBar = true
 
   let mode: BindingMode
@@ -71,14 +72,13 @@
   let expressionError: string | undefined
   let evaluating = false
 
-  $: useSnippets = allowSnippets && !$licensing.isFreePlan
+  const SidePanelIcons: Record<SidePanel, string> = {
+    Bindings: "FlashOn",
+    Evaluation: "Play",
+  }
+
   $: editorModeOptions = getModeOptions(allowHBS, allowJS)
-  $: sidePanelOptions = getSidePanelOptions(
+  $: sidePanelOptions = getSidePanelOptions(bindings, context)
-    bindings,
-    context,
-    allowSnippets,
-    mode
-  )
   $: enrichedBindings = enrichBindings(bindings, context, snippets)
   $: usingJS = mode === BindingMode.JavaScript
   $: editorMode =
@@ -90,10 +90,12 @@
   $: requestEval(runtimeExpression, context, snippets)
   $: bindingHelpers = new BindingHelpers(getCaretPosition, insertAtPos)
 
-  $: bindingOptions = bindingsToCompletions(bindings, editorMode)
+  $: bindingOptions = bindingsToCompletions(enrichedBindings, editorMode)
   $: helperOptions = allowHelpers ? getHelperCompletions(editorMode) : []
   $: snippetsOptions =
-    usingJS && useSnippets && snippets?.length ? snippets : []
+    usingJS && allowSnippets && !$licensing.isFreePlan && snippets?.length
+      ? snippets
+      : []
 
   $: completions = !usingJS
     ? [hbAutocomplete([...bindingOptions, ...helperOptions])]
@@ -137,21 +139,13 @@
     return options
   }
 
-  const getSidePanelOptions = (
+  const getSidePanelOptions = (bindings: EnrichedBinding[], context: any) => {
-    bindings: EnrichedBinding[],
+    let options: SidePanel[] = []
-    context: any,
-    useSnippets: boolean,
-    mode: BindingMode | null
-  ) => {
-    let options = []
     if (bindings?.length) {
-      options.push(SidePanel.Bindings)
+      options.push("Bindings")
     }
     if (context && Object.keys(context).length > 0) {
-      options.push(SidePanel.Evaluation)
+      options.push("Evaluation")
-    }
-    if (useSnippets && mode === BindingMode.JavaScript) {
-      options.push(SidePanel.Snippets)
     }
     return options
   }
@@ -342,14 +336,15 @@
             {/each}
           </div>
           <div class="side-tabs">
-            {#each sidePanelOptions as panel}
+            {#each sidePanelOptions as panelOption}
               <ActionButton
                 size="M"
                 quiet
-                selected={sidePanel === panel}
+                selected={sidePanel === panelOption}
-                on:click={() => changeSidePanel(panel)}
+                on:click={() => changeSidePanel(panelOption)}
+                tooltip={panelOption}
               >
-                <Icon name={panel} size="S" />
+                <Icon name={SidePanelIcons[panelOption]} size="S" />
               </ActionButton>
             {/each}
           </div>
@@ -377,6 +372,7 @@
               value={jsValue ? decodeJSBinding(jsValue) : jsValue}
               on:change={onChangeJSValue}
               {completions}
+              {validations}
               mode={EditorModes.JS}
               bind:getCaretPosition
               bind:insertAtPos
@@ -414,16 +410,19 @@
       </div>
     </div>
     <div class="side" class:visible={!!sidePanel}>
-      {#if sidePanel === SidePanel.Bindings}
+      {#if sidePanel === "Bindings"}
         <BindingSidePanel
           bindings={enrichedBindings}
           {allowHelpers}
+          {allowSnippets}
           {context}
           addHelper={onSelectHelper}
           addBinding={onSelectBinding}
-          mode={editorMode}
+          {addSnippet}
+          {mode}
+          {snippets}
         />
-      {:else if sidePanel === SidePanel.Evaluation}
+      {:else if sidePanel === "Evaluation"}
         <EvaluationSidePanel
           {expressionResult}
           {expressionError}
@@ -431,8 +430,6 @@
           {evaluating}
           expression={editorValue ? editorValue : ""}
         />
-      {:else if sidePanel === SidePanel.Snippets}
-        <SnippetSidePanel {addSnippet} {snippets} />
       {/if}
     </div>
   </div>

packages/builder/src/components/common/bindings/BindingSidePanel.svelte

@@ -1,34 +1,69 @@
-<script>
+<script lang="ts">
   import groupBy from "lodash/fp/groupBy"
   import { convertToJS } from "@budibase/string-templates"
-  import { Input, Layout, Icon, Popover } from "@budibase/bbui"
+  import { licensing } from "@/stores/portal"
+  import {
+    Input,
+    Layout,
+    Icon,
+    Popover,
+    Tags,
+    Tag,
+    Body,
+    Button,
+  } from "@budibase/bbui"
   import { handlebarsCompletions } from "@/constants/completions"
+  import type { EnrichedBinding, Helper, Snippet } from "@budibase/types"
+  import { BindingMode } from "@budibase/types"
+  import { EditorModes } from "../CodeEditor"
+  import CodeEditor from "../CodeEditor/CodeEditor.svelte"
 
-  export let addHelper
+  import SnippetDrawer from "./SnippetDrawer.svelte"
-  export let addBinding
+  import UpgradeButton from "@/pages/builder/portal/_components/UpgradeButton.svelte"
-  export let bindings
+
-  export let mode
+  export let addHelper: (_helper: Helper, _js?: boolean) => void
-  export let allowHelpers
+  export let addBinding: (_binding: EnrichedBinding) => void
+  export let addSnippet: (_snippet: Snippet) => void
+  export let bindings: EnrichedBinding[]
+  export let snippets: Snippet[] | null
+  export let mode: BindingMode
+  export let allowHelpers: boolean
+  export let allowSnippets: boolean
   export let context = null
 
   let search = ""
   let searching = false
-  let popover
+  let popover: Popover
-  let popoverAnchor
+  let popoverAnchor: HTMLElement | undefined
-  let hoverTarget
+  let hoverTarget: {
+    type: "binding" | "helper" | "snippet"
+    code: string
+    description?: string
+  } | null
   let helpers = handlebarsCompletions()
-  let selectedCategory
+  let selectedCategory: string | null
-  let hideTimeout
+  let hideTimeout: ReturnType<typeof setTimeout> | null
+  let snippetDrawer: SnippetDrawer
+  let editableSnippet: Snippet | null
 
-  $: bindingIcons = bindings?.reduce((acc, ele) => {
+  $: enableSnippets = !$licensing.isFreePlan
+  $: bindingIcons = bindings?.reduce<Record<string, string>>((acc, ele) => {
     if (ele.icon) {
       acc[ele.category] = acc[ele.category] || ele.icon
     }
     return acc
   }, {})
-  $: categoryIcons = { ...bindingIcons, Helpers: "MagicWand" }
+  $: categoryIcons = {
+    ...bindingIcons,
+    Helpers: "MagicWand",
+    Snippets: "Code",
+  } as Record<string, string>
   $: categories = Object.entries(groupBy("category", bindings))
-  $: categoryNames = getCategoryNames(categories)
+
+  $: categoryNames = getCategoryNames(
+    categories,
+    allowSnippets && mode === BindingMode.JavaScript
+  )
   $: searchRgx = new RegExp(search, "ig")
   $: filteredCategories = categories
     .map(([name, categoryBindings]) => ({
@@ -48,11 +83,22 @@
       (!search ||
         helper.label.match(searchRgx) ||
         helper.description.match(searchRgx)) &&
-      (mode.name !== "javascript" || helper.allowsJs)
+      (mode !== BindingMode.JavaScript || helper.allowsJs)
     )
   })
 
-  const getHelperExample = (helper, js) => {
+  $: filteredSnippets = getFilteredSnippets(
+    enableSnippets,
+    snippets || [],
+    search
+  )
+
+  function onModeChange(_mode: BindingMode) {
+    selectedCategory = null
+  }
+  $: onModeChange(mode)
+
+  const getHelperExample = (helper: Helper, js: boolean) => {
     let example = helper.example || ""
     if (js) {
       example = convertToJS(example).split("\n")[0].split("= ")[1]
@@ -63,37 +109,46 @@
     return example || ""
   }
 
-  const getCategoryNames = categories => {
+  const getCategoryNames = (
-    let names = [...categories.map(cat => cat[0])]
+    categories: [string, EnrichedBinding[]][],
+    showSnippets: boolean
+  ) => {
+    const names = [...categories.map(cat => cat[0])]
     if (allowHelpers) {
       names.push("Helpers")
     }
+    if (showSnippets) {
+      names.push("Snippets")
+    }
     return names
   }
 
-  const showBindingPopover = (binding, target) => {
+  const showBindingPopover = (
+    binding: EnrichedBinding,
+    target: HTMLElement
+  ) => {
     if (!context || !binding.value || binding.value === "") {
       return
     }
     stopHidingPopover()
     popoverAnchor = target
     hoverTarget = {
-      helper: false,
+      type: "binding",
       code: binding.valueHTML,
     }
     popover.show()
   }
 
-  const showHelperPopover = (helper, target) => {
+  const showHelperPopover = (helper: Helper, target: HTMLElement) => {
     stopHidingPopover()
     if (!helper.displayText && helper.description) {
       return
     }
     popoverAnchor = target
     hoverTarget = {
-      helper: true,
+      type: "helper",
       description: helper.description,
-      code: getHelperExample(helper, mode.name === "javascript"),
+      code: getHelperExample(helper, mode === BindingMode.JavaScript),
     }
     popover.show()
   }
@@ -101,7 +156,7 @@
   const hidePopover = () => {
     hideTimeout = setTimeout(() => {
       popover.hide()
-      popoverAnchor = null
+      popoverAnchor = undefined
       hoverTarget = null
       hideTimeout = null
     }, 100)
@@ -119,11 +174,53 @@
     search = ""
   }
 
-  const stopSearching = e => {
+  const stopSearching = (e: Event) => {
     e.stopPropagation()
     searching = false
     search = ""
   }
+
+  const getFilteredSnippets = (
+    enableSnippets: boolean,
+    snippets: Snippet[],
+    search: string
+  ) => {
+    if (!enableSnippets || !snippets.length) {
+      return []
+    }
+    if (!search?.length) {
+      return snippets
+    }
+    return snippets.filter(snippet =>
+      snippet.name.toLowerCase().includes(search.toLowerCase())
+    )
+  }
+
+  const showSnippet = (snippet: Snippet, target: HTMLElement) => {
+    stopHidingPopover()
+    if (!snippet.code) {
+      return
+    }
+    popoverAnchor = target
+    hoverTarget = {
+      type: "snippet",
+      code: snippet.code,
+    }
+
+    popover.show()
+  }
+
+  const createSnippet = () => {
+    editableSnippet = null
+    snippetDrawer.show()
+  }
+
+  const editSnippet = (e: Event, snippet: Snippet) => {
+    e.preventDefault()
+    e.stopPropagation()
+    editableSnippet = snippet
+    snippetDrawer.show()
+  }
 </script>
 
 <Popover
@@ -137,7 +234,11 @@
   on:mouseenter={stopHidingPopover}
   on:mouseleave={hidePopover}
 >
-  <div class="binding-popover" class:helper={hoverTarget.helper}>
+  {#if hoverTarget}
+    <div
+      class="binding-popover"
+      class:has-code={hoverTarget.type !== "binding"}
+    >
       {#if hoverTarget.description}
         <div>
           <!-- eslint-disable-next-line svelte/no-at-html-tags-->
@@ -145,10 +246,19 @@
         </div>
       {/if}
       {#if hoverTarget.code}
+        {#if mode === BindingMode.JavaScript}
+          <CodeEditor
+            value={hoverTarget.code?.trim()}
+            mode={EditorModes.JS}
+            readonly
+          />
+        {:else if mode === BindingMode.Text}
           <!-- eslint-disable-next-line svelte/no-at-html-tags-->
           <pre>{@html hoverTarget.code}</pre>
         {/if}
+      {/if}
     </div>
+  {/if}
 </Popover>
 
 <!-- svelte-ignore a11y-click-events-have-key-events -->
@@ -164,6 +274,25 @@
           on:click={() => (selectedCategory = null)}
         />
         {selectedCategory}
+        {#if selectedCategory === "Snippets"}
+          {#if enableSnippets}
+            <div class="add-snippet-button">
+              <Icon
+                size="S"
+                name="Add"
+                hoverable
+                newStyles
+                on:click={createSnippet}
+              />
+            </div>
+          {:else}
+            <div class="title">
+              <Tags>
+                <Tag icon="LockClosed">Premium</Tag>
+              </Tags>
+            </div>
+          {/if}
+        {/if}
       </div>
     {/if}
 
@@ -173,7 +302,7 @@
           <div class="search-input">
             <Input
               placeholder="Search for bindings"
-              autocomplete="off"
+              autocomplete={false}
               bind:value={search}
               autofocus
             />
@@ -230,7 +359,8 @@
               {#each category.bindings as binding}
                 <li
                   class="binding"
-                  on:mouseenter={e => showBindingPopover(binding, e.target)}
+                  on:mouseenter={e =>
+                    showBindingPopover(binding, e.currentTarget)}
                   on:mouseleave={hidePopover}
                   on:click={() => addBinding(binding)}
                 >
@@ -264,9 +394,10 @@
               {#each filteredHelpers as helper}
                 <li
                   class="binding"
-                  on:mouseenter={e => showHelperPopover(helper, e.target)}
+                  on:mouseenter={e =>
-                  on:mouseleave={hidePopover}
+                    showHelperPopover(helper, e.currentTarget)}
-                  on:click={() => addHelper(helper, mode.name === "javascript")}
+                  on:click={() =>
+                    addHelper(helper, mode === BindingMode.JavaScript)}
                 >
                   <span class="binding__label">{helper.displayText}</span>
                   <span class="binding__typeWrap">
@@ -278,10 +409,48 @@
           </div>
         {/if}
       {/if}
+
+      {#if selectedCategory === "Snippets" || search}
+        <div class="snippet-list">
+          {#if enableSnippets && filteredSnippets.length}
+            {#each filteredSnippets as snippet}
+              <li
+                class="snippet"
+                on:mouseenter={e => showSnippet(snippet, e.currentTarget)}
+                on:mouseleave={hidePopover}
+                on:click={() => addSnippet(snippet)}
+              >
+                {snippet.name}
+                <Icon
+                  name="Edit"
+                  hoverable
+                  newStyles
+                  size="S"
+                  on:click={e => editSnippet(e, snippet)}
+                />
+              </li>
+            {/each}
+          {:else if !search}
+            <div class="upgrade">
+              <Body size="S">
+                Snippets let you create reusable JS functions and values that
+                can all be managed in one place
+              </Body>
+              {#if enableSnippets}
+                <Button cta on:click={createSnippet}>Create snippet</Button>
+              {:else}
+                <UpgradeButton />
+              {/if}
+            </div>
+          {/if}
+        </div>
+      {/if}
     {/if}
   </Layout>
 </div>
 
+<SnippetDrawer bind:this={snippetDrawer} snippet={editableSnippet} />
+
 <style>
   .binding-side-panel {
     border-left: var(--border-light);
@@ -346,6 +515,7 @@
     display: flex;
     align-items: center;
     gap: var(--spacing-m);
+    justify-content: space-between;
   }
   li.binding .binding__typeWrap {
     flex: 1;
@@ -421,7 +591,7 @@
     text-overflow: ellipsis;
     overflow: hidden;
   }
-  .binding-popover.helper pre {
+  .binding-popover.has-code pre {
     color: var(--spectrum-global-color-blue-700);
   }
   .binding-popover pre :global(span) {
@@ -433,7 +603,50 @@
     padding: 0;
     margin: 0;
   }
-  .binding-popover.helper :global(code) {
+  .binding-popover.has-code :global(code) {
     font-size: 12px;
   }
+  .binding-popover.has-code :global(.cm-line),
+  .binding-popover.has-code :global(.cm-content) {
+    padding: 0;
+  }
+
+  /* Snippets */
+  .add-snippet-button {
+    margin-left: auto;
+  }
+  .snippet-list {
+    padding: 0 var(--spacing-l);
+    padding-bottom: var(--spacing-l);
+    display: flex;
+    flex-direction: column;
+  }
+  .snippet {
+    font-size: var(--font-size-s);
+    padding: var(--spacing-m);
+    border-radius: 4px;
+    background-color: var(--spectrum-global-color-gray-200);
+    transition: background-color 130ms ease-out, color 130ms ease-out,
+      border-color 130ms ease-out;
+    word-wrap: break-word;
+    display: flex;
+    justify-content: space-between;
+  }
+  .snippet:hover {
+    color: var(--spectrum-global-color-gray-900);
+    background-color: var(--spectrum-global-color-gray-50);
+    cursor: pointer;
+  }
+
+  /* Upgrade */
+  .upgrade {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    gap: var(--spacing-l);
+  }
+  .upgrade :global(p) {
+    text-align: center;
+    align-self: center;
+  }
 </style>

packages/builder/src/components/common/bindings/SnippetDrawer.svelte

@@ -1,4 +1,4 @@
-<script>
+<script lang="ts">
   import {
     Button,
     Drawer,
@@ -14,19 +14,27 @@
   import { getSequentialName } from "@/helpers/duplicate"
   import ConfirmDialog from "@/components/common/ConfirmDialog.svelte"
   import { ValidSnippetNameRegex } from "@budibase/shared-core"
+  import type { Snippet } from "@budibase/types"
 
-  export let snippet
+  export const show = () => {
-
+    if (!snippet) {
-  export const show = () => drawer.show()
+      key = Math.random().toString()
+      // Reset state when creating multiple snippets
+      code = ""
+      name = defaultName
+    }
+    drawer.show()
+  }
   export const hide = () => drawer.hide()
+  export let snippet: Snippet | null
 
   const firstCharNumberRegex = /^[0-9].*$/
 
-  let drawer
+  let drawer: Drawer
   let name = ""
   let code = ""
   let loading = false
-  let deleteConfirmationDialog
+  let deleteConfirmationDialog: ConfirmDialog
 
   $: defaultName = getSequentialName($snippets, "MySnippet", {
     getName: x => x.name,
@@ -40,11 +48,11 @@
   const saveSnippet = async () => {
     loading = true
     try {
-      const newSnippet = { name, code: rawJS }
+      const newSnippet: Snippet = { name, code: rawJS || "" }
       await snippets.saveSnippet(newSnippet)
       drawer.hide()
       notifications.success(`Snippet ${newSnippet.name} saved`)
-    } catch (error) {
+    } catch (error: any) {
       notifications.error(error.message || "Error saving snippet")
     }
     loading = false
@@ -53,7 +61,9 @@
   const deleteSnippet = async () => {
     loading = true
     try {
+      if (snippet) {
         await snippets.deleteSnippet(snippet.name)
+      }
       drawer.hide()
     } catch (error) {
       notifications.error("Error deleting snippet")
@@ -61,7 +71,7 @@
     loading = false
   }
 
-  const validateName = (name, snippets) => {
+  const validateName = (name: string, snippets: Snippet[]) => {
     if (!name?.length) {
       return "Name is required"
     }
@@ -108,7 +118,11 @@
         Delete
       </Button>
     {/if}
-    <Button cta on:click={saveSnippet} disabled={!code || loading || nameError}>
+    <Button
+      cta
+      on:click={saveSnippet}
+      disabled={!code || loading || !!nameError}
+    >
       Save
     </Button>
   </svelte:fragment>
@@ -124,9 +138,7 @@
         value={code}
         on:change={e => (code = e.detail)}
       >
-        <div slot="tabs">
         <Input placeholder="Name" />
-        </div>
       </BindingPanel>
     {/key}
   </svelte:fragment>

packages/builder/src/components/common/bindings/SnippetSidePanel.svelte

@@ -1,278 +0,0 @@
-<script>
-  import {
-    Input,
-    Layout,
-    Icon,
-    Popover,
-    Tags,
-    Tag,
-    Body,
-    Button,
-  } from "@budibase/bbui"
-  import CodeEditor from "@/components/common/CodeEditor/CodeEditor.svelte"
-  import { EditorModes } from "@/components/common/CodeEditor"
-  import SnippetDrawer from "./SnippetDrawer.svelte"
-  import { licensing } from "@/stores/portal"
-  import UpgradeButton from "@/pages/builder/portal/_components/UpgradeButton.svelte"
-
-  export let addSnippet
-  export let snippets
-
-  let search = ""
-  let searching = false
-  let popover
-  let popoverAnchor
-  let hoveredSnippet
-  let hideTimeout
-  let snippetDrawer
-  let editableSnippet
-
-  $: enableSnippets = !$licensing.isFreePlan
-  $: filteredSnippets = getFilteredSnippets(enableSnippets, snippets, search)
-
-  const getFilteredSnippets = (enableSnippets, snippets, search) => {
-    if (!enableSnippets || !snippets?.length) {
-      return []
-    }
-    if (!search?.length) {
-      return snippets
-    }
-    return snippets.filter(snippet =>
-      snippet.name.toLowerCase().includes(search.toLowerCase())
-    )
-  }
-
-  const showSnippet = (snippet, target) => {
-    stopHidingPopover()
-    popoverAnchor = target
-    hoveredSnippet = snippet
-    popover.show()
-  }
-
-  const hidePopover = () => {
-    hideTimeout = setTimeout(() => {
-      popover.hide()
-      popoverAnchor = null
-      hoveredSnippet = null
-      hideTimeout = null
-    }, 100)
-  }
-
-  const stopHidingPopover = () => {
-    if (hideTimeout) {
-      clearTimeout(hideTimeout)
-      hideTimeout = null
-    }
-  }
-
-  const startSearching = () => {
-    searching = true
-    search = ""
-  }
-
-  const stopSearching = () => {
-    searching = false
-    search = ""
-  }
-
-  const createSnippet = () => {
-    editableSnippet = null
-    snippetDrawer.show()
-  }
-
-  const editSnippet = (e, snippet) => {
-    e.preventDefault()
-    e.stopPropagation()
-    editableSnippet = snippet
-    snippetDrawer.show()
-  }
-</script>
-
-<!-- svelte-ignore a11y-no-static-element-interactions -->
-<!-- svelte-ignore a11y-click-events-have-key-events -->
-<div class="snippet-side-panel">
-  <Layout noPadding gap="S">
-    <div class="header">
-      {#if enableSnippets}
-        {#if searching}
-          <div class="search-input">
-            <Input
-              placeholder="Search for snippets"
-              autocomplete="off"
-              bind:value={search}
-              autofocus
-            />
-          </div>
-          <Icon
-            size="S"
-            name="Close"
-            hoverable
-            newStyles
-            on:click={stopSearching}
-          />
-        {:else}
-          <div class="title">Snippets</div>
-          <Icon
-            size="S"
-            name="Search"
-            hoverable
-            newStyles
-            on:click={startSearching}
-          />
-          <Icon
-            size="S"
-            name="Add"
-            hoverable
-            newStyles
-            on:click={createSnippet}
-          />
-        {/if}
-      {:else}
-        <div class="title">
-          Snippets
-          <Tags>
-            <Tag icon="LockClosed">Premium</Tag>
-          </Tags>
-        </div>
-      {/if}
-    </div>
-    <div class="snippet-list">
-      {#if enableSnippets && filteredSnippets?.length}
-        {#each filteredSnippets as snippet}
-          <div
-            class="snippet"
-            on:mouseenter={e => showSnippet(snippet, e.target)}
-            on:mouseleave={hidePopover}
-            on:click={() => addSnippet(snippet)}
-          >
-            {snippet.name}
-            <Icon
-              name="Edit"
-              hoverable
-              newStyles
-              size="S"
-              on:click={e => editSnippet(e, snippet)}
-            />
-          </div>
-        {/each}
-      {:else}
-        <div class="upgrade">
-          <Body size="S">
-            Snippets let you create reusable JS functions and values that can
-            all be managed in one place
-          </Body>
-          {#if enableSnippets}
-            <Button cta on:click={createSnippet}>Create snippet</Button>
-          {:else}
-            <UpgradeButton />
-          {/if}
-        </div>
-      {/if}
-    </div>
-  </Layout>
-</div>
-
-<Popover
-  align="left-outside"
-  bind:this={popover}
-  anchor={popoverAnchor}
-  minWidth={0}
-  maxWidth={480}
-  maxHeight={480}
-  dismissible={false}
-  on:mouseenter={stopHidingPopover}
-  on:mouseleave={hidePopover}
->
-  <div class="snippet-popover">
-    {#key hoveredSnippet}
-      <CodeEditor
-        value={hoveredSnippet.code?.trim()}
-        mode={EditorModes.JS}
-        readonly
-      />
-    {/key}
-  </div>
-</Popover>
-
-<SnippetDrawer bind:this={snippetDrawer} snippet={editableSnippet} />
-
-<style>
-  .snippet-side-panel {
-    border-left: var(--border-light);
-    height: 100%;
-    overflow: auto;
-  }
-
-  /* Header */
-  .header {
-    height: 53px;
-    padding: 0 var(--spacing-l);
-    display: flex;
-    align-items: center;
-    border-bottom: var(--border-light);
-    position: sticky;
-    top: 0;
-    gap: var(--spacing-m);
-    background: var(--background);
-    z-index: 1;
-  }
-  .header :global(input) {
-    border: none;
-    border-radius: 0;
-    background: none;
-    padding: 0;
-  }
-  .search-input,
-  .title {
-    flex: 1 1 auto;
-  }
-  .title {
-    display: flex;
-    flex-direction: row;
-    justify-content: flex-start;
-    align-items: center;
-    gap: var(--spacing-m);
-  }
-
-  /* Upgrade */
-  .upgrade {
-    display: flex;
-    flex-direction: column;
-    align-items: center;
-    gap: var(--spacing-l);
-  }
-  .upgrade :global(p) {
-    text-align: center;
-    align-self: center;
-  }
-
-  /* List */
-  .snippet-list {
-    padding: 0 var(--spacing-l);
-    padding-bottom: var(--spacing-l);
-    display: flex;
-    flex-direction: column;
-    gap: var(--spacing-s);
-  }
-  .snippet {
-    font-size: var(--font-size-s);
-    padding: var(--spacing-m);
-    border-radius: 4px;
-    background-color: var(--spectrum-global-color-gray-200);
-    transition: background-color 130ms ease-out, color 130ms ease-out,
-      border-color 130ms ease-out;
-    word-wrap: break-word;
-    display: flex;
-    justify-content: space-between;
-  }
-  .snippet:hover {
-    color: var(--spectrum-global-color-gray-900);
-    background-color: var(--spectrum-global-color-gray-50);
-    cursor: pointer;
-  }
-
-  /* Popover */
-  .snippet-popover {
-    width: 400px;
-  }
-</style>

packages/builder/src/components/common/bindings/index.js

@@ -8,5 +8,3 @@ export { default as DrawerBindableSlot } from "./DrawerBindableSlot.svelte"
 export { default as EvaluationSidePanel } from "./EvaluationSidePanel.svelte"
 export { default as ModalBindableInput } from "./ModalBindableInput.svelte"
 export { default as ServerBindingPanel } from "./ServerBindingPanel.svelte"
-export { default as SnippetDrawer } from "./SnippetDrawer.svelte"
-export { default as SnippetSidePanel } from "./SnippetSidePanel.svelte"

packages/builder/src/constants/completions.ts

@@ -1,10 +1,10 @@
 import { getManifest, helpersToRemoveForJs } from "@budibase/string-templates"
+import { Helper } from "@budibase/types"
 
-export function handlebarsCompletions() {
+export function handlebarsCompletions(): Helper[] {
   const manifest = getManifest()
-
+  return Object.values(manifest).flatMap(helpersObj =>
-  return Object.keys(manifest).flatMap(key =>
+    Object.entries(helpersObj).map<Helper>(([helperName, helperConfig]) => ({
-    Object.entries(manifest[key]).map(([helperName, helperConfig]) => ({
       text: helperName,
       path: helperName,
       example: helperConfig.example,
@@ -14,6 +14,7 @@ export function handlebarsCompletions() {
       allowsJs:
         !helperConfig.requiresBlock &&
         !helpersToRemoveForJs.includes(helperName),
+      args: helperConfig.args,
     }))
   )
 }

packages/builder/src/dataBinding.js

@@ -373,6 +373,18 @@ const getContextBindings = (asset, componentId) => {
     .flat()
 }
 
+export const makeReadableKeyPropSafe = key => {
+  if (!key.includes(" ")) {
+    return key
+  }
+
+  if (new RegExp(/^\[(.+)\]$/).test(key.test)) {
+    return key
+  }
+
+  return `[${key}]`
+}
+
 /**
  * Generates a set of bindings for a given component context
  */
@@ -457,11 +469,11 @@ const generateComponentContextBindings = (asset, componentContext) => {
       const runtimeBinding = `${safeComponentId}.${safeKey}`
 
       // Optionally use a prefix with readable bindings
-      let readableBinding = component._instanceName
+      let readableBinding = makeReadableKeyPropSafe(component._instanceName)
       if (readablePrefix) {
         readableBinding += `.${readablePrefix}`
       }
-      readableBinding += `.${fieldSchema.name || key}`
+      readableBinding += `.${makeReadableKeyPropSafe(fieldSchema.name || key)}`
 
       // Determine which category this binding belongs in
       const bindingCategory = getComponentBindingCategory(
@@ -473,7 +485,7 @@ const generateComponentContextBindings = (asset, componentContext) => {
       bindings.push({
         type: "context",
         runtimeBinding,
-        readableBinding: `${readableBinding}`,
+        readableBinding,
         // Field schema and provider are required to construct relationship
         // datasource options, based on bindable properties
         fieldSchema,
@@ -1354,13 +1366,14 @@ const bindingReplacement = (
   }
   // work from longest to shortest
   const convertFromProps = bindableProperties
+    // TODO check whitespaces
     .map(el => el[convertFrom])
     .sort((a, b) => {
       return b.length - a.length
     })
   const boundValues = textWithBindings.match(regex) || []
   let result = textWithBindings
-  for (let boundValue of boundValues) {
+  for (const boundValue of boundValues) {
     let newBoundValue = boundValue
     // we use a search string, where any time we replace something we blank it out
     // in the search, working from longest to shortest so always use best match first

packages/builder/src/pages/builder/app/[application]/_components/BuilderGroupPopover.svelte

@@ -0,0 +1,31 @@
+<script>
+  import { Icon } from "@budibase/bbui"
+  import { helpers } from "@budibase/shared-core"
+
+  export let groups = []
+  function tooltip(groups) {
+    const sortedNames = groups
+      .sort((a, b) => a.name.localeCompare(b.name))
+      .map(group => group.name)
+    return `Member of ${helpers.lists.punctuateList(sortedNames)}`
+  }
+</script>
+
+<div class="icon">
+  <Icon
+    name="Info"
+    size="XS"
+    color="grey"
+    hoverable
+    tooltip={tooltip(groups)}
+    tooltipPosition="top"
+  />
+</div>
+
+<style>
+  .icon {
+    height: auto;
+    display: flex;
+    justify-content: center;
+  }
+</style>

packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte

@@ -37,6 +37,7 @@
   import { emailValidator } from "@/helpers/validation"
   import { fly } from "svelte/transition"
   import InfoDisplay from "../design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte"
+  import BuilderGroupPopover from "./BuilderGroupPopover.svelte"
 
   let query = null
   let loaded = false
@@ -197,12 +198,19 @@
       return
     }
     const update = await users.get(user._id)
-    await users.save({
+    const newRoles = {
-      ...update,
-      roles: {
       ...update.roles,
       [prodAppId]: role,
-      },
+    }
+    // make sure no undefined/null roles (during removal)
+    for (let [appId, role] of Object.entries(newRoles)) {
+      if (!role) {
+        delete newRoles[appId]
+      }
+    }
+    await users.save({
+      ...update,
+      roles: newRoles,
     })
     await searchUsers(query, $builderStore.builderSidePanel, loaded)
   }
@@ -539,6 +547,10 @@
       creationAccessType = Constants.Roles.CREATOR
     }
   }
+
+  const itemCountText = (word, count) => {
+    return `${count} ${word}${count !== 1 ? "s" : ""}`
+  }
 </script>
 
 <svelte:window on:keydown={handleKeyDown} />
@@ -701,13 +713,11 @@
                 >
                   <div class="details">
                     <GroupIcon {group} size="S" />
-                    <div>
+                    <div class="group-name">
                       {group.name}
                     </div>
                     <div class="auth-entity-meta">
-                      {`${group.users?.length} user${
+                      {itemCountText("user", group.users?.length)}
-                        group.users?.length != 1 ? "s" : ""
-                      }`}
                     </div>
                   </div>
                   <div class="auth-entity-access">
@@ -741,16 +751,33 @@
                 <div class="auth-entity-access-title">Access</div>
               </div>
               {#each allUsers as user}
+                {@const userGroups = sdk.users.getUserAppGroups(
+                  $appStore.appId,
+                  user,
+                  $groups
+                )}
                 <div class="auth-entity">
                   <div class="details">
+                    <div class="user-groups">
                       <div class="user-email" title={user.email}>
                         {user.email}
                       </div>
+                      {#if userGroups.length}
+                        <div class="group-info">
+                          <div class="auth-entity-meta">
+                            {itemCountText("group", userGroups.length)}
+                          </div>
+                          <BuilderGroupPopover groups={userGroups} />
+                        </div>
+                      {/if}
+                    </div>
                   </div>
                   <div class="auth-entity-access" class:muted={user.group}>
                     <RoleSelect
                       footer={getRoleFooter(user)}
-                      placeholder={false}
+                      placeholder={userGroups?.length
+                        ? "Controlled by group"
+                        : false}
                       value={parseRole(user)}
                       allowRemove={user.role && !user.group}
                       allowPublic={false}
@@ -915,6 +942,7 @@
     color: var(--spectrum-global-color-gray-600);
     font-size: 12px;
     white-space: nowrap;
+    text-align: end;
   }
 
   .auth-entity-access {
@@ -931,7 +959,7 @@
 
   .auth-entity,
   .auth-entity-header {
-    padding: 0px var(--spacing-xl);
+    padding: 0 var(--spacing-xl);
   }
 
   .auth-entity,
@@ -946,15 +974,17 @@
     display: flex;
     align-items: center;
     gap: var(--spacing-m);
-    color: var(--spectrum-global-color-gray-900);
     overflow: hidden;
+    width: 100%;
   }
 
-  .auth-entity .user-email {
+  .auth-entity .user-email,
-    text-overflow: ellipsis;
+  .group-name {
-    white-space: nowrap;
+    flex: 1 1 0;
+    min-width: 0;
     overflow: hidden;
-    color: var(--spectrum-global-color-gray-900);
+    white-space: nowrap;
+    text-overflow: ellipsis;
   }
 
   #builder-side-panel-container {
@@ -1048,4 +1078,23 @@
   .alert {
     padding: 0 var(--spacing-xl);
   }
+
+  .user-groups {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+    gap: var(--spacing-m);
+    width: 100%;
+    min-width: 0;
+  }
+
+  .group-info {
+    display: flex;
+    flex-direction: row;
+    gap: var(--spacing-xs);
+    justify-content: end;
+    width: 60px;
+    flex: 0 0 auto;
+  }
 </style>

packages/builder/src/pages/builder/portal/users/users/[userId].svelte

@@ -98,7 +98,9 @@
   $: privileged = sdk.users.isAdminOrGlobalBuilder(user)
   $: nameLabel = getNameLabel(user)
   $: filteredGroups = getFilteredGroups(internalGroups, searchTerm)
-  $: availableApps = getAvailableApps($appsStore.apps, privileged, user?.roles)
+  $: availableApps = user
+    ? getApps(user, sdk.users.userAppAccessList(user, $groups || []))
+    : []
   $: userGroups = $groups.filter(x => {
     return x.users?.find(y => {
       return y._id === userId
@@ -107,23 +109,19 @@
   $: globalRole = users.getUserRole(user)
   $: isTenantOwner = tenantOwner?.email && tenantOwner.email === user?.email
 
-  const getAvailableApps = (appList, privileged, roles) => {
+  const getApps = (user, appIds) => {
-    let availableApps = appList.slice()
+    let availableApps = $appsStore.apps
-    if (!privileged) {
+      .slice()
-      availableApps = availableApps.filter(x => {
+      .filter(app =>
-        let roleKeys = Object.keys(roles || {})
+        appIds.find(id => id === appsStore.getProdAppID(app.devId))
-        return roleKeys.concat(user?.builder?.apps).find(y => {
+      )
-          return x.appId === appsStore.extractAppId(y)
-        })
-      })
-    }
     return availableApps.map(app => {
       const prodAppId = appsStore.getProdAppID(app.devId)
       return {
         name: app.name,
         devId: app.devId,
         icon: app.icon,
-        role: getRole(prodAppId, roles),
+        role: getRole(prodAppId, user),
       }
     })
   }
@@ -136,7 +134,7 @@
     return groups.filter(group => group.name?.toLowerCase().includes(search))
   }
 
-  const getRole = (prodAppId, roles) => {
+  const getRole = (prodAppId, user) => {
     if (privileged) {
       return Constants.Roles.ADMIN
     }
@@ -145,7 +143,21 @@
       return Constants.Roles.CREATOR
     }
 
-    return roles[prodAppId]
+    if (user?.roles[prodAppId]) {
+      return user.roles[prodAppId]
+    }
+
+    // check if access via group for creator
+    const foundGroup = $groups?.find(
+      group => group.roles[prodAppId] || group.builder?.apps[prodAppId]
+    )
+    if (foundGroup.builder?.apps[prodAppId]) {
+      return Constants.Roles.CREATOR
+    }
+    // can't tell how groups will control role
+    if (foundGroup.roles[prodAppId]) {
+      return Constants.Roles.GROUP
+    }
   }
 
   const getNameLabel = user => {

packages/builder/src/pages/builder/portal/users/users/_components/AppRoleTableRenderer.svelte

@@ -15,7 +15,9 @@
   }
 </script>
 
-{#if value === Constants.Roles.CREATOR}
+{#if value === Constants.Roles.GROUP}
+  Controlled by group
+{:else if value === Constants.Roles.CREATOR}
   Can edit
 {:else}
   <StatusLight

packages/builder/src/pages/builder/portal/users/users/index.svelte

@@ -128,7 +128,7 @@
           $auth.user?.email === user.email
             ? false
             : true,
-        apps: [...new Set(Object.keys(user.roles))],
+        apps: sdk.users.userAppAccessList(user, $groups),
         access: role.sortOrder,
       }
     })

packages/builder/src/stores/builder/automations.ts

@@ -15,7 +15,6 @@ import {
 import {
   AutomationTriggerStepId,
   AutomationEventType,
-  AutomationStepType,
   AutomationActionStepId,
   Automation,
   AutomationStep,
@@ -26,10 +25,14 @@ import {
   UILogicalOperator,
   EmptyFilterOption,
   AutomationIOType,
-  AutomationStepSchema,
-  AutomationTriggerSchema,
   BranchPath,
   BlockDefinitions,
+  isBranchStep,
+  isTrigger,
+  isRowUpdateTrigger,
+  isRowSaveTrigger,
+  isAppTrigger,
+  BranchStep,
 } from "@budibase/types"
 import { ActionStepID } from "@/constants/backend/automations"
 import { FIELDS } from "@/constants/backend"
@@ -291,16 +294,16 @@ const automationActions = (store: AutomationStore) => ({
     let result: (AutomationStep | AutomationTrigger)[] = []
     pathWay.forEach(path => {
       const { stepIdx, branchIdx } = path
-      let last = result.length ? result[result.length - 1] : []
       if (!result.length) {
         // Preceeding steps.
         result = steps.slice(0, stepIdx + 1)
         return
       }
-      if (last && "inputs" in last) {
+      let last = result[result.length - 1]
+      if (isBranchStep(last)) {
         if (Number.isInteger(branchIdx)) {
           const branchId = last.inputs.branches[branchIdx].id
-          const children = last.inputs.children[branchId]
+          const children = last.inputs.children?.[branchId] || []
           const stepChildren = children.slice(0, stepIdx + 1)
           // Preceeding steps.
           result = result.concat(stepChildren)
@@ -473,23 +476,28 @@ const automationActions = (store: AutomationStore) => ({
           id: block.id,
         },
       ]
-      const branches: Branch[] = block.inputs?.branches || []
+
+      if (isBranchStep(block)) {
+        const branches = block.inputs?.branches || []
+        const children = block.inputs?.children || {}
 
         branches.forEach((branch, bIdx) => {
-        block.inputs?.children[branch.id].forEach(
+          children[branch.id].forEach(
             (bBlock: AutomationStep, sIdx: number, array: AutomationStep[]) => {
-            const ended =
+              const ended = array.length - 1 === sIdx
-              array.length - 1 === sIdx && !bBlock.inputs?.branches?.length
               treeTraverse(bBlock, pathToCurrentNode, sIdx, bIdx, ended)
             }
           )
         })
 
+        terminating = terminating && !branches.length
+      }
+
       store.actions.registerBlock(
         blockRefs,
         block,
         pathToCurrentNode,
-        terminating && !branches.length
+        terminating
       )
     }
 
@@ -497,7 +505,6 @@ const automationActions = (store: AutomationStore) => ({
     blocks.forEach((block, idx, array) => {
       treeTraverse(block, null, idx, null, array.length - 1 === idx)
     })
-
     return blockRefs
   },
 
@@ -575,7 +582,6 @@ const automationActions = (store: AutomationStore) => ({
           pathBlock.stepId === ActionStepID.LOOP &&
           pathBlock.blockToLoop in blocks
       }
-      const isTrigger = pathBlock.type === AutomationStepType.TRIGGER
 
       if (isLoopBlock && loopBlockCount == 0) {
         schema = {
@@ -586,17 +592,14 @@ const automationActions = (store: AutomationStore) => ({
         }
       }
 
-      const icon = isTrigger
+      const icon = isTrigger(pathBlock)
         ? pathBlock.icon
         : isLoopBlock
         ? "Reuse"
         : pathBlock.icon
 
-      if (blockIdx === 0 && isTrigger) {
+      if (blockIdx === 0 && isTrigger(pathBlock)) {
-        if (
+        if (isRowUpdateTrigger(pathBlock) || isRowSaveTrigger(pathBlock)) {
-          pathBlock.event === AutomationEventType.ROW_UPDATE ||
-          pathBlock.event === AutomationEventType.ROW_SAVE
-        ) {
           let table: any = get(tables).list.find(
             (table: Table) => table._id === pathBlock.inputs.tableId
           )
@@ -608,7 +611,7 @@ const automationActions = (store: AutomationStore) => ({
             }
           }
           delete schema.row
-        } else if (pathBlock.event === AutomationEventType.APP_TRIGGER) {
+        } else if (isAppTrigger(pathBlock)) {
           schema = Object.fromEntries(
             Object.keys(pathBlock.inputs.fields || []).map(key => [
               key,
@@ -915,8 +918,10 @@ const automationActions = (store: AutomationStore) => ({
     ]
 
     let cache:
-      | AutomationStepSchema<AutomationActionStepId>
+      | AutomationStep
-      | AutomationTriggerSchema<AutomationTriggerStepId>
+      | AutomationTrigger
+      | AutomationStep[]
+      | undefined = undefined
 
     pathWay.forEach((path, pathIdx, array) => {
       const { stepIdx, branchIdx } = path
@@ -938,9 +943,13 @@ const automationActions = (store: AutomationStore) => ({
         }
         return
       }
-      if (Number.isInteger(branchIdx)) {
+      if (
+        Number.isInteger(branchIdx) &&
+        !Array.isArray(cache) &&
+        isBranchStep(cache)
+      ) {
         const branchId = cache.inputs.branches[branchIdx].id
-        const children = cache.inputs.children[branchId]
+        const children = cache.inputs.children?.[branchId] || []
 
         if (final) {
           insertBlock(children, stepIdx)
@@ -1090,7 +1099,7 @@ const automationActions = (store: AutomationStore) => ({
   branchLeft: async (
     pathTo: Array<any>,
     automation: Automation,
-    block: AutomationStep
+    block: BranchStep
   ) => {
     const update = store.actions.shiftBranch(pathTo, block)
     if (update) {
@@ -1113,7 +1122,7 @@ const automationActions = (store: AutomationStore) => ({
   branchRight: async (
     pathTo: Array<BranchPath>,
     automation: Automation,
-    block: AutomationStep
+    block: BranchStep
   ) => {
     const update = store.actions.shiftBranch(pathTo, block, 1)
     if (update) {
@@ -1133,7 +1142,7 @@ const automationActions = (store: AutomationStore) => ({
    * @param {Number} direction - the direction of the swap. Defaults to -1 for left, add 1 for right
    * @returns
    */
-  shiftBranch: (pathTo: Array<any>, block: AutomationStep, direction = -1) => {
+  shiftBranch: (pathTo: Array<any>, block: BranchStep, direction = -1) => {
     let newBlock = cloneDeep(block)
     const branchPath = pathTo.at(-1)
     const targetIdx = branchPath.branchIdx

packages/builder/src/stores/builder/screenComponent.ts

@@ -8,6 +8,7 @@ import {
   UIComponentError,
   ComponentDefinition,
   DependsOnComponentSetting,
+  Screen,
 } from "@budibase/types"
 import { queries } from "./queries"
 import { views } from "./views"
@@ -66,6 +67,7 @@ export const screenComponentErrorList = derived(
     if (!$selectedScreen) {
       return []
     }
+    const screen = $selectedScreen
 
     const datasources = {
       ...reduceBy("_id", $tables.list),
@@ -79,9 +81,11 @@ export const screenComponentErrorList = derived(
     const errors: UIComponentError[] = []
 
     function checkComponentErrors(component: Component, ancestors: string[]) {
-      errors.push(...getInvalidDatasources(component, datasources, definitions))
-      errors.push(...getMissingRequiredSettings(component, definitions))
       errors.push(...getMissingAncestors(component, definitions, ancestors))
+      errors.push(
+        ...getInvalidDatasources(screen, component, datasources, definitions)
+      )
+      errors.push(...getMissingRequiredSettings(component, definitions))
 
       for (const child of component._children || []) {
         checkComponentErrors(child, [...ancestors, component._component])
@@ -95,6 +99,7 @@ export const screenComponentErrorList = derived(
 )
 
 function getInvalidDatasources(
+  screen: Screen,
   component: Component,
   datasources: Record<string, any>,
   definitions: Record<string, ComponentDefinition>
@@ -234,7 +239,10 @@ function getMissingAncestors(
   ancestors: string[]
 ): UIComponentError[] {
   const definition = definitions[component._component]
-
+  if (ancestors.some(a => !a.startsWith(BudibasePrefix))) {
+    // We don't have a way to know what components are used within a plugin component
+    return []
+  }
   if (!definition?.requiredAncestors?.length) {
     return []
   }

packages/client/manifest.json

@@ -4492,6 +4492,12 @@
           }
         ]
       },
+      {
+        "type": "text",
+        "label": "Zoom level",
+        "key": "defaultZoom",
+        "defaultValue": "1"
+      },
       {
         "type": "event",
         "label": "On change",

packages/client/package.json

@@ -28,7 +28,7 @@
     "apexcharts": "^3.48.0",
     "dayjs": "^1.10.8",
     "downloadjs": "1.4.7",
-    "html5-qrcode": "^2.2.1",
+    "html5-qrcode": "^2.3.8",
     "leaflet": "^1.7.1",
     "sanitize-html": "^2.13.0",
     "screenfull": "^6.0.1",

packages/client/src/components/app/DataProvider.svelte

@@ -2,9 +2,7 @@
   import { getContext } from "svelte"
   import { Pagination, ProgressCircle } from "@budibase/bbui"
   import { fetchData, QueryUtils } from "@budibase/frontend-core"
-  import {
+  import type {
-    LogicalOperator,
-    EmptyFilterOption,
     TableSchema,
     SortOrder,
     SearchFilters,
@@ -14,6 +12,7 @@
     GroupUserDatasource,
     DataFetchOptions,
   } from "@budibase/types"
+  import { LogicalOperator, EmptyFilterOption } from "@budibase/types"
 
   type ProviderDatasource = Exclude<
     DataFetchDatasource,

packages/client/src/components/app/blocks/form/FormBlock.svelte

@@ -4,7 +4,7 @@
   import { Utils } from "@budibase/frontend-core"
   import FormBlockWrapper from "./FormBlockWrapper.svelte"
   import { get } from "svelte/store"
-  import { TableSchema, UIDatasource } from "@budibase/types"
+  import type { TableSchema, UIDatasource } from "@budibase/types"
 
   type Field = { name: string; active: boolean }
 

packages/client/src/components/app/forms/CodeScanner.svelte

@@ -20,6 +20,7 @@
   export let beepFrequency = 2637
   export let customFrequency = 1046
   export let preferredCamera = "environment"
+  export let defaultZoom = 1
   export let validator
 
   const dispatch = createEventDispatcher()
@@ -58,6 +59,14 @@
       html5QrCode
         .start(cameraSetting, cameraConfig, onScanSuccess)
         .then(() => {
+          if (defaultZoom > 1) {
+            const cameraOptions =
+              html5QrCode.getRunningTrackCameraCapabilities()
+            const zoom = cameraOptions.zoomFeature()
+            if (zoom.isSupported()) {
+              zoom.apply(defaultZoom)
+            }
+          }
           resolve({ initialised: true })
         })
         .catch(err => {

packages/client/src/components/app/forms/CodeScannerField.svelte

@@ -17,6 +17,7 @@
   export let beepFrequency
   export let customFrequency
   export let preferredCamera
+  export let defaultZoom
   export let helpText = null
 
   let fieldState
@@ -56,6 +57,7 @@
       {beepFrequency}
       {customFrequency}
       {preferredCamera}
+      {defaultZoom}
       validator={fieldState.validator}
     />
   {/if}

packages/client/src/components/app/forms/RelationshipField.svelte

@@ -1,18 +1,15 @@
 <script lang="ts">
   import { CoreSelect, CoreMultiselect } from "@budibase/bbui"
-  import { FieldType } from "@budibase/types"
+  import { FieldType, InternalTable } from "@budibase/types"
   import { fetchData, Utils } from "@budibase/frontend-core"
   import { getContext } from "svelte"
   import Field from "./Field.svelte"
   import type {
     SearchFilter,
     RelationshipFieldMetadata,
-    Table,
     Row,
   } from "@budibase/types"
 
-  const { API } = getContext("sdk")
-
   export let field: string | undefined = undefined
   export let label: string | undefined = undefined
   export let placeholder: any = undefined
@@ -20,10 +17,10 @@
   export let readonly: boolean = false
   export let validation: any
   export let autocomplete: boolean = true
-  export let defaultValue: string | undefined = undefined
+  export let defaultValue: string | string[] | undefined = undefined
   export let onChange: any
   export let filter: SearchFilter[]
-  export let datasourceType: "table" | "user" | "groupUser" = "table"
+  export let datasourceType: "table" | "user" = "table"
   export let primaryDisplay: string | undefined = undefined
   export let span: number | undefined = undefined
   export let helpText: string | undefined = undefined
@@ -32,191 +29,305 @@
     | FieldType.BB_REFERENCE
     | FieldType.BB_REFERENCE_SINGLE = FieldType.LINK
 
-  type RelationshipValue = { _id: string; [key: string]: any }
+  type BasicRelatedRow = { _id: string; primaryDisplay: string }
-  type OptionObj = Record<string, RelationshipValue>
+  type OptionsMap = Record<string, BasicRelatedRow>
-  type OptionsObjType = Record<string, OptionObj>
 
+  const { API } = getContext("sdk")
+
+  // Field state
   let fieldState: any
   let fieldApi: any
   let fieldSchema: RelationshipFieldMetadata | undefined
-  let tableDefinition: Table | null | undefined
-  let searchTerm: any
-  let open: boolean
-  let selectedValue: string[] | string
 
-  // need a cast version of this for reactivity, components below aren't typed
+  // Local UI state
-  $: castSelectedValue = selectedValue as any
+  let searchTerm: any
+  let open: boolean = false
+
+  // Options state
+  let options: BasicRelatedRow[] = []
+  let optionsMap: OptionsMap = {}
+  let loadingMissingOptions: boolean = false
+
+  // Determine if we can select multiple rows or not
   $: multiselect =
     [FieldType.LINK, FieldType.BB_REFERENCE].includes(type) &&
     fieldSchema?.relationshipType !== "one-to-many"
-  $: linkedTableId = fieldSchema?.tableId!
+
-  $: fetch = fetchData({
+  // Get the proper string representation of the value
+  $: realValue = fieldState?.value
+  $: selectedValue = parseSelectedValue(realValue, multiselect)
+  $: selectedIDs = getSelectedIDs(selectedValue)
+
+  // If writable, we use a fetch to load options
+  $: linkedTableId = fieldSchema?.tableId
+  $: writable = !disabled && !readonly
+  $: fetch = createFetch(writable, datasourceType, filter, linkedTableId)
+
+  // Attempt to determine the primary display field to use
+  $: tableDefinition = $fetch?.definition
+  $: primaryDisplayField = primaryDisplay || tableDefinition?.primaryDisplay
+
+  // Build our options map
+  $: rows = $fetch?.rows || []
+  $: processOptions(realValue, rows, primaryDisplayField)
+
+  // If we ever have a value selected for which we don't have an option, we must
+  // fetch those rows to ensure we can render them as options
+  $: missingIDs = selectedIDs.filter(id => !optionsMap[id])
+  $: loadMissingOptions(missingIDs, linkedTableId, primaryDisplayField)
+
+  // Convert our options map into an array for display
+  $: updateOptions(optionsMap)
+  $: !open && sortOptions()
+
+  // Search for new options when search term changes
+  $: debouncedSearchOptions(searchTerm || "", primaryDisplayField)
+
+  // Ensure backwards compatibility
+  $: enrichedDefaultValue = enrichDefaultValue(defaultValue)
+
+  // We need to cast value to pass it down, as those components aren't typed
+  $: emptyValue = multiselect ? [] : null
+  $: displayValue = missingIDs.length ? emptyValue : (selectedValue as any)
+
+  // Ensures that we flatten any objects so that only the IDs of the selected
+  // rows are passed down. Not sure how this can be an object to begin with?
+  const parseSelectedValue = (
+    value: any,
+    multiselect: boolean
+  ): undefined | string | string[] => {
+    return multiselect ? flatten(value) : flatten(value)[0]
+  }
+
+  // Where applicable, creates the fetch instance to load row options
+  const createFetch = (
+    writable: boolean,
+    dsType: typeof datasourceType,
+    filter: SearchFilter[],
+    linkedTableId?: string
+  ) => {
+    if (!linkedTableId) {
+      return undefined
+    }
+    const datasource =
+      datasourceType === "table"
+        ? {
+            type: datasourceType,
+            tableId: fieldSchema?.tableId!,
+          }
+        : {
+            type: datasourceType,
+            tableId: InternalTable.USER_METADATA,
+          }
+    return fetchData({
       API,
-    datasource: {
+      datasource,
-      // typing here doesn't seem correct - we have the correct datasourceType options
-      // but when we configure the fetchData, it seems to think only "table" is valid
-      type: datasourceType as any,
-      tableId: linkedTableId,
-    },
       options: {
         filter,
-      limit: 100,
+        limit: writable ? 100 : 1,
       },
     })
-
-  $: tableDefinition = $fetch.definition
-  $: selectedValue = multiselect
-    ? flatten(fieldState?.value) ?? []
-    : flatten(fieldState?.value)?.[0]
-  $: component = multiselect ? CoreMultiselect : CoreSelect
-  $: primaryDisplay = primaryDisplay || tableDefinition?.primaryDisplay
-
-  let optionsObj: OptionsObjType = {}
-  const debouncedFetchRows = Utils.debounce(fetchRows, 250)
-
-  $: {
-    if (primaryDisplay && fieldState && !optionsObj) {
-      // Persist the initial values as options, allowing them to be present in the dropdown,
-      // even if they are not in the inital fetch results
-      let valueAsSafeArray = fieldState.value || []
-      if (!Array.isArray(valueAsSafeArray)) {
-        valueAsSafeArray = [fieldState.value]
   }
-      optionsObj = valueAsSafeArray.reduce(
+
-        (
+  // Small helper to represent the selected value as an array
-          accumulator: OptionObj,
+  const getSelectedIDs = (
-          value: { _id: string; primaryDisplay: any }
+    selectedValue: undefined | string | string[]
+  ): string[] => {
+    if (!selectedValue) {
+      return []
+    }
+    return Array.isArray(selectedValue) ? selectedValue : [selectedValue]
+  }
+
+  // Builds a map of all available options, in a consistent structure
+  const processOptions = (
+    realValue: any | any[],
+    rows: Row[],
+    primaryDisplay?: string
   ) => {
-          // fieldState has to be an array of strings to be valid for an update
+    // First ensure that all options included in the value are present as valid
-          // therefore we cannot guarantee value will be an object
+    // options. These can be basic related row shapes which already include
-          // https://linear.app/budibase/issue/BUDI-7577/refactor-the-relationshipfield-component-to-have-better-support-for
+    // a value for primary display
-          if (!value._id) {
+    if (realValue) {
-            return accumulator
+      const valueArray = Array.isArray(realValue) ? realValue : [realValue]
+      for (let val of valueArray) {
+        const option = parseOption(val, primaryDisplay)
+        if (option) {
+          optionsMap[option._id] = option
         }
-          accumulator[value._id] = {
-            _id: value._id,
-            [primaryDisplay]: value.primaryDisplay,
-          }
-          return accumulator
-        },
-        {}
-      )
       }
     }
 
-  $: enrichedOptions = enrichOptions(optionsObj, $fetch.rows)
+    // Process all rows loaded from our fetch
-  const enrichOptions = (optionsObj: OptionsObjType, fetchResults: Row[]) => {
+    for (let row of rows) {
-    const result = (fetchResults || [])?.reduce((accumulator, row) => {
+      const option = parseOption(row, primaryDisplay)
-      if (!accumulator[row._id!]) {
+      if (option) {
-        accumulator[row._id!] = row
+        optionsMap[option._id] = option
-      }
-      return accumulator
-    }, optionsObj || {})
-
-    return Object.values(result)
-  }
-  $: {
-    // We don't want to reorder while the dropdown is open, to avoid UX jumps
-    if (!open && primaryDisplay) {
-      enrichedOptions = enrichedOptions.sort((a: OptionObj, b: OptionObj) => {
-        const selectedValues = flatten(fieldState?.value) || []
-
-        const aIsSelected = selectedValues.find(
-          (v: RelationshipValue) => v === a._id
-        )
-        const bIsSelected = selectedValues.find(
-          (v: RelationshipValue) => v === b._id
-        )
-        if (aIsSelected && !bIsSelected) {
-          return -1
-        } else if (!aIsSelected && bIsSelected) {
-          return 1
-        }
-
-        return (a[primaryDisplay] > b[primaryDisplay]) as unknown as number
-      })
       }
     }
 
-  $: {
+    // Reassign to trigger reactivity
-    if (filter || defaultValue) {
+    optionsMap = optionsMap
-      forceFetchRows()
   }
-  }
-  $: debouncedFetchRows(searchTerm, primaryDisplay, defaultValue)
 
-  const forceFetchRows = async () => {
+  // Parses a row-like structure into a properly shaped option
-    // if the filter has changed, then we need to reset the options, clear the selection, and re-fetch
+  const parseOption = (
-    optionsObj = {}
+    option: any | BasicRelatedRow | Row,
-    fieldApi?.setValue([])
+    primaryDisplay?: string
-    selectedValue = []
+  ): BasicRelatedRow | null => {
-    debouncedFetchRows(searchTerm, primaryDisplay, defaultValue)
+    if (!option || typeof option !== "object" || !option?._id) {
+      return null
     }
-  async function fetchRows(
+    // If this is a basic related row shape (_id and PD only) then just use
-    searchTerm: any,
+    // that
-    primaryDisplay: string,
+    if (Object.keys(option).length === 2 && "primaryDisplay" in option) {
-    defaultVal: string | string[]
+      return {
+        _id: option._id,
+        primaryDisplay: ensureString(option.primaryDisplay),
+      }
+    }
+    // Otherwise use the primary display field specified
+    if (primaryDisplay) {
+      return {
+        _id: option._id,
+        primaryDisplay: ensureString(
+          option[primaryDisplay as keyof typeof option]
+        ),
+      }
+    } else {
+      return {
+        _id: option._id,
+        primaryDisplay: option._id,
+      }
+    }
+  }
+
+  // Loads any rows which are selected and aren't present in the currently
+  // available option set. This is typically only IDs specified as default
+  // values.
+  const loadMissingOptions = async (
+    missingIDs: string[],
+    linkedTableId?: string,
+    primaryDisplay?: string
+  ) => {
+    if (
+      loadingMissingOptions ||
+      !missingIDs.length ||
+      !linkedTableId ||
+      !primaryDisplay
     ) {
-    const allRowsFetched =
-      $fetch.loaded &&
-      !Object.keys($fetch.query?.string || {}).length &&
-      !$fetch.hasNextPage
-    // Don't request until we have the primary display or default value has been fetched
-    if (allRowsFetched || !primaryDisplay) {
       return
     }
-    // must be an array
+    loadingMissingOptions = true
-    const defaultValArray: string[] = !defaultVal
+    try {
-      ? []
+      const res = await API.searchTable(linkedTableId, {
-      : !Array.isArray(defaultVal)
+        query: {
-      ? defaultVal.split(",")
+          oneOf: {
-      : defaultVal
+            _id: missingIDs,
+          },
+        },
+      })
+      for (let row of res.rows) {
+        const option = parseOption(row, primaryDisplay)
+        if (option) {
+          optionsMap[option._id] = option
+        }
+      }
 
-    if (
+      // Reassign to trigger reactivity
-      defaultVal &&
+      optionsMap = optionsMap
-      optionsObj &&
+      updateOptions(optionsMap)
-      defaultValArray.some(val => !optionsObj[val])
+    } catch (error) {
-    ) {
+      console.error("Error loading missing row IDs", error)
-      await fetch.update({
+    } finally {
-        query: { oneOf: { _id: defaultValArray } },
+      // Ensure we have some sort of option for all IDs
+      for (let id of missingIDs) {
+        if (!optionsMap[id]) {
+          optionsMap[id] = {
+            _id: id,
+            primaryDisplay: id,
+          }
+        }
+      }
+      loadingMissingOptions = false
+    }
+  }
+
+  // Updates the options list to reflect the currently available options
+  const updateOptions = (optionsMap: OptionsMap) => {
+    let newOptions = Object.values(optionsMap)
+
+    // Only override options if the quantity of options changes
+    if (newOptions.length !== options.length) {
+      options = newOptions
+      sortOptions()
+    }
+  }
+
+  // Sorts the options list by selected state, then by primary display
+  const sortOptions = () => {
+    // Create a quick lookup map so we can test whether options are selected
+    const selectedMap: Record<string, boolean> = selectedIDs.reduce(
+      (map, id) => ({ ...map, [id]: true }),
+      {}
+    )
+    options.sort((a, b) => {
+      const aSelected = !!selectedMap[a._id]
+      const bSelected = !!selectedMap[b._id]
+      if (aSelected === bSelected) {
+        return a.primaryDisplay < b.primaryDisplay ? -1 : 1
+      } else {
+        return aSelected ? -1 : 1
+      }
     })
   }
 
-    if (
+  // Util to ensure a value is stringified
-      (Array.isArray(selectedValue) &&
+  const ensureString = (val: any): string => {
-        selectedValue.some(val => !optionsObj[val])) ||
+    return typeof val === "string" ? val : JSON.stringify(val)
-      (selectedValue && !optionsObj[selectedValue as string])
+  }
-    ) {
+
-      await fetch.update({
+  // We previously included logic to manually process default value, which
-        query: {
+  // should not be done as it is handled by the core form logic.
-          oneOf: {
+  // This logic included handling a comma separated list of IDs, so for
-            _id: Array.isArray(selectedValue) ? selectedValue : [selectedValue],
+  // backwards compatibility we must now unfortunately continue to handle that
-          },
+  // at this level.
-        },
+  const enrichDefaultValue = (val: any) => {
-      })
+    if (!val || typeof val !== "string") {
+      return val
+    }
+    return val.includes(",") ? val.split(",") : val
+  }
+
+  // Searches for new options matching the given term
+  async function searchOptions(searchTerm: string, primaryDisplay?: string) {
+    if (!primaryDisplay) {
+      return
     }
 
     // Ensure we match all filters, rather than any
+    let newFilter: any = filter
+    if (searchTerm) {
       // @ts-expect-error this doesn't fit types, but don't want to change it yet
-    const baseFilter: any = (filter || []).filter(x => x.operator !== "allOr")
+      newFilter = (newFilter || []).filter(x => x.operator !== "allOr")
-    await fetch.update({
+      newFilter.push({
-      filter: [
-        ...baseFilter,
-        {
         // Use a big numeric prefix to avoid clashing with an existing filter
         field: `999:${primaryDisplay}`,
         operator: "string",
         value: searchTerm,
-        },
-      ],
       })
     }
+    await fetch?.update({
+      filter: newFilter,
+    })
+  }
+  const debouncedSearchOptions = Utils.debounce(searchOptions, 250)
 
-  const flatten = (values: any | any[]) => {
+  // Flattens an array of row-like objects into a simple array of row IDs
+  const flatten = (values: any | any[]): string[] => {
     if (!values) {
       return []
     }
-
     if (!Array.isArray(values)) {
       values = [values]
     }
@@ -226,16 +337,11 @@
     return values
   }
 
-  const getDisplayName = (row: Row) => {
-    return row?.[primaryDisplay!] || "-"
-  }
-
   const handleChange = (e: any) => {
     let value = e.detail
     if (!multiselect) {
       value = value == null ? [] : [value]
     }
-
     if (
       type === FieldType.BB_REFERENCE_SINGLE &&
       value &&
@@ -243,7 +349,6 @@
     ) {
       value = value[0] || null
     }
-
     const changed = fieldApi.setValue(value)
     if (onChange && changed) {
       onChange({
@@ -251,12 +356,6 @@
       })
     }
   }
-
-  const loadMore = () => {
-    if (!$fetch.loading) {
-      fetch.nextPage()
-    }
-  }
 </script>
 
 <Field
@@ -265,31 +364,31 @@
   {disabled}
   {readonly}
   {validation}
-  {defaultValue}
   {type}
   {span}
   {helpText}
+  defaultValue={enrichedDefaultValue}
   bind:fieldState
   bind:fieldApi
   bind:fieldSchema
 >
   {#if fieldState}
     <svelte:component
-      this={component}
+      this={multiselect ? CoreMultiselect : CoreSelect}
-      options={enrichedOptions}
+      value={displayValue}
-      {autocomplete}
+      id={fieldState?.fieldId}
-      value={castSelectedValue}
+      disabled={fieldState?.disabled}
-      on:change={handleChange}
+      readonly={fieldState?.readonly}
-      on:loadMore={loadMore}
+      loading={!!$fetch?.loading}
-      id={fieldState.fieldId}
+      getOptionLabel={option => option.primaryDisplay}
-      disabled={fieldState.disabled}
-      readonly={fieldState.readonly}
-      getOptionLabel={getDisplayName}
       getOptionValue={option => option._id}
+      {options}
       {placeholder}
+      {autocomplete}
       bind:searchTerm
-      loading={$fetch.loading}
       bind:open
+      on:change={handleChange}
+      on:loadMore={() => fetch?.nextPage()}
     />
   {/if}
 </Field>

packages/client/src/components/error-states/ComponentErrorState.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
   import { getContext } from "svelte"
   import { Icon } from "@budibase/bbui"
-  import { UIComponentError } from "@budibase/types"
+  import type { UIComponentError } from "@budibase/types"
   import ComponentErrorStateCta from "./ComponentErrorStateCTA.svelte"
 
   export let componentErrors: UIComponentError[] | undefined

packages/client/src/components/error-states/ComponentErrorStateCTA.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
   import { getContext } from "svelte"
-  import { UIComponentError } from "@budibase/types"
+  import type { UIComponentError } from "@budibase/types"
 
   export let error: UIComponentError | undefined
 

packages/client/src/components/preview/DNDHandler.svelte

@@ -6,7 +6,7 @@
   import { findComponentById } from "@/utils/components.js"
   import { isGridEvent } from "@/utils/grid"
   import { DNDPlaceholderID } from "@/constants"
-  import { Component } from "@budibase/types"
+  import type { Component } from "@budibase/types"
 
   type ChildCoords = {
     placeholder: boolean

packages/client/src/index.js

@@ -1,142 +0,0 @@
-import ClientApp from "./components/ClientApp.svelte"
-import UpdatingApp from "./components/UpdatingApp.svelte"
-import {
-  builderStore,
-  appStore,
-  blockStore,
-  componentStore,
-  environmentStore,
-  dndStore,
-  eventStore,
-  hoverStore,
-  stateStore,
-  routeStore,
-} from "./stores"
-import loadSpectrumIcons from "@budibase/bbui/spectrum-icons-vite.js"
-import { get } from "svelte/store"
-import { initWebsocket } from "./websocket.js"
-
-// Provide svelte and svelte/internal as globals for custom components
-import * as svelte from "svelte"
-import * as internal from "svelte/internal"
-
-window.svelte_internal = internal
-window.svelte = svelte
-
-// Initialise spectrum icons
-loadSpectrumIcons()
-
-let app
-
-const loadBudibase = async () => {
-  // Update builder store with any builder flags
-  builderStore.set({
-    ...get(builderStore),
-    inBuilder: !!window["##BUDIBASE_IN_BUILDER##"],
-    layout: window["##BUDIBASE_PREVIEW_LAYOUT##"],
-    screen: window["##BUDIBASE_PREVIEW_SCREEN##"],
-    selectedComponentId: window["##BUDIBASE_SELECTED_COMPONENT_ID##"],
-    previewId: window["##BUDIBASE_PREVIEW_ID##"],
-    theme: window["##BUDIBASE_PREVIEW_THEME##"],
-    customTheme: window["##BUDIBASE_PREVIEW_CUSTOM_THEME##"],
-    previewDevice: window["##BUDIBASE_PREVIEW_DEVICE##"],
-    navigation: window["##BUDIBASE_PREVIEW_NAVIGATION##"],
-    hiddenComponentIds: window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"],
-    usedPlugins: window["##BUDIBASE_USED_PLUGINS##"],
-    location: window["##BUDIBASE_LOCATION##"],
-    snippets: window["##BUDIBASE_SNIPPETS##"],
-    componentErrors: window["##BUDIBASE_COMPONENT_ERRORS##"],
-  })
-
-  // Set app ID - this window flag is set by both the preview and the real
-  // server rendered app HTML
-  appStore.actions.setAppId(window["##BUDIBASE_APP_ID##"])
-
-  // Set the flag used to determine if the app is being loaded via an iframe
-  appStore.actions.setAppEmbedded(
-    window["##BUDIBASE_APP_EMBEDDED##"] === "true"
-  )
-
-  if (window.MIGRATING_APP) {
-    new UpdatingApp({
-      target: window.document.body,
-    })
-    return
-  }
-
-  // Fetch environment info
-  if (!get(environmentStore)?.loaded) {
-    await environmentStore.actions.fetchEnvironment()
-  }
-
-  // Register handler for runtime events from the builder
-  window.handleBuilderRuntimeEvent = (type, data) => {
-    if (!window["##BUDIBASE_IN_BUILDER##"]) {
-      return
-    }
-    if (type === "event-completed") {
-      eventStore.actions.resolveEvent(data)
-    } else if (type === "eject-block") {
-      const block = blockStore.actions.getBlock(data)
-      block?.eject()
-    } else if (type === "dragging-new-component") {
-      const { dragging, component } = data
-      if (dragging) {
-        const definition =
-          componentStore.actions.getComponentDefinition(component)
-        dndStore.actions.startDraggingNewComponent({ component, definition })
-      } else {
-        dndStore.actions.reset()
-      }
-    } else if (type === "request-context") {
-      const { selectedComponentInstance, screenslotInstance } =
-        get(componentStore)
-      const instance = selectedComponentInstance || screenslotInstance
-      const context = instance?.getDataContext()
-      let stringifiedContext = null
-      try {
-        stringifiedContext = JSON.stringify(context)
-      } catch (error) {
-        // Ignore - invalid context
-      }
-      eventStore.actions.dispatchEvent("provide-context", {
-        context: stringifiedContext,
-      })
-    } else if (type === "hover-component") {
-      hoverStore.actions.hoverComponent(data, false)
-    } else if (type === "builder-meta") {
-      builderStore.actions.setMetadata(data)
-    } else if (type === "builder-state") {
-      const [[key, value]] = Object.entries(data)
-      stateStore.actions.setValue(key, value)
-    } else if (type === "builder-url-test-data") {
-      const { route, testValue } = data
-      routeStore.actions.setTestUrlParams(route, testValue)
-    }
-  }
-
-  // Register any custom components
-  if (window["##BUDIBASE_CUSTOM_COMPONENTS##"]) {
-    window["##BUDIBASE_CUSTOM_COMPONENTS##"].forEach(component => {
-      componentStore.actions.registerCustomComponent(component)
-    })
-  }
-
-  // Make a callback available for custom component bundles to register
-  // themselves at runtime
-  window.registerCustomComponent =
-    componentStore.actions.registerCustomComponent
-
-  // Initialise websocket
-  initWebsocket()
-
-  // Create app if one hasn't been created yet
-  if (!app) {
-    app = new ClientApp({
-      target: window.document.body,
-    })
-  }
-}
-
-// Attach to window so the HTML template can call this when it loads
-window.loadBudibase = loadBudibase

packages/frontend-core/src/constants.ts

@@ -106,6 +106,7 @@ export const Roles = {
   PUBLIC: "PUBLIC",
   BUILDER: "BUILDER",
   CREATOR: "CREATOR",
+  GROUP: "GROUP",
 }
 
 export const EventPublishType = {

packages/frontend-core/src/fetch/GroupUserFetch.ts

@@ -4,7 +4,7 @@ import { GroupUserDatasource, InternalTable } from "@budibase/types"
 
 interface GroupUserQuery {
   groupId: string
-  emailSearch: string
+  emailSearch?: string
 }
 
 interface GroupUserDefinition {

packages/frontend-core/src/fetch/UserFetch.ts

@@ -9,8 +9,8 @@ import {
 } from "@budibase/types"
 
 interface UserFetchQuery {
-  appId: string
+  appId?: string
-  paginated: boolean
+  paginated?: boolean
 }
 
 interface UserDefinition {

packages/pro

@@ -1 +1 @@
-Subproject commit eb96d8b2f2029033b0f758078ed30c888e8fb249
+Subproject commit b28dbd549284cf450be7f25ad85aadf614d08f0b

packages/server/__mocks__/@elastic/elasticsearch.ts

@@ -1,24 +0,0 @@
-const elastic: any = {}
-
-elastic.Client = function () {
-  this.index = jest.fn().mockResolvedValue({ body: [] })
-  this.search = jest.fn().mockResolvedValue({
-    body: {
-      hits: {
-        hits: [
-          {
-            _source: {
-              name: "test",
-            },
-          },
-        ],
-      },
-    },
-  })
-  this.update = jest.fn().mockResolvedValue({ body: [] })
-  this.delete = jest.fn().mockResolvedValue({ body: [] })
-
-  this.close = jest.fn()
-}
-
-module.exports = elastic

packages/server/datasource-sha.env

@@ -1,5 +1,6 @@
-MSSQL_SHA=sha256:3b913841850a4d57fcfcb798be06acc88ea0f2acc5418bc0c140a43e91c4a545
+MSSQL_SHA=sha256:d252932ef839c24c61c1139cc98f69c85ca774fa7c6bfaaa0015b7eb02b9dc87
 MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588ebe
 POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e
 MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d
 MARIADB_SHA=sha256:e59ba8783bf7bc02a4779f103bb0d8751ac0e10f9471089709608377eded7aa8
+ELASTICSEARCH_SHA=sha256:9a6443f55243f6acbfeb4a112d15eb3b9aac74bf25e0e39fa19b3ddd3a6879d0

packages/server/package.json

@@ -156,6 +156,7 @@
     "@types/pouchdb": "6.4.2",
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.14",
+    "@types/swagger-jsdoc": "^6.0.4",
     "@types/tar": "6.1.5",
     "@types/tmp": "0.2.6",
     "@types/uuid": "8.3.4",

packages/server/specs/generate.ts

@@ -4,11 +4,11 @@ import { examples, schemas } from "./resources"
 import * as parameters from "./parameters"
 import * as security from "./security"
 
-const swaggerJsdoc = require("swagger-jsdoc")
+import swaggerJsdoc from "swagger-jsdoc"
 
 const VARIABLES = {}
 
-const options = {
+const opts: swaggerJsdoc.Options = {
   definition: {
     openapi: "3.0.0",
     info: {
@@ -58,7 +58,6 @@ const options = {
 }
 
 function writeFile(output: any, filename: string) {
-  try {
   const path = join(__dirname, filename)
   let spec = output
   if (filename.endsWith("json")) {
@@ -71,17 +70,15 @@ function writeFile(output: any, filename: string) {
   writeFileSync(path, spec)
   console.log(`Wrote spec to ${path}`)
   return path
-  } catch (err) {
-    console.error("Error writing spec file", err)
 }
+
+export function spec() {
+  return swaggerJsdoc({ ...opts, format: ".json" })
 }
 
 export function run() {
-  const outputJSON = swaggerJsdoc(options)
+  writeFile(swaggerJsdoc({ ...opts, format: ".json" }), "openapi.json")
-  options.format = ".yaml"
+  return writeFile(swaggerJsdoc({ ...opts, format: ".yaml" }), "openapi.yaml")
-  const outputYAML = swaggerJsdoc(options)
-  writeFile(outputJSON, "openapi.json")
-  return writeFile(outputYAML, "openapi.yaml")
 }
 
 if (require.main === module) {

packages/server/specs/resources/error.ts

@@ -0,0 +1,21 @@
+import { object } from "./utils"
+import Resource from "./utils/Resource"
+
+const errorSchema = object({
+  status: {
+    type: "number",
+    description: "The HTTP status code of the error.",
+  },
+  message: {
+    type: "string",
+    description: "A descriptive message about the error.",
+  },
+})
+
+export default new Resource()
+  .setExamples({
+    error: {},
+  })
+  .setSchemas({
+    error: errorSchema,
+  })

packages/server/src/api/controllers/application.ts

@@ -635,6 +635,11 @@ async function unpublishApp(ctx: UserCtx) {
   return result
 }
 
+async function invalidateAppCache(appId: string) {
+  await cache.app.invalidateAppMetadata(dbCore.getDevAppID(appId))
+  await cache.app.invalidateAppMetadata(dbCore.getProdAppID(appId))
+}
+
 async function destroyApp(ctx: UserCtx) {
   let appId = ctx.params.appId
   appId = dbCore.getProdAppID(appId)
@@ -654,17 +659,21 @@ async function destroyApp(ctx: UserCtx) {
   await quotas.removeApp()
   await events.app.deleted(app)
 
-  if (!env.isTest()) {
+  if (!env.USE_LOCAL_COMPONENT_LIBS) {
     await deleteAppFiles(appId)
   }
 
   await removeAppFromUserRoles(ctx, appId)
-  await cache.app.invalidateAppMetadata(devAppId)
+  await invalidateAppCache(appId)
   return result
 }
 
 async function preDestroyApp(ctx: UserCtx) {
-  const { rows } = await getUniqueRows([ctx.params.appId])
+  // invalidate the cache immediately in-case they are leading to
+  // zombie appearing apps
+  const appId = ctx.params.appId
+  await invalidateAppCache(appId)
+  const { rows } = await getUniqueRows([appId])
   ctx.rowCount = rows.length
 }
 

packages/server/src/api/controllers/integration.ts

@@ -9,6 +9,7 @@ import {
 const DISABLED_EXTERNAL_INTEGRATIONS = [
   SourceName.AIRTABLE,
   SourceName.BUDIBASE,
+  SourceName.ARANGODB,
 ]
 
 export async function fetch(ctx: UserCtx<void, FetchIntegrationsResponse>) {

packages/server/src/api/controllers/plugin/index.ts

@@ -11,6 +11,7 @@ import {
   UploadPluginResponse,
   FetchPluginResponse,
   DeletePluginResponse,
+  PluginMetadata,
 } from "@budibase/types"
 import env from "../../../environment"
 import { clientAppSocket } from "../../../websockets"
@@ -53,10 +54,11 @@ export async function create(
   const { source, url, headers, githubToken } = ctx.request.body
 
   try {
-    let metadata
+    let metadata: PluginMetadata
-    let directory
+    let directory: string
+
     // Generating random name as a backup and needed for url
-    let name = "PLUGIN_" + Math.floor(100000 + Math.random() * 900000)
+    const name = "PLUGIN_" + Math.floor(100000 + Math.random() * 900000)
 
     switch (source) {
       case PluginSource.NPM: {
@@ -81,12 +83,14 @@ export async function create(
         directory = directoryUrl
         break
       }
+      default:
+        ctx.throw(400, "Invalid source")
     }
 
-    pluginCore.validate(metadata?.schema)
+    pluginCore.validate(metadata.schema)
 
     // Only allow components in cloud
-    if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) {
+    if (!env.SELF_HOSTED && metadata.schema?.type !== PluginType.COMPONENT) {
       throw new Error(
         "Only component plugins are supported outside of self-host"
       )

packages/server/src/api/controllers/public/users.ts

@@ -48,7 +48,7 @@ function getUser(ctx: UserCtx, userId?: string) {
   if (userId) {
     ctx.params = { userId }
   } else if (!ctx.params?.userId) {
-    throw "No user ID provided for getting"
+    throw new Error("No user ID provided for getting")
   }
   return readGlobalUser(ctx)
 }

packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts

@@ -1,45 +1,49 @@
 import {
   AIOperationEnum,
   CalculationType,
+  Datasource,
   FieldType,
   RelationshipType,
-  SourceName,
   Table,
   ViewV2,
   ViewV2Type,
 } from "@budibase/types"
 import { buildSqlFieldList } from "../sqlUtils"
 import { structures } from "../../../../routes/tests/utilities"
-import { sql } from "@budibase/backend-core"
 import { generator } from "@budibase/backend-core/tests"
 import { generateViewID } from "../../../../../db/utils"
 
-import sdk from "../../../../../sdk"
-import { cloneDeep } from "lodash"
 import { utils } from "@budibase/shared-core"
+import {
+  DatabaseName,
+  datasourceDescribe,
+} from "../../../../../integrations/tests/utils"
+import { context } from "@budibase/backend-core"
 
-jest.mock("../../../../../sdk/app/views", () => ({
+const descriptions = datasourceDescribe({
-  ...jest.requireActual("../../../../../sdk/app/views"),
+  only: [DatabaseName.POSTGRES],
-  getTable: jest.fn(),
+})
-}))
-const getTableMock = sdk.views.getTable as jest.MockedFunction<
-  typeof sdk.views.getTable
->
 
-describe("buildSqlFieldList", () => {
+if (descriptions.length) {
+  describe.each(descriptions)(
+    "buildSqlFieldList ($dbName)",
+    ({ config, dsProvider }) => {
       let allTables: Record<string, Table>
+      let datasource: Datasource
+
+      beforeEach(async () => {
+        const ds = await dsProvider()
+        datasource = ds.datasource!
+        allTables = {}
+      })
 
       class TableConfig {
-    private _table: Table & { _id: string }
+        private _table: Table
 
         constructor(name: string) {
           this._table = {
-        ...structures.tableForDatasource({
+            ...structures.tableForDatasource(datasource),
-          type: "datasource",
-          source: SourceName.POSTGRES,
-        }),
             name,
-        _id: sql.utils.buildExternalTableId("ds_id", name),
             schema: {
               name: {
                 name: "name",
@@ -55,8 +59,6 @@ describe("buildSqlFieldList", () => {
               },
             },
           }
-
-      allTables[name] = this._table
         }
 
         withHiddenField(field: string) {
@@ -110,6 +112,7 @@ describe("buildSqlFieldList", () => {
             type: FieldType.LINK,
             relationshipType: RelationshipType.ONE_TO_MANY,
             fieldName: "link",
+            foreignKey: "link",
             tableId: toTableId,
           }
           return this
@@ -125,17 +128,17 @@ describe("buildSqlFieldList", () => {
           return this
         }
 
-    create() {
+        async create() {
-      return cloneDeep(this._table)
+          const table = await config.api.table.save(this._table)
+          allTables[table.name] = table
+          return table
         }
       }
 
       class ViewConfig {
-    private _table: Table
         private _view: ViewV2
 
         constructor(table: Table) {
-      this._table = table
           this._view = {
             version: 2,
             id: generateViewID(table._id!),
@@ -183,197 +186,208 @@ describe("buildSqlFieldList", () => {
           return this
         }
 
-    create() {
+        async create() {
-      getTableMock.mockResolvedValueOnce(this._table)
+          return await config.api.viewV2.create(this._view)
-      return cloneDeep(this._view)
         }
       }
 
-  beforeEach(() => {
+      const buildSqlFieldListInApp: typeof buildSqlFieldList = async (
-    jest.clearAllMocks()
+        table,
-    allTables = {}
+        allTables,
-  })
+        opts
+      ) => {
+        return context.doInAppContext(config.getAppId(), () =>
+          buildSqlFieldList(table, allTables, opts)
+        )
+      }
 
       describe("table", () => {
         it("extracts fields from table schema", async () => {
-      const table = new TableConfig("table").create()
+          const table = await new TableConfig("table").create()
-      const result = await buildSqlFieldList(table, {})
+          const result = await buildSqlFieldListInApp(table, {})
           expect(result).toEqual([
             "table.name",
             "table.description",
             "table.amount",
+            "table.id",
           ])
         })
 
         it("excludes hidden fields", async () => {
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
             .withHiddenField("description")
             .create()
-      const result = await buildSqlFieldList(table, {})
+          const result = await buildSqlFieldListInApp(table, {})
-      expect(result).toEqual(["table.name", "table.amount"])
+          expect(result).toEqual(["table.name", "table.amount", "table.id"])
         })
 
         it("excludes non-sql fields fields", async () => {
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
             .withField("formula", FieldType.FORMULA)
             .withField("ai", FieldType.AI)
-        .withRelation("link", "otherTableId")
             .create()
 
-      const result = await buildSqlFieldList(table, {})
+          const result = await buildSqlFieldListInApp(table, {})
           expect(result).toEqual([
             "table.name",
             "table.description",
             "table.amount",
+            "table.id",
           ])
         })
 
         it("includes hidden fields if there is a formula column", async () => {
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
             .withHiddenField("description")
             .withField("formula", FieldType.FORMULA)
             .create()
 
-      const result = await buildSqlFieldList(table, {})
+          const result = await buildSqlFieldListInApp(table, {})
           expect(result).toEqual([
             "table.name",
             "table.description",
             "table.amount",
+            "table.id",
           ])
         })
 
         it("includes relationships fields when flagged", async () => {
-      const otherTable = new TableConfig("linkedTable")
+          const otherTable = await new TableConfig("linkedTable")
             .withField("id", FieldType.NUMBER)
             .withPrimary("id")
             .withDisplay("name")
             .create()
 
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
-        .withRelation("link", otherTable._id)
+            .withRelation("link", otherTable._id!)
             .create()
 
-      const result = await buildSqlFieldList(table, allTables, {
+          const result = await buildSqlFieldListInApp(table, allTables, {
             relationships: true,
           })
           expect(result).toEqual([
             "table.name",
             "table.description",
             "table.amount",
+            "table.id",
             "linkedTable.id",
             "linkedTable.name",
           ])
         })
 
         it("includes all relationship fields if there is a formula column", async () => {
-      const otherTable = new TableConfig("linkedTable")
+          const otherTable = await new TableConfig("linkedTable")
             .withField("hidden", FieldType.STRING, { visible: false })
             .create()
 
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
-        .withRelation("link", otherTable._id)
+            .withRelation("link", otherTable._id!)
             .withField("formula", FieldType.FORMULA)
             .create()
 
-      const result = await buildSqlFieldList(table, allTables, {
+          const result = await buildSqlFieldListInApp(table, allTables, {
             relationships: true,
           })
           expect(result).toEqual([
             "table.name",
             "table.description",
             "table.amount",
+            "table.id",
             "linkedTable.name",
             "linkedTable.description",
             "linkedTable.amount",
             "linkedTable.hidden",
+            "linkedTable.id",
           ])
         })
 
         it("never includes non-sql columns from relationships", async () => {
-      const otherTable = new TableConfig("linkedTable")
+          const otherTable = await new TableConfig("linkedTable")
-        .withField("id", FieldType.NUMBER)
             .withField("hidden", FieldType.STRING, { visible: false })
             .withField("formula", FieldType.FORMULA)
             .withField("ai", FieldType.AI)
-        .withRelation("link", "otherTableId")
             .create()
 
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
-        .withRelation("link", otherTable._id)
+            .withRelation("link", otherTable._id!)
             .withField("formula", FieldType.FORMULA)
             .create()
 
-      const result = await buildSqlFieldList(table, allTables, {
+          const result = await buildSqlFieldListInApp(table, allTables, {
             relationships: true,
           })
           expect(result).toEqual([
             "table.name",
             "table.description",
             "table.amount",
+            "table.id",
             "linkedTable.name",
             "linkedTable.description",
             "linkedTable.amount",
-        "linkedTable.id",
             "linkedTable.hidden",
+            "linkedTable.id",
           ])
         })
       })
 
       describe("view", () => {
         it("extracts fields from table schema", async () => {
-      const view = new ViewConfig(new TableConfig("table").create())
+          const view = await new ViewConfig(
+            await new TableConfig("table").create()
+          )
             .withVisible("amount")
             .withHidden("name")
             .create()
 
-      const result = await buildSqlFieldList(view, {})
+          const result = await buildSqlFieldListInApp(view, {})
-      expect(result).toEqual(["table.amount"])
+          expect(result).toEqual(["table.amount", "table.id"])
         })
 
         it("includes all fields if there is a formula column", async () => {
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
             .withField("formula", FieldType.FORMULA)
             .create()
-      const view = new ViewConfig(table)
+          const view = await new ViewConfig(table)
             .withHidden("name")
             .withVisible("amount")
             .withVisible("formula")
             .create()
 
-      const result = await buildSqlFieldList(view, {})
+          const result = await buildSqlFieldListInApp(view, {})
           expect(result).toEqual([
             "table.name",
             "table.description",
             "table.amount",
+            "table.id",
           ])
         })
 
         it("does not includes all fields if the formula column is not included", async () => {
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
             .withField("formula", FieldType.FORMULA)
             .create()
-      const view = new ViewConfig(table)
+          const view = await new ViewConfig(table)
             .withHidden("name")
             .withVisible("amount")
             .withHidden("formula")
             .create()
 
-      const result = await buildSqlFieldList(view, {})
+          const result = await buildSqlFieldListInApp(view, {})
-      expect(result).toEqual(["table.amount"])
+          expect(result).toEqual(["table.amount", "table.id"])
         })
 
         it("includes relationships columns", async () => {
-      const otherTable = new TableConfig("linkedTable")
+          const otherTable = await new TableConfig("linkedTable")
             .withField("id", FieldType.NUMBER)
             .withField("formula", FieldType.FORMULA)
             .withPrimary("id")
             .create()
 
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
-        .withRelation("link", otherTable._id)
+            .withRelation("link", otherTable._id!)
             .create()
 
-      const view = new ViewConfig(table)
+          const view = await new ViewConfig(table)
             .withVisible("name")
             .withVisible("link")
             .withRelationshipColumns("link", {
@@ -383,51 +397,52 @@ describe("buildSqlFieldList", () => {
             })
             .create()
 
-      const result = await buildSqlFieldList(view, allTables, {
+          const result = await buildSqlFieldListInApp(view, allTables, {
             relationships: true,
           })
           expect(result).toEqual([
             "table.name",
+            "table.id",
             "linkedTable.id",
             "linkedTable.amount",
           ])
         })
 
         it("excludes relationships fields when view is not included in the view", async () => {
-      const otherTable = new TableConfig("linkedTable")
+          const otherTable = await new TableConfig("linkedTable")
             .withField("id", FieldType.NUMBER)
             .withPrimary("id")
             .withDisplay("name")
             .create()
 
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
-        .withRelation("link", otherTable._id)
+            .withRelation("link", otherTable._id!)
             .withField("formula", FieldType.FORMULA)
             .create()
 
-      const view = new ViewConfig(table)
+          const view = await new ViewConfig(table)
             .withVisible("name")
             .withHidden("amount")
             .create()
 
-      const result = await buildSqlFieldList(view, allTables, {
+          const result = await buildSqlFieldListInApp(view, allTables, {
             relationships: true,
           })
-      expect(result).toEqual(["table.name"])
+          expect(result).toEqual(["table.name", "table.id"])
         })
 
         it("does not include relationships columns for hidden links", async () => {
-      const otherTable = new TableConfig("linkedTable")
+          const otherTable = await new TableConfig("linkedTable")
             .withField("id", FieldType.NUMBER)
             .withField("formula", FieldType.FORMULA)
             .withPrimary("id")
             .create()
 
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
-        .withRelation("link", otherTable._id)
+            .withRelation("link", otherTable._id!)
             .create()
 
-      const view = new ViewConfig(table)
+          const view = await new ViewConfig(table)
             .withVisible("name")
             .withHidden("link")
             .withRelationshipColumns("link", {
@@ -437,28 +452,27 @@ describe("buildSqlFieldList", () => {
             })
             .create()
 
-      const result = await buildSqlFieldList(view, allTables, {
+          const result = await buildSqlFieldListInApp(view, allTables, {
             relationships: true,
           })
-      expect(result).toEqual(["table.name"])
+          expect(result).toEqual(["table.name", "table.id"])
         })
 
         it("includes all relationship fields if there is a formula column", async () => {
-      const otherTable = new TableConfig("linkedTable")
+          const otherTable = await new TableConfig("linkedTable")
             .withField("id", FieldType.NUMBER)
             .withField("hidden", FieldType.STRING, { visible: false })
             .withField("formula", FieldType.FORMULA)
             .withField("ai", FieldType.AI)
-        .withRelation("link", "otherTableId")
             .withPrimary("id")
             .create()
 
-      const table = new TableConfig("table")
+          const table = await new TableConfig("table")
-        .withRelation("link", otherTable._id)
+            .withRelation("link", otherTable._id!)
             .withField("formula", FieldType.FORMULA)
             .create()
 
-      const view = new ViewConfig(table)
+          const view = await new ViewConfig(table)
             .withVisible("name")
             .withVisible("formula")
             .withHidden("link")
@@ -469,13 +483,14 @@ describe("buildSqlFieldList", () => {
             })
             .create()
 
-      const result = await buildSqlFieldList(view, allTables, {
+          const result = await buildSqlFieldListInApp(view, allTables, {
             relationships: true,
           })
           expect(result).toEqual([
             "table.name",
             "table.description",
             "table.amount",
+            "table.id",
             "linkedTable.name",
             "linkedTable.description",
             "linkedTable.amount",
@@ -487,25 +502,31 @@ describe("buildSqlFieldList", () => {
 
       describe("calculation view", () => {
         it("does not include calculation fields", async () => {
-      const view = new ViewConfig(new TableConfig("table").create())
+          const view = await new ViewConfig(
+            await new TableConfig("table").create()
+          )
             .withCalculation("average", "amount", CalculationType.AVG)
 
             .create()
 
-      const result = await buildSqlFieldList(view, {})
+          const result = await buildSqlFieldListInApp(view, {})
           expect(result).toEqual([])
         })
 
         it("includes visible fields calculation fields", async () => {
-      const view = new ViewConfig(new TableConfig("table").create())
+          const view = await new ViewConfig(
+            await new TableConfig("table").create()
+          )
             .withCalculation("average", "amount", CalculationType.AVG)
             .withHidden("name")
             .withVisible("amount")
 
             .create()
 
-      const result = await buildSqlFieldList(view, {})
+          const result = await buildSqlFieldListInApp(view, {})
           expect(result).toEqual(["table.amount"])
         })
       })
-})
+    }
+  )
+}

packages/server/src/api/controllers/view/exporters.ts

@@ -26,7 +26,7 @@ export function csv(
       headers.map(header => {
         const val = row[header]
         if (typeof val === "object" && !(val instanceof Date)) {
-          return `"${JSON.stringify(val).replace(/"/g, "'")}"`
+          return `"${escapeCsvString(JSON.stringify(val))}"`
         }
         if (val !== undefined) {
           return `"${escapeCsvString(val.toString())}"`

packages/server/src/api/routes/public/index.ts

@@ -12,6 +12,7 @@ import { paramResource, paramSubResource } from "../../../middleware/resourceId"
 import { PermissionLevel, PermissionType } from "@budibase/types"
 import { CtxFn } from "./utils/Endpoint"
 import mapperMiddleware from "./middleware/mapper"
+import testErrorHandling from "./middleware/testErrorHandling"
 import env from "../../../environment"
 import { middleware, redis } from "@budibase/backend-core"
 import { SelectableDatabase } from "@budibase/backend-core/src/redis/utils"
@@ -144,6 +145,10 @@ function applyRoutes(
   // add the output mapper middleware
   addMiddleware(endpoints.read, mapperMiddleware, { output: true })
   addMiddleware(endpoints.write, mapperMiddleware, { output: true })
+  if (env.isTest()) {
+    addMiddleware(endpoints.read, testErrorHandling())
+    addMiddleware(endpoints.write, testErrorHandling())
+  }
   addToRouter(endpoints.read)
   addToRouter(endpoints.write)
 }

packages/server/src/api/routes/public/middleware/testErrorHandling.ts

@@ -0,0 +1,28 @@
+import { Ctx } from "@budibase/types"
+import environment from "../../../../environment"
+
+export default () => {
+  if (!environment.isTest()) {
+    throw new Error("This middleware is only for testing")
+  }
+
+  return async (ctx: Ctx, next: any) => {
+    try {
+      await next()
+    } catch (err: any) {
+      if (!ctx.headers["x-budibase-include-stacktrace"]) {
+        throw err
+      }
+
+      const status = err.status || err.statusCode || 500
+
+      let error = err
+      while (error.cause) {
+        error = error.cause
+      }
+
+      ctx.status = status
+      ctx.body = { status, message: error.message, stack: error.stack }
+    }
+  }
+}

packages/server/src/api/routes/public/tests/compare.spec.ts

@@ -2,11 +2,14 @@ import jestOpenAPI from "jest-openapi"
 import { run as generateSchema } from "../../../../../specs/generate"
 import * as setup from "../../tests/utilities"
 import { generateMakeRequest } from "./utils"
-import { Table, App, Row, User } from "@budibase/types"
+import { Table, App, Row } from "@budibase/types"
+import nock from "nock"
+import environment from "../../../../environment"
 
 const yamlPath = generateSchema()
 jestOpenAPI(yamlPath!)
 
+describe("compare", () => {
   let config = setup.getConfig()
   let apiKey: string, table: Table, app: App, makeRequest: any
 
@@ -19,6 +22,10 @@ beforeAll(async () => {
 
   afterAll(setup.afterAll)
 
+  beforeEach(() => {
+    nock.cleanAll()
+  })
+
   describe("check the applications endpoints", () => {
     it("should allow retrieving applications through search", async () => {
       const res = await makeRequest("post", "/applications/search")
@@ -58,6 +65,10 @@ describe("check the applications endpoints", () => {
     })
 
     it("should allow deleting an application", async () => {
+      nock(environment.WORKER_URL!)
+        .delete(`/api/global/roles/${config.getProdAppId()}`)
+        .reply(200, {})
+
       const res = await makeRequest(
         "delete",
         `/applications/${config.getAppId()}`
@@ -109,9 +120,13 @@ describe("check the rows endpoints", () => {
     let row: Row
     it("should allow retrieving rows through search", async () => {
       table = await config.upsertTable()
-    const res = await makeRequest("post", `/tables/${table._id}/rows/search`, {
+      const res = await makeRequest(
+        "post",
+        `/tables/${table._id}/rows/search`,
+        {
           query: {},
-    })
+        }
+      )
       expect(res).toSatisfyApiSpec()
     })
 
@@ -135,7 +150,10 @@ describe("check the rows endpoints", () => {
     })
 
     it("should allow retrieving a row", async () => {
-    const res = await makeRequest("get", `/tables/${table._id}/rows/${row._id}`)
+      const res = await makeRequest(
+        "get",
+        `/tables/${table._id}/rows/${row._id}`
+      )
       expect(res).toSatisfyApiSpec()
     })
 
@@ -148,38 +166,10 @@ describe("check the rows endpoints", () => {
     })
   })
 
-describe("check the users endpoints", () => {
-  let user: User
-  it("should allow retrieving users through search", async () => {
-    user = await config.createUser()
-    const res = await makeRequest("post", "/users/search")
-    expect(res).toSatisfyApiSpec()
-  })
-
-  it("should allow creating a user", async () => {
-    const res = await makeRequest("post", "/users")
-    expect(res).toSatisfyApiSpec()
-  })
-
-  it("should allow updating a user", async () => {
-    const res = await makeRequest("put", `/users/${user._id}`)
-    expect(res).toSatisfyApiSpec()
-  })
-
-  it("should allow retrieving a user", async () => {
-    const res = await makeRequest("get", `/users/${user._id}`)
-    expect(res).toSatisfyApiSpec()
-  })
-
-  it("should allow deleting a user", async () => {
-    const res = await makeRequest("delete", `/users/${user._id}`)
-    expect(res).toSatisfyApiSpec()
-  })
-})
-
   describe("check the queries endpoints", () => {
     it("should allow retrieving queries through search", async () => {
       const res = await makeRequest("post", "/queries/search")
       expect(res).toSatisfyApiSpec()
     })
   })
+})

packages/server/src/api/routes/public/tests/users.spec.ts

@@ -1,132 +1,143 @@
 import * as setup from "../../tests/utilities"
-import { generateMakeRequest, MakeRequestResponse } from "./utils"
 import { User } from "@budibase/types"
-import { mocks } from "@budibase/backend-core/tests"
+import { generator, mocks } from "@budibase/backend-core/tests"
+import nock from "nock"
+import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
+import { mockWorkerUserAPI } from "./utils"
 
-import * as workerRequests from "../../../../utilities/workerRequests"
+describe("public users API", () => {
-
+  const config = new TestConfiguration()
-const mockedWorkerReq = jest.mocked(workerRequests)
+  let globalUser: User
-
-let config = setup.getConfig()
-let apiKey: string, globalUser: User, makeRequest: MakeRequestResponse
 
   beforeAll(async () => {
     await config.init()
-  globalUser = await config.globalUser()
-  apiKey = await config.generateApiKey(globalUser._id)
-  makeRequest = generateMakeRequest(apiKey)
-  mockedWorkerReq.readGlobalUser.mockImplementation(() =>
-    Promise.resolve(globalUser)
-  )
   })
 
   afterAll(setup.afterAll)
 
-function base() {
+  beforeEach(async () => {
-  return {
+    globalUser = await config.globalUser()
-    tenantId: config.getTenantId(),
-    firstName: "Test",
-    lastName: "Test",
-  }
-}
 
-function updateMock() {
+    nock.cleanAll()
-  mockedWorkerReq.readGlobalUser.mockImplementation(ctx => ctx.request.body)
+    mockWorkerUserAPI(globalUser)
-}
-
-describe("check user endpoints", () => {
-  it("should not allow a user to update their own roles", async () => {
-    const res = await makeRequest("put", `/users/${globalUser._id}`, {
-      ...globalUser,
-      roles: {
-        app_1: "ADMIN",
-      },
-    })
-    expect(
-      mockedWorkerReq.saveGlobalUser.mock.lastCall?.[0].body.data.roles["app_1"]
-    ).toBeUndefined()
-    expect(res.status).toBe(200)
-    expect(res.body.data.roles["app_1"]).toBeUndefined()
   })
 
-  it("should not allow a user to delete themselves", async () => {
+  describe("read", () => {
-    const res = await makeRequest("delete", `/users/${globalUser._id}`)
+    it("should allow a user to read themselves", async () => {
-    expect(res.status).toBe(405)
+      const user = await config.api.user.find(globalUser._id!)
-    expect(mockedWorkerReq.deleteGlobalUser.mock.lastCall).toBeUndefined()
+      expect(user._id).toBe(globalUser._id)
+    })
+
+    it("should allow a user to read another user", async () => {
+      const otherUser = await config.api.public.user.create({
+        email: generator.email({ domain: "example.com" }),
+        roles: {},
+      })
+      const user = await config.withUser(globalUser, () =>
+        config.api.public.user.find(otherUser._id!)
+      )
+      expect(user._id).toBe(otherUser._id)
     })
   })
 
-describe("no user role update in free", () => {
+  describe("create", () => {
-  beforeAll(() => {
+    it("can successfully create a new user", async () => {
-    updateMock()
+      const email = generator.email({ domain: "example.com" })
+      const newUser = await config.api.public.user.create({
+        email,
+        roles: {},
+      })
+      expect(newUser.email).toBe(email)
+      expect(newUser._id).toBeDefined()
     })
 
+    describe("role creation on free tier", () => {
       it("should not allow 'roles' to be updated", async () => {
-    const res = await makeRequest("post", "/users", {
+        const newUser = await config.api.public.user.create({
-      ...base(),
+          email: generator.email({ domain: "example.com" }),
           roles: { app_a: "BASIC" },
         })
-    expect(res.status).toBe(200)
+        expect(newUser.roles["app_a"]).toBeUndefined()
-    expect(res.body.data.roles["app_a"]).toBeUndefined()
-    expect(res.body.message).toBeDefined()
       })
 
       it("should not allow 'admin' to be updated", async () => {
-    const res = await makeRequest("post", "/users", {
+        const newUser = await config.api.public.user.create({
-      ...base(),
+          email: generator.email({ domain: "example.com" }),
+          roles: {},
           admin: { global: true },
         })
-    expect(res.status).toBe(200)
+        expect(newUser.admin).toBeUndefined()
-    expect(res.body.data.admin).toBeUndefined()
-    expect(res.body.message).toBeDefined()
       })
 
       it("should not allow 'builder' to be updated", async () => {
-    const res = await makeRequest("post", "/users", {
+        const newUser = await config.api.public.user.create({
-      ...base(),
+          email: generator.email({ domain: "example.com" }),
+          roles: {},
           builder: { global: true },
         })
-    expect(res.status).toBe(200)
+        expect(newUser.builder).toBeUndefined()
-    expect(res.body.data.builder).toBeUndefined()
-    expect(res.body.message).toBeDefined()
       })
     })
 
-describe("no user role update in business", () => {
+    describe("role creation on business tier", () => {
       beforeAll(() => {
-    updateMock()
         mocks.licenses.useExpandedPublicApi()
       })
 
       it("should allow 'roles' to be updated", async () => {
-    const res = await makeRequest("post", "/users", {
+        const newUser = await config.api.public.user.create({
-      ...base(),
+          email: generator.email({ domain: "example.com" }),
           roles: { app_a: "BASIC" },
         })
-    expect(res.status).toBe(200)
+        expect(newUser.roles["app_a"]).toBe("BASIC")
-    expect(res.body.data.roles["app_a"]).toBe("BASIC")
-    expect(res.body.message).toBeUndefined()
       })
 
       it("should allow 'admin' to be updated", async () => {
-    mocks.licenses.useExpandedPublicApi()
+        const newUser = await config.api.public.user.create({
-    const res = await makeRequest("post", "/users", {
+          email: generator.email({ domain: "example.com" }),
-      ...base(),
+          roles: {},
           admin: { global: true },
         })
-    expect(res.status).toBe(200)
+        expect(newUser.admin?.global).toBe(true)
-    expect(res.body.data.admin.global).toBe(true)
-    expect(res.body.message).toBeUndefined()
       })
 
       it("should allow 'builder' to be updated", async () => {
-    mocks.licenses.useExpandedPublicApi()
+        const newUser = await config.api.public.user.create({
-    const res = await makeRequest("post", "/users", {
+          email: generator.email({ domain: "example.com" }),
-      ...base(),
+          roles: {},
           builder: { global: true },
         })
-    expect(res.status).toBe(200)
+        expect(newUser.builder?.global).toBe(true)
-    expect(res.body.data.builder.global).toBe(true)
+      })
-    expect(res.body.message).toBeUndefined()
+    })
+  })
+
+  describe("update", () => {
+    it("can update a user", async () => {
+      const updatedUser = await config.api.public.user.update({
+        ...globalUser,
+        email: `updated-${globalUser.email}`,
+      })
+      expect(updatedUser.email).toBe(`updated-${globalUser.email}`)
+    })
+
+    it("should not allow a user to update their own roles", async () => {
+      await config.withUser(globalUser, () =>
+        config.api.public.user.update({
+          ...globalUser,
+          roles: { app_1: "ADMIN" },
+        })
+      )
+      const updatedUser = await config.api.user.find(globalUser._id!)
+      expect(updatedUser.roles?.app_1).toBeUndefined()
+    })
+  })
+
+  describe("delete", () => {
+    it("should not allow a user to delete themselves", async () => {
+      await config.withUser(globalUser, () =>
+        config.api.public.user.destroy(globalUser._id!, { status: 405 })
+      )
+    })
   })
 })