Merge branch 'master' into borderless-grid

.github/workflows/budibase_ci.yml

@@ -92,7 +92,6 @@ jobs:
   test-libraries:
     runs-on: ubuntu-latest
     env:
-      DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull
       REUSE_CONTAINERS: true
     steps:
       - name: Checkout repo
@@ -110,7 +109,7 @@ jobs:
       - name: Pull testcontainers images
         run: |
           docker pull testcontainers/ryuk:0.5.1 &
-          docker pull budibase/couchdb &
+          docker pull budibase/couchdb:v3.2.1-sql &
           docker pull redis &
 
           wait $(jobs -p)
@@ -151,7 +150,6 @@ jobs:
   test-server:
     runs-on: budi-tubby-tornado-quad-core-150gb
     env:
-      DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull
       REUSE_CONTAINERS: true
     steps:
       - name: Checkout repo
@@ -175,7 +173,7 @@ jobs:
           docker pull mongo:7.0-jammy &
           docker pull mariadb:lts &
           docker pull testcontainers/ryuk:0.5.1 &
-          docker pull budibase/couchdb &
+          docker pull budibase/couchdb:v3.2.1-sql &
           docker pull redis &
 
           wait $(jobs -p)

globalSetup.ts

@@ -13,8 +13,8 @@ export default async function setup() {
   }
 
   try {
-    let couchdb = new GenericContainer("budibase/couchdb")
-      .withExposedPorts(5984)
+    let couchdb = new GenericContainer("budibase/couchdb:v3.2.1-sqs")
+      .withExposedPorts(5984, 4984)
       .withEnvironment({
         COUCHDB_PASSWORD: "budibase",
         COUCHDB_USER: "budibase",

hosting/couchdb/Dockerfile

@@ -128,4 +128,4 @@ ADD couch/vm.args couch/local.ini ./etc/
 WORKDIR /
 ADD runner.sh ./bbcouch-runner.sh
 RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau
-CMD ["./bbcouch-runner.sh"]
+CMD ["./bbcouch-runner.sh"]

hosting/couchdb/Dockerfile.v2

@@ -0,0 +1,139 @@
+# Modified from https://github.com/apache/couchdb-docker/blob/main/3.3.3/Dockerfile
+#
+# Everything in this `base` image is adapted from the official `couchdb` image's
+# Dockerfile. Only modifications related to upgrading from Debian bullseye to
+# bookworm have been included. The `runner` image contains Budibase's
+# customisations to the image, e.g. adding Clouseau.
+FROM node:20-slim AS base
+
+# Add CouchDB user account to make sure the IDs are assigned consistently
+RUN groupadd -g 5984 -r couchdb && useradd -u 5984 -d /opt/couchdb -g couchdb couchdb
+
+# be sure GPG and apt-transport-https are available and functional
+RUN set -ex; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+        apt-transport-https \
+        ca-certificates \
+        dirmngr \
+        gnupg \
+     ; \
+    rm -rf /var/lib/apt/lists/*
+
+# grab tini for signal handling and zombie reaping
+# see https://github.com/apache/couchdb-docker/pull/28#discussion_r141112407
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends tini; \
+    rm -rf /var/lib/apt/lists/*; \
+    tini --version
+
+# http://docs.couchdb.org/en/latest/install/unix.html#installing-the-apache-couchdb-packages
+ENV GPG_COUCH_KEY \
+# gpg: rsa8192 205-01-19 The Apache Software Foundation (Package repository signing key) <root@apache.org>
+    390EF70BB1EA12B2773962950EE62FB37A00258D
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y curl; \
+    export GNUPGHOME="$(mktemp -d)"; \
+    curl -fL -o keys.asc https://couchdb.apache.org/repo/keys.asc; \
+    gpg --batch --import keys.asc; \
+    gpg --batch --export "${GPG_COUCH_KEY}" > /usr/share/keyrings/couchdb-archive-keyring.gpg; \
+    command -v gpgconf && gpgconf --kill all || :; \
+    rm -rf "$GNUPGHOME"; \
+    apt-key list; \
+    apt purge -y --autoremove curl; \
+    rm -rf /var/lib/apt/lists/*
+
+ENV COUCHDB_VERSION 3.3.3
+
+RUN . /etc/os-release; \
+    echo "deb [signed-by=/usr/share/keyrings/couchdb-archive-keyring.gpg] https://apache.jfrog.io/artifactory/couchdb-deb/ ${VERSION_CODENAME} main" | \
+        tee /etc/apt/sources.list.d/couchdb.list >/dev/null
+
+# https://github.com/apache/couchdb-pkg/blob/master/debian/README.Debian
+RUN set -eux; \
+    apt-get update; \
+    \
+    echo "couchdb couchdb/mode select none" | debconf-set-selections; \
+# we DO want recommends this time
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \
+            couchdb="$COUCHDB_VERSION"~bookworm \
+    ; \
+# Undo symlinks to /var/log and /var/lib
+    rmdir /var/lib/couchdb /var/log/couchdb; \
+    rm /opt/couchdb/data /opt/couchdb/var/log; \
+    mkdir -p /opt/couchdb/data /opt/couchdb/var/log; \
+    chown couchdb:couchdb /opt/couchdb/data /opt/couchdb/var/log; \
+    chmod 777 /opt/couchdb/data /opt/couchdb/var/log; \
+# Remove file that sets logging to a file
+    rm /opt/couchdb/etc/default.d/10-filelog.ini; \
+# Check we own everything in /opt/couchdb. Matches the command in dockerfile_entrypoint.sh
+    find /opt/couchdb \! \( -user couchdb -group couchdb \) -exec chown -f couchdb:couchdb '{}' +; \
+# Setup directories and permissions for config. Technically these could be 555 and 444 respectively
+# but we keep them as 755 and 644 for consistency with CouchDB defaults and the dockerfile_entrypoint.sh.
+    find /opt/couchdb/etc -type d ! -perm 0755 -exec chmod -f 0755 '{}' +; \
+    find /opt/couchdb/etc -type f ! -perm 0644 -exec chmod -f 0644 '{}' +; \
+# only local.d needs to be writable for the docker_entrypoint.sh
+    chmod -f 0777 /opt/couchdb/etc/local.d; \
+# apt clean-up
+    rm -rf /var/lib/apt/lists/*;
+
+# Add configuration
+COPY --chown=couchdb:couchdb couch/10-docker-default.ini /opt/couchdb/etc/default.d/
+# COPY --chown=couchdb:couchdb vm.args /opt/couchdb/etc/
+
+COPY docker-entrypoint.sh /usr/local/bin
+RUN ln -s usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh # backwards compat
+ENTRYPOINT ["tini", "--", "/docker-entrypoint.sh"]
+
+VOLUME /opt/couchdb/data
+
+# 5984: Main CouchDB endpoint
+# 4369: Erlang portmap daemon (epmd)
+# 9100: CouchDB cluster communication port
+EXPOSE 5984 4369 9100
+CMD ["/opt/couchdb/bin/couchdb"]
+
+FROM base as runner
+ARG TARGETARCH
+ENV TARGETARCH $TARGETARCH
+
+ENV COUCHDB_USER admin
+ENV COUCHDB_PASSWORD admin
+EXPOSE 5984
+EXPOSE 4984
+
+RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \
+    wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - && \
+    apt-add-repository 'deb http://security.debian.org/debian-security bookworm-security/updates main' && \
+    apt-add-repository 'deb http://archive.debian.org/debian stretch-backports main' && \
+    apt-add-repository 'deb https://packages.adoptium.net/artifactory/deb bookworm main' && \
+    apt-get update && apt-get install -y --no-install-recommends temurin-8-jdk && \
+    rm -rf /var/lib/apt/lists/
+
+# setup clouseau
+WORKDIR /
+RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clouseau-2.21.0-dist.zip && \
+  unzip clouseau-2.21.0-dist.zip && \
+  mv clouseau-2.21.0 /opt/clouseau && \
+  rm clouseau-2.21.0-dist.zip
+
+WORKDIR /opt/clouseau
+RUN mkdir ./bin
+ADD clouseau/clouseau ./bin/
+ADD clouseau/log4j.properties clouseau/clouseau.ini ./
+
+# setup CouchDB
+WORKDIR /opt/couchdb
+ADD couch/vm.args couch/local.ini ./etc/
+
+# setup SQS
+WORKDIR /opt/sqs
+ADD sqs ./
+RUN chmod +x ./install.sh && ./install.sh
+
+WORKDIR /
+ADD runner.v2.sh ./bbcouch-runner.sh
+RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau /opt/sqs/sqs
+CMD ["./bbcouch-runner.sh"]

hosting/couchdb/runner.v2.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+DATA_DIR=${DATA_DIR:-/data}
+COUCHDB_ERLANG_COOKIE=${COUCHDB_ERLANG_COOKIE:-B9CFC32C-3458-4A86-8448-B3C753991CA7}
+
+mkdir -p ${DATA_DIR}
+mkdir -p ${DATA_DIR}/couch/{dbs,views}
+mkdir -p ${DATA_DIR}/search
+chown -R couchdb:couchdb ${DATA_DIR}/couch
+
+echo ${TARGETBUILD} > /buildtarget.txt
+if [[ "${TARGETBUILD}" = "aas" ]]; then
+    # Azure AppService uses /home for persistent data & SSH on port 2222
+    DATA_DIR="${DATA_DIR:-/home}"
+    WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
+    mkdir -p $DATA_DIR/{search,minio,couch}
+    mkdir -p $DATA_DIR/couch/{dbs,views}
+    chown -R couchdb:couchdb $DATA_DIR/couch/
+    apt update
+    apt-get install -y openssh-server
+    echo "root:Docker!" | chpasswd
+    mkdir -p /tmp
+    chmod +x /tmp/ssh_setup.sh \
+        && (sleep 1;/tmp/ssh_setup.sh 2>&1 > /dev/null)
+    cp /etc/sshd_config /etc/ssh/sshd_config
+    /etc/init.d/ssh restart
+    sed -i "s#DATA_DIR#/home#g" /opt/clouseau/clouseau.ini
+    sed -i "s#DATA_DIR#/home#g" /opt/couchdb/etc/local.ini
+elif [[ "${TARGETBUILD}" = "single" ]]; then
+    # In the single image build, the Dockerfile specifies /data as a volume
+    # mount, so we use that for all persistent data.
+    sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
+    sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
+elif [[ "${TARGETBUILD}" = "docker-compose" ]]; then
+    # We remove the database_dir and view_index_dir settings from the local.ini
+    # in docker-compose because it will default to /opt/couchdb/data which is what
+    # our docker-compose was using prior to us switching to using our own CouchDB
+    # image.
+    sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini
+    sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini
+    sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini
+elif [[ -n $KUBERNETES_SERVICE_HOST ]]; then
+    # In Kubernetes the directory /opt/couchdb/data has a persistent volume
+    # mount for storing database data.
+    sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini
+
+    # We remove the database_dir and view_index_dir settings from the local.ini
+    # in Kubernetes because it will default to /opt/couchdb/data which is what
+    # our Helm chart was using prior to us switching to using our own CouchDB
+    # image.
+    sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini
+    sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini
+
+    # We remove the -name setting from the vm.args file in Kubernetes because
+    # it will default to the pod FQDN, which is what's required for clustering
+    # to work.
+    sed -i "s/^-name .*$//g" /opt/couchdb/etc/vm.args
+else
+    # For all other builds, we use /data for persistent data.
+    sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
+    sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
+fi
+
+sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/couchdb/etc/vm.args
+sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/clouseau/clouseau.ini
+
+# Start Clouseau. Budibase won't function correctly without Clouseau running, it
+# powers the search API endpoints which are used to do all sorts, including
+# populating app grids.
+/opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
+
+# Start CouchDB.
+/docker-entrypoint.sh /opt/couchdb/bin/couchdb &
+
+# Start SQS.
+/opt/sqs/sqs --server "http://localhost:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 &
+
+# Wait for CouchDB to start up.
+while [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; do
+    echo 'Waiting for CouchDB to start...';
+    sleep 5;
+done
+
+# CouchDB needs the `_users` and `_replicator` databases to exist before it will
+# function correctly, so we create them here.
+curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_users
+curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_replicator
+sleep infinity

hosting/couchdb/sqs/install.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )"
+if [[ $TARGETARCH == arm* ]] ;
+then
+  echo "Installing ARM SQS Client..."
+  mv $SCRIPT_DIR/arm/* .
+  rm -r $SCRIPT_DIR/arm
+  rm -r $SCRIPT_DIR/x86
+else
+  echo "Installing x86-64 SQS Client..."
+  mv $SCRIPT_DIR/x86/* .
+  rm -r $SCRIPT_DIR/arm
+  rm -r $SCRIPT_DIR/x86
+fi

hosting/docker-compose.dev.yaml

@@ -40,7 +40,6 @@ services:
       - PROXY_ADDRESS=host.docker.internal
 
   couchdb-service:
-    # platform: linux/amd64
     container_name: budi-couchdb3-dev
     restart: on-failure
     image: budibase/couchdb

hosting/proxy/nginx.prod.conf

@@ -51,11 +51,11 @@ http {
     proxy_buffering off;
 
     set $csp_default "default-src 'self'";
-    set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io https://d2l5prqdbvm3op.cloudfront.net";
+    set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://*.budibase.net https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io https://d2l5prqdbvm3op.cloudfront.net https://us-assets.i.posthog.com";
     set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
     set $csp_object "object-src 'none'";
     set $csp_base_uri "base-uri 'self'";
-    set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibaseqa.app https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com";    
+    set $csp_connect "connect-src 'self' https://*.budibase.app https://*.budibaseqa.app https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com https://us.i.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com";    
     set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com  https://js.intercomcdn.com  https://fonts.intercomcdn.com";
     set $csp_frame "frame-src 'self' https:";
     set $csp_img "img-src http: https: data: blob:";

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.22.18",
+  "version": "2.23.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

package.json

@@ -59,7 +59,7 @@
     "dev:all": "yarn run kill-all && lerna run --stream dev",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
     "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
-    "test": "lerna run --stream test --stream",
+    "test": "REUSE_CONTAINERS=1 lerna run --concurrency 1 --stream test --stream",
     "lint:eslint": "eslint packages --max-warnings=0",
     "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"",
     "lint": "yarn run lint:eslint && yarn run lint:prettier",
@@ -74,6 +74,7 @@
     "build:docker:single": "./scripts/build-single-image.sh",
     "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting",
     "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.2.1 --push ./hosting/couchdb",
+    "publish:docker:couch-sqs": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile.v2 -t budibase/couchdb:v3.2.1-sqs --push ./hosting/couchdb",
     "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting",
     "release:helm": "node scripts/releaseHelmChart",
     "env:multi:enable": "lerna run --stream env:multi:enable",

packages/backend-core/src/constants/db.ts

@@ -66,3 +66,4 @@ export const APP_PREFIX = prefixed(DocumentType.APP)
 export const APP_DEV = prefixed(DocumentType.APP_DEV)
 export const APP_DEV_PREFIX = APP_DEV
 export const BUDIBASE_DATASOURCE_TYPE = "budibase"
+export const SQLITE_DESIGN_DOC_ID = "_design/sqlite"

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -12,12 +12,14 @@ import {
   isDocument,
   RowResponse,
   RowValue,
+  SqlQueryBinding,
 } from "@budibase/types"
 import { getCouchInfo } from "./connections"
 import { directCouchUrlCall } from "./utils"
 import { getPouchDB } from "./pouchDB"
 import { WriteStream, ReadStream } from "fs"
 import { newid } from "../../docIds/newid"
+import { SQLITE_DESIGN_DOC_ID } from "../../constants"
 import { DDInstrumentedDatabase } from "../instrumentation"
 
 const DATABASE_NOT_FOUND = "Database does not exist."
@@ -247,6 +249,27 @@ export class DatabaseImpl implements Database {
     })
   }
 
+  async sql<T extends Document>(
+    sql: string,
+    parameters?: SqlQueryBinding
+  ): Promise<T[]> {
+    const dbName = this.name
+    const url = `/${dbName}/${SQLITE_DESIGN_DOC_ID}`
+    const response = await directCouchUrlCall({
+      url: `${this.couchInfo.sqlUrl}/${url}`,
+      method: "POST",
+      cookie: this.couchInfo.cookie,
+      body: {
+        query: sql,
+        args: parameters,
+      },
+    })
+    if (response.status > 300) {
+      throw new Error(await response.text())
+    }
+    return (await response.json()) as T[]
+  }
+
   async query<T extends Document>(
     viewName: string,
     params: DatabaseQueryOpts

packages/backend-core/src/db/couch/connections.ts

@@ -25,6 +25,7 @@ export const getCouchInfo = (connection?: string) => {
   const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
   return {
     url: urlInfo.url!,
+    sqlUrl: env.COUCH_DB_SQL_URL,
     auth: {
       username: username,
       password: password,

packages/backend-core/src/db/couch/utils.ts

@@ -30,8 +30,13 @@ export async function directCouchUrlCall({
     },
   }
   if (body && method !== "GET") {
-    params.body = JSON.stringify(body)
-    params.headers["Content-Type"] = "application/json"
+    if (typeof body === "string") {
+      params.body = body
+      params.headers["Content-Type"] = "text/plain"
+    } else {
+      params.body = JSON.stringify(body)
+      params.headers["Content-Type"] = "application/json"
+    }
   }
   return await fetch(checkSlashesInUrl(encodeURI(url)), params)
 }

packages/backend-core/src/db/instrumentation.ts

@@ -13,6 +13,7 @@ import {
   DatabaseQueryOpts,
   Document,
   RowValue,
+  SqlQueryBinding,
 } from "@budibase/types"
 import tracer from "dd-trace"
 import { Writable } from "stream"
@@ -149,4 +150,14 @@ export class DDInstrumentedDatabase implements Database {
       return this.db.getIndexes(...args)
     })
   }
+
+  sql<T extends Document>(
+    sql: string,
+    parameters?: SqlQueryBinding
+  ): Promise<T[]> {
+    return tracer.trace("db.sql", span => {
+      span?.addTags({ db_name: this.name })
+      return this.db.sql(sql, parameters)
+    })
+  }
 }

packages/backend-core/src/db/lucene.ts

@@ -1,28 +1,16 @@
 import fetch from "node-fetch"
 import { getCouchInfo } from "./couch"
-import { SearchFilters, Row, EmptyFilterOption } from "@budibase/types"
+import {
+  SearchFilters,
+  Row,
+  EmptyFilterOption,
+  SearchResponse,
+  SearchParams,
+  WithRequired,
+} from "@budibase/types"
 
 const QUERY_START_REGEX = /\d[0-9]*:/g
 
-interface SearchResponse<T> {
-  rows: T[] | any[]
-  bookmark?: string
-  totalRows: number
-}
-
-export type SearchParams<T> = {
-  tableId?: string
-  sort?: string
-  sortOrder?: string
-  sortType?: string
-  limit?: number
-  bookmark?: string
-  version?: string
-  indexer?: () => Promise<any>
-  disableEscaping?: boolean
-  rows?: T | Row[]
-}
-
 export function removeKeyNumbering(key: any): string {
   if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) {
     const parts = key.split(":")
@@ -44,7 +32,7 @@ export class QueryBuilder<T> {
   #query: SearchFilters
   #limit: number
   #sort?: string
-  #bookmark?: string
+  #bookmark?: string | number
   #sortOrder: string
   #sortType: string
   #includeDocs: boolean
@@ -130,7 +118,7 @@ export class QueryBuilder<T> {
     return this
   }
 
-  setBookmark(bookmark?: string) {
+  setBookmark(bookmark?: string | number) {
     if (bookmark != null) {
       this.#bookmark = bookmark
     }
@@ -226,14 +214,20 @@ export class QueryBuilder<T> {
     }
   }
 
-  /**
-   * Preprocesses a value before going into a lucene search.
-   * Transforms strings to lowercase and wraps strings and bools in quotes.
-   * @param value The value to process
-   * @param options The preprocess options
-   * @returns {string|*}
-   */
-  preprocess(value: any, { escape, lowercase, wrap, type }: any = {}) {
+  preprocess(
+    value: any,
+    {
+      escape,
+      lowercase,
+      wrap,
+      type,
+    }: {
+      escape?: boolean
+      lowercase?: boolean
+      wrap?: boolean
+      type?: string
+    } = {}
+  ): string | any {
     const hasVersion = !!this.#version
     // Determine if type needs wrapped
     const originalType = typeof value
@@ -561,7 +555,7 @@ async function runQuery<T>(
   url: string,
   body: any,
   cookie: string
-): Promise<SearchResponse<T>> {
+): Promise<WithRequired<SearchResponse<T>, "totalRows">> {
   const response = await fetch(url, {
     body: JSON.stringify(body),
     method: "POST",
@@ -575,7 +569,7 @@ async function runQuery<T>(
   }
   const json = await response.json()
 
-  let output: SearchResponse<T> = {
+  let output: WithRequired<SearchResponse<T>, "totalRows"> = {
     rows: [],
     totalRows: 0,
   }
@@ -613,63 +607,51 @@ async function recursiveSearch<T>(
   dbName: string,
   index: string,
   query: any,
-  params: any
+  params: SearchParams
 ): Promise<any> {
   const bookmark = params.bookmark
   const rows = params.rows || []
-  if (rows.length >= params.limit) {
+  if (params.limit && rows.length >= params.limit) {
     return rows
   }
   let pageSize = QueryBuilder.maxLimit
-  if (rows.length > params.limit - QueryBuilder.maxLimit) {
+  if (params.limit && rows.length > params.limit - QueryBuilder.maxLimit) {
     pageSize = params.limit - rows.length
   }
-  const page = await new QueryBuilder<T>(dbName, index, query)
+  const queryBuilder = new QueryBuilder<T>(dbName, index, query)
+  queryBuilder
     .setVersion(params.version)
-    .setTable(params.tableId)
     .setBookmark(bookmark)
     .setLimit(pageSize)
     .setSort(params.sort)
     .setSortOrder(params.sortOrder)
     .setSortType(params.sortType)
-    .run()
+
+  if (params.tableId) {
+    queryBuilder.setTable(params.tableId)
+  }
+
+  const page = await queryBuilder.run()
   if (!page.rows.length) {
     return rows
   }
   if (page.rows.length < QueryBuilder.maxLimit) {
     return [...rows, ...page.rows]
   }
-  const newParams = {
+  const newParams: SearchParams = {
     ...params,
     bookmark: page.bookmark,
-    rows: [...rows, ...page.rows],
+    rows: [...rows, ...page.rows] as Row[],
   }
   return await recursiveSearch(dbName, index, query, newParams)
 }
 
-/**
- * Performs a paginated search. A bookmark will be returned to allow the next
- * page to be fetched. There is a max limit off 200 results per page in a
- * paginated search.
- * @param dbName Which database to run a lucene query on
- * @param index Which search index to utilise
- * @param query The JSON query structure
- * @param params The search params including:
- *   tableId {string} The table ID to search
- *   sort {string} The sort column
- *   sortOrder {string} The sort order ("ascending" or "descending")
- *   sortType {string} Whether to treat sortable values as strings or
- *     numbers. ("string" or "number")
- *   limit {number} The desired page size
- *   bookmark {string} The bookmark to resume from
- * @returns {Promise<{hasNextPage: boolean, rows: *[]}>}
- */
 export async function paginatedSearch<T>(
   dbName: string,
   index: string,
   query: SearchFilters,
-  params: SearchParams<T>
-) {
+  params: SearchParams
+): Promise<SearchResponse<T>> {
   let limit = params.limit
   if (limit == null || isNaN(limit) || limit < 0) {
     limit = 50
@@ -713,29 +695,12 @@ export async function paginatedSearch<T>(
   }
 }
 
-/**
- * Performs a full search, fetching multiple pages if required to return the
- * desired amount of results. There is a limit of 1000 results to avoid
- * heavy performance hits, and to avoid client components breaking from
- * handling too much data.
- * @param dbName Which database to run a lucene query on
- * @param index Which search index to utilise
- * @param query The JSON query structure
- * @param params The search params including:
- *   tableId {string} The table ID to search
- *   sort {string} The sort column
- *   sortOrder {string} The sort order ("ascending" or "descending")
- *   sortType {string} Whether to treat sortable values as strings or
- *     numbers. ("string" or "number")
- *   limit {number} The desired number of results
- * @returns {Promise<{rows: *}>}
- */
 export async function fullSearch<T>(
   dbName: string,
   index: string,
   query: SearchFilters,
-  params: SearchParams<T>
-) {
+  params: SearchParams
+): Promise<{ rows: Row[] }> {
   let limit = params.limit
   if (limit == null || isNaN(limit) || limit < 0) {
     limit = 1000

packages/backend-core/src/db/tests/lucene.spec.ts

@@ -1,23 +1,39 @@
 import { newid } from "../../docIds/newid"
 import { getDB } from "../db"
-import { Database, EmptyFilterOption } from "@budibase/types"
-import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene"
+import {
+  Database,
+  EmptyFilterOption,
+  SortOrder,
+  SortType,
+  DocumentType,
+  SEPARATOR,
+} from "@budibase/types"
+import { fullSearch, paginatedSearch, QueryBuilder } from "../lucene"
 
 const INDEX_NAME = "main"
+const TABLE_ID = DocumentType.TABLE + SEPARATOR + newid()
 
 const index = `function(doc) {
-  let props = ["property", "number", "array"]
-  for (let key of props) {
-    if (Array.isArray(doc[key])) {
-      for (let val of doc[key]) {
+  if (!doc._id.startsWith("ro_")) {
+    return
+  }
+  let keys = Object.keys(doc).filter(key => !key.startsWith("_"))
+  for (let key of keys) {
+    const value = doc[key]
+    if (Array.isArray(value)) {
+      for (let val of value) {
         index(key, val)
       }
-    } else if (doc[key]) {
-      index(key, doc[key])
+    } else if (value) {
+      index(key, value)
     }
   }
 }`
 
+function rowId(id?: string) {
+  return DocumentType.ROW + SEPARATOR + (id || newid())
+}
+
 describe("lucene", () => {
   let db: Database, dbName: string
 
@@ -25,10 +41,21 @@ describe("lucene", () => {
     dbName = `db-${newid()}`
     // create the DB for testing
     db = getDB(dbName)
-    await db.put({ _id: newid(), property: "word", array: ["1", "4"] })
-    await db.put({ _id: newid(), property: "word2", array: ["3", "1"] })
     await db.put({
-      _id: newid(),
+      _id: rowId(),
+      tableId: TABLE_ID,
+      property: "word",
+      array: ["1", "4"],
+    })
+    await db.put({
+      _id: rowId(),
+      tableId: TABLE_ID,
+      property: "word2",
+      array: ["3", "1"],
+    })
+    await db.put({
+      _id: rowId(),
+      tableId: TABLE_ID,
       property: "word3",
       number: 1,
       array: ["1", "2"],
@@ -240,7 +267,8 @@ describe("lucene", () => {
         docs = Array(QueryBuilder.maxLimit * 2.5)
           .fill(0)
           .map((_, i) => ({
-            _id: i.toString().padStart(3, "0"),
+            _id: rowId(i.toString().padStart(3, "0")),
+            tableId: TABLE_ID,
             property: `value_${i.toString().padStart(3, "0")}`,
             array: [],
           }))
@@ -338,10 +366,11 @@ describe("lucene", () => {
           },
         },
         {
+          tableId: TABLE_ID,
           limit: 1,
           sort: "property",
-          sortType: "string",
-          sortOrder: "desc",
+          sortType: SortType.STRING,
+          sortOrder: SortOrder.DESCENDING,
         }
       )
       expect(page.rows.length).toBe(1)
@@ -360,7 +389,10 @@ describe("lucene", () => {
             property: "wo",
           },
         },
-        {}
+        {
+          tableId: TABLE_ID,
+          query: {},
+        }
       )
       expect(page.rows.length).toBe(3)
     })

packages/backend-core/src/index.ts

@@ -32,7 +32,6 @@ export { default as env } from "./environment"
 export * as blacklist from "./blacklist"
 export * as docUpdates from "./docUpdates"
 export * from "./utils/Duration"
-export { SearchParams } from "./db"
 export * as docIds from "./docIds"
 export * as security from "./security"
 // Add context to tenancy for backwards compatibility

packages/backend-core/tests/core/utilities/testContainerUtils.ts

@@ -77,9 +77,15 @@ export function setupEnv(...envs: any[]) {
     throw new Error("CouchDB port not found")
   }
 
+  const couchSqlPort = getExposedV4Port(couch, 4984)
+  if (!couchSqlPort) {
+    throw new Error("CouchDB SQL port not found")
+  }
+
   const configs = [
     { key: "COUCH_DB_PORT", value: `${couchPort}` },
     { key: "COUCH_DB_URL", value: `http://127.0.0.1:${couchPort}` },
+    { key: "COUCH_DB_SQL_URL", value: `http://127.0.0.1:${couchSqlPort}` },
   ]
 
   for (const config of configs.filter(x => !!x.value)) {

packages/bbui/src/Form/Core/Combobox.svelte

@@ -4,6 +4,7 @@
   import "@spectrum-css/menu/dist/index-vars.css"
   import { createEventDispatcher } from "svelte"
   import clickOutside from "../../Actions/click_outside"
+  import Popover from "../../Popover/Popover.svelte"
 
   export let value = null
   export let id = null
@@ -15,8 +16,10 @@
   export let getOptionValue = option => option
 
   const dispatch = createEventDispatcher()
+
   let open = false
   let focus = false
+  let anchor
 
   const selectOption = value => {
     dispatch("change", value)
@@ -35,11 +38,11 @@
   }
 </script>
 
-<!-- svelte-ignore a11y-click-events-have-key-events -->
 <div
   class="spectrum-InputGroup"
   class:is-focused={open || focus}
   class:is-disabled={disabled}
+  bind:this={anchor}
 >
   <div
     class="spectrum-Textfield spectrum-InputGroup-textfield"
@@ -67,7 +70,7 @@
     tabindex="-1"
     aria-haspopup="true"
     {disabled}
-    on:click={() => (open = true)}
+    on:click={() => (open = !open)}
   >
     <svg
       class="spectrum-Icon spectrum-UIIcon-ChevronDown100 spectrum-Picker-menuIcon spectrum-InputGroup-icon"
@@ -77,42 +80,44 @@
       <use xlink:href="#spectrum-css-icon-Chevron100" />
     </svg>
   </button>
-  {#if open}
-    <div
-      class="spectrum-Popover spectrum-Popover--bottom is-open"
-      use:clickOutside={() => {
-        open = false
-      }}
-    >
-      <ul class="spectrum-Menu" role="listbox">
-        {#if options && Array.isArray(options)}
-          {#each options as option}
-            <li
-              class="spectrum-Menu-item"
-              class:is-selected={getOptionValue(option) === value}
-              role="option"
-              aria-selected="true"
-              tabindex="0"
-              on:click={() => onPick(getOptionValue(option))}
-            >
-              <span class="spectrum-Menu-itemLabel"
-                >{getOptionLabel(option)}</span
-              >
-              <svg
-                class="spectrum-Icon spectrum-UIIcon-Checkmark100 spectrum-Menu-checkmark spectrum-Menu-itemIcon"
-                focusable="false"
-                aria-hidden="true"
-              >
-                <use xlink:href="#spectrum-css-icon-Checkmark100" />
-              </svg>
-            </li>
-          {/each}
-        {/if}
-      </ul>
-    </div>
-  {/if}
 </div>
 
+<!-- svelte-ignore a11y-click-events-have-key-events -->
+<Popover
+  {anchor}
+  {open}
+  align="left"
+  on:close={() => (open = false)}
+  useAnchorWidth
+>
+  <div class="popover-content" use:clickOutside={() => (open = false)}>
+    <ul class="spectrum-Menu" role="listbox">
+      {#if options && Array.isArray(options)}
+        {#each options as option}
+          <li
+            class="spectrum-Menu-item"
+            class:is-selected={getOptionValue(option) === value}
+            role="option"
+            aria-selected="true"
+            tabindex="0"
+            on:click={() => onPick(getOptionValue(option))}
+          >
+            <span class="spectrum-Menu-itemLabel">{getOptionLabel(option)}</span
+            >
+            <svg
+              class="spectrum-Icon spectrum-UIIcon-Checkmark100 spectrum-Menu-checkmark spectrum-Menu-itemIcon"
+              focusable="false"
+              aria-hidden="true"
+            >
+              <use xlink:href="#spectrum-css-icon-Checkmark100" />
+            </svg>
+          </li>
+        {/each}
+      {/if}
+    </ul>
+  </div>
+</Popover>
+
 <style>
   .spectrum-InputGroup {
     min-width: 0;
@@ -124,10 +129,13 @@
   .spectrum-Textfield-input {
     width: 0;
   }
-  .spectrum-Popover {
-    max-height: 240px;
-    width: 100%;
-    z-index: 999;
-    top: 100%;
+
+  /* Popover */
+  .popover-content {
+    display: contents;
+  }
+  .popover-content:not(.auto-width) .spectrum-Menu-itemLabel {
+    width: 0;
+    flex: 1 1 auto;
   }
 </style>

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -37,6 +37,7 @@
     "svg",
     "bmp",
     "jfif",
+    "webp",
   ]
 
   const fieldId = id || uuid()
@@ -67,7 +68,7 @@
   }
 
   $: showDropzone =
-    (!maximum || (maximum && value?.length < maximum)) && !disabled
+    (!maximum || (maximum && (value?.length || 0) < maximum)) && !disabled
 
   async function processFileList(fileList) {
     if (

packages/bbui/src/Form/Core/Picker.svelte

@@ -45,7 +45,6 @@
   const dispatch = createEventDispatcher()
 
   let button
-  let popover
   let component
 
   $: sortedOptions = getSortedOptions(options, getOptionLabel, sort)
@@ -146,11 +145,11 @@
     <use xlink:href="#spectrum-css-icon-Chevron100" />
   </svg>
 </button>
+
 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <Popover
   anchor={customAnchor ? customAnchor : button}
   align={align || "left"}
-  bind:this={popover}
   {open}
   on:close={() => (open = false)}
   useAnchorWidth={!autoWidth}
@@ -266,16 +265,6 @@
     width: 100%;
     box-shadow: none;
   }
-
-  .subtitle-text {
-    font-size: 12px;
-    line-height: 15px;
-    font-weight: 500;
-    color: var(--spectrum-global-color-gray-600);
-    display: block;
-    margin-top: var(--spacing-s);
-  }
-
   .spectrum-Picker-label.auto-width {
     margin-right: var(--spacing-xs);
   }
@@ -356,11 +345,9 @@
   .option-extra.icon.field-icon {
     display: flex;
   }
-
   .option-tag {
     margin: 0 var(--spacing-m) 0 var(--spacing-m);
   }
-
   .option-tag :global(.spectrum-Tags-item > .spectrum-Icon) {
     margin-top: 2px;
   }
@@ -374,4 +361,13 @@
   .loading--withAutocomplete {
     top: calc(34px + var(--spacing-m));
   }
+
+  .subtitle-text {
+    font-size: 12px;
+    line-height: 15px;
+    font-weight: 500;
+    color: var(--spectrum-global-color-gray-600);
+    display: block;
+    margin-top: var(--spacing-s);
+  }
 </style>

packages/bbui/src/Popover/Popover.svelte

@@ -99,10 +99,10 @@
       on:keydown={handleEscape}
       class="spectrum-Popover is-open"
       class:customZindex
-      class:hide-popover={open && !showPopover}
+      class:hidden={!showPopover}
       role="presentation"
       style="height: {customHeight}; --customZindex: {customZindex};"
-      transition:fly|local={{ y: -20, duration: animate ? 200 : 0 }}
+      transition:fly|local={{ y: -20, duration: animate ? 260 : 0 }}
       on:mouseenter
       on:mouseleave
     >
@@ -112,16 +112,17 @@
 {/if}
 
 <style>
-  .hide-popover {
-    display: contents;
-  }
-
   .spectrum-Popover {
     min-width: var(--spectrum-global-dimension-size-2000);
     border-color: var(--spectrum-global-color-gray-300);
     overflow: auto;
+    transition: opacity 260ms ease-out, transform 260ms ease-out;
+  }
+  .hidden {
+    opacity: 0;
+    pointer-events: none;
+    transform: translateY(-20px);
   }
-
   .customZindex {
     z-index: var(--customZindex) !important;
   }

packages/builder/src/analytics/IntercomClient.js

@@ -1,95 +0,0 @@
-export default class IntercomClient {
-  constructor(token) {
-    this.token = token
-  }
-
-  //
-  /**
-   * Instantiate intercom using their provided script.
-   */
-  init() {
-    if (!this.token) return
-
-    const token = this.token
-
-    var w = window
-    var ic = w.Intercom
-    if (typeof ic === "function") {
-      ic("reattach_activator")
-      ic("update", w.intercomSettings)
-    } else {
-      var d = document
-      var i = function () {
-        i.c(arguments)
-      }
-      i.q = []
-      i.c = function (args) {
-        i.q.push(args)
-      }
-      w.Intercom = i
-      var l = function () {
-        var s = d.createElement("script")
-        s.type = "text/javascript"
-        s.async = true
-        s.src = "https://widget.intercom.io/widget/" + token
-        var x = d.getElementsByTagName("script")[0]
-        x.parentNode.insertBefore(s, x)
-      }
-      if (document.readyState === "complete") {
-        l()
-      } else if (w.attachEvent) {
-        w.attachEvent("onload", l)
-      } else {
-        w.addEventListener("load", l, false)
-      }
-
-      this.initialised = true
-    }
-  }
-
-  /**
-   * Show the intercom chat bubble.
-   * @param {Object} user - user to identify
-   * @returns Intercom global object
-   */
-  show(user = {}, enabled) {
-    if (!this.initialised || !enabled) return
-
-    return window.Intercom("boot", {
-      app_id: this.token,
-      ...user,
-    })
-  }
-
-  /**
-   * Update intercom user details and messages.
-   * @returns Intercom global object
-   */
-  update() {
-    if (!this.initialised) return
-
-    return window.Intercom("update")
-  }
-
-  /**
-   * Capture analytics events and send them to intercom.
-   * @param {String} event - event identifier
-   * @param {Object} props - properties for the event
-   * @returns Intercom global object
-   */
-  captureEvent(event, props = {}) {
-    if (!this.initialised) return
-
-    return window.Intercom("trackEvent", event, props)
-  }
-
-  /**
-   * Disassociate the user from the current session.
-   * @returns Intercom global object
-   */
-  logout() {
-    if (!this.initialised) return
-
-    return window.Intercom("shutdown")
-  }
-}

packages/builder/src/analytics/index.js

@@ -1,14 +1,12 @@
 import { API } from "api"
 import PosthogClient from "./PosthogClient"
-import IntercomClient from "./IntercomClient"
 import { Events, EventSource } from "./constants"
 
 const posthog = new PosthogClient(process.env.POSTHOG_TOKEN)
-const intercom = new IntercomClient(process.env.INTERCOM_TOKEN)
 
 class AnalyticsHub {
   constructor() {
-    this.clients = [posthog, intercom]
+    this.clients = [posthog]
     this.initialised = false
   }
 
@@ -31,20 +29,10 @@ class AnalyticsHub {
 
   captureEvent(eventName, props = {}) {
     posthog.captureEvent(eventName, props)
-    intercom.captureEvent(eventName, props)
-  }
-
-  showChat(user) {
-    intercom.show(user)
-  }
-
-  initPosthog() {
-    posthog.init()
   }
 
   async logout() {
     posthog.logout()
-    intercom.logout()
   }
 }
 

packages/builder/src/components/backend/DataTable/formula.js

@@ -9,7 +9,7 @@ const MAX_DEPTH = 1
 const TYPES_TO_SKIP = [
   FieldType.FORMULA,
   FieldType.LONGFORM,
-  FieldType.ATTACHMENT,
+  FieldType.ATTACHMENTS,
   //https://github.com/Budibase/budibase/issues/3030
   FieldType.INTERNAL,
 ]

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -394,7 +394,8 @@
         FIELDS.BIGINT,
         FIELDS.BOOLEAN,
         FIELDS.DATETIME,
-        FIELDS.ATTACHMENT,
+        FIELDS.ATTACHMENT_SINGLE,
+        FIELDS.ATTACHMENTS,
         FIELDS.LINK,
         FIELDS.FORMULA,
         FIELDS.JSON,

packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte

@@ -1,6 +1,7 @@
 <script>
+  import { FieldType, FieldSubtype } from "@budibase/types"
   import { Select, Toggle, Multiselect } from "@budibase/bbui"
-  import { DB_TYPE_INTERNAL, FIELDS } from "constants/backend"
+  import { DB_TYPE_INTERNAL } from "constants/backend"
   import { API } from "api"
   import { parseFile } from "./utils"
 
@@ -23,43 +24,47 @@
   const typeOptions = [
     {
       label: "Text",
-      value: FIELDS.STRING.type,
+      value: FieldType.STRING,
     },
     {
       label: "Number",
-      value: FIELDS.NUMBER.type,
+      value: FieldType.NUMBER,
     },
     {
       label: "Date",
-      value: FIELDS.DATETIME.type,
+      value: FieldType.DATETIME,
     },
     {
       label: "Options",
-      value: FIELDS.OPTIONS.type,
+      value: FieldType.OPTIONS,
     },
     {
       label: "Multi-select",
-      value: FIELDS.ARRAY.type,
+      value: FieldType.ARRAY.type,
     },
     {
       label: "Barcode/QR",
-      value: FIELDS.BARCODEQR.type,
+      value: FieldType.BARCODEQR,
     },
     {
       label: "Long Form Text",
-      value: FIELDS.LONGFORM.type,
+      value: FieldType.LONGFORM,
     },
     {
       label: "Attachment",
-      value: FIELDS.ATTACHMENT.type,
+      value: FieldType.ATTACHMENT_SINGLE,
+    },
+    {
+      label: "Attachment list",
+      value: FieldType.ATTACHMENTS,
     },
     {
       label: "User",
-      value: `${FIELDS.USER.type}${FIELDS.USER.subtype}`,
+      value: `${FieldType.BB_REFERENCE}${FieldSubtype.USER}`,
     },
     {
       label: "Users",
-      value: `${FIELDS.USERS.type}${FIELDS.USERS.subtype}`,
+      value: `${FieldType.BB_REFERENCE}${FieldSubtype.USERS}`,
     },
   ]
 

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -131,7 +131,7 @@
     if (bindings?.length) {
       options.push(SidePanels.Bindings)
     }
-    if (context) {
+    if (context && Object.keys(context).length > 0) {
       options.push(SidePanels.Evaluation)
     }
     if (useSnippets && mode === Modes.JavaScript) {

packages/builder/src/components/common/bindings/DrawerBindableCombobox.svelte

@@ -78,7 +78,12 @@
   {/if}
 </div>
 
-<Drawer bind:this={bindingDrawer} title={title ?? placeholder ?? "Bindings"}>
+<Drawer
+  bind:this={bindingDrawer}
+  title={title ?? placeholder ?? "Bindings"}
+  on:drawerHide
+  on:drawerShow
+>
   <Button cta slot="buttons" on:click={handleClose}>Save</Button>
   <svelte:component
     this={panel}

packages/builder/src/components/deploy/AppActions.svelte

@@ -32,7 +32,6 @@
   import TourWrap from "components/portal/onboarding/TourWrap.svelte"
   import { TOUR_STEP_KEYS } from "components/portal/onboarding/tours.js"
   import { goto } from "@roxi/routify"
-  import { onMount } from "svelte"
 
   export let application
   export let loaded
@@ -151,10 +150,6 @@
       notifications.error("Error refreshing app")
     }
   }
-
-  onMount(() => {
-    analytics.initPosthog()
-  })
 </script>
 
 <!-- svelte-ignore a11y-click-events-have-key-events -->

packages/builder/src/components/design/settings/componentSettings.js

@@ -1,4 +1,6 @@
 import { Checkbox, Select, RadioGroup, Stepper, Input } from "@budibase/bbui"
+import { licensing } from "stores/portal"
+import { get } from "svelte/store"
 import DataSourceSelect from "./controls/DataSourceSelect/DataSourceSelect.svelte"
 import S3DataSourceSelect from "./controls/S3DataSourceSelect.svelte"
 import DataProviderSelect from "./controls/DataProviderSelect.svelte"
@@ -26,7 +28,8 @@ import FieldConfiguration from "./controls/FieldConfiguration/FieldConfiguration
 import ButtonConfiguration from "./controls/ButtonConfiguration/ButtonConfiguration.svelte"
 import RelationshipFilterEditor from "./controls/RelationshipFilterEditor.svelte"
 import FormStepConfiguration from "./controls/FormStepConfiguration.svelte"
-import FormStepControls from "components/design/settings/controls/FormStepControls.svelte"
+import FormStepControls from "./controls/FormStepControls.svelte"
+import PaywalledSetting from "./controls/PaywalledSetting.svelte"
 
 const componentMap = {
   text: DrawerBindableInput,
@@ -67,6 +70,7 @@ const componentMap = {
   "field/longform": FormFieldSelect,
   "field/datetime": FormFieldSelect,
   "field/attachment": FormFieldSelect,
+  "field/attachment_single": FormFieldSelect,
   "field/s3": Input,
   "field/link": FormFieldSelect,
   "field/array": FormFieldSelect,
@@ -86,11 +90,16 @@ const componentMap = {
 }
 
 export const getComponentForSetting = setting => {
-  const { type, showInBar, barStyle } = setting || {}
+  const { type, showInBar, barStyle, license } = setting || {}
   if (!type) {
     return null
   }
 
+  // Check for paywalled settings
+  if (license && get(licensing).isFreePlan) {
+    return PaywalledSetting
+  }
+
   // We can show a clone of the bar settings for certain select settings
   if (showInBar && type === "select" && barStyle === "buttons") {
     return BarButtonList

packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonConfiguration.svelte

@@ -14,7 +14,6 @@
   export let key
   export let nested
   export let max
-  export let context
 
   const dispatch = createEventDispatcher()
 

packages/builder/src/components/design/settings/controls/DraggableList/DraggableList.svelte

@@ -126,7 +126,7 @@
       <div class="right-content">
         <svelte:component
           this={listType}
-          anchor={anchors[draggableItem.item._id]}
+          anchor={anchors[draggableItem.id]}
           item={draggableItem.item}
           {...listTypeProps}
           on:change={onItemChanged}

packages/builder/src/components/design/settings/controls/FieldConfiguration/utils.js

@@ -41,7 +41,8 @@ export const FieldTypeToComponentMap = {
   [FieldType.BOOLEAN]: "booleanfield",
   [FieldType.LONGFORM]: "longformfield",
   [FieldType.DATETIME]: "datetimefield",
-  [FieldType.ATTACHMENT]: "attachmentfield",
+  [FieldType.ATTACHMENTS]: "attachmentfield",
+  [FieldType.ATTACHMENT_SINGLE]: "attachmentsinglefield",
   [FieldType.LINK]: "relationshipfield",
   [FieldType.JSON]: "jsonfield",
   [FieldType.BARCODEQR]: "codescanner",

packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte

@@ -34,7 +34,7 @@
     $selectedScreen,
     datasource
   )?.table?.primaryDisplay
-  $: schema = getSchema(selectedScreen, datasource)
+  $: schema = getSchema($selectedScreen, datasource)
   $: columns = getColumns({
     columns: value,
     schema,

packages/builder/src/components/design/settings/controls/PaywalledSetting.svelte

@@ -0,0 +1,23 @@
+<script>
+  import { Tag, Tags } from "@budibase/bbui"
+  import { getFormattedPlanName } from "helpers/planTitle"
+
+  export let license
+
+  $: title = getFormattedPlanName(license)
+</script>
+
+<div>
+  <Tags>
+    <Tag icon="LockClosed">{title}</Tag>
+  </Tags>
+</div>
+
+<style>
+  div {
+    height: 32px;
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+  }
+</style>

packages/builder/src/constants/backend/index.js

@@ -5,6 +5,9 @@ import {
   AutoFieldSubType,
   Hosting,
 } from "@budibase/types"
+import { Constants } from "@budibase/frontend-core"
+
+const { TypeIconMap } = Constants
 
 export { RelationshipType } from "@budibase/types"
 
@@ -22,7 +25,7 @@ export const FIELDS = {
   STRING: {
     name: "Text",
     type: FieldType.STRING,
-    icon: "Text",
+    icon: TypeIconMap[FieldType.STRING],
     constraints: {
       type: "string",
       length: {},
@@ -32,7 +35,7 @@ export const FIELDS = {
   BARCODEQR: {
     name: "Barcode/QR",
     type: FieldType.BARCODEQR,
-    icon: "Camera",
+    icon: TypeIconMap[FieldType.BARCODEQR],
     constraints: {
       type: "string",
       length: {},
@@ -42,7 +45,7 @@ export const FIELDS = {
   LONGFORM: {
     name: "Long Form Text",
     type: FieldType.LONGFORM,
-    icon: "TextAlignLeft",
+    icon: TypeIconMap[FieldType.LONGFORM],
     constraints: {
       type: "string",
       length: {},
@@ -52,7 +55,7 @@ export const FIELDS = {
   OPTIONS: {
     name: "Options",
     type: FieldType.OPTIONS,
-    icon: "Dropdown",
+    icon: TypeIconMap[FieldType.OPTIONS],
     constraints: {
       type: "string",
       presence: false,
@@ -62,7 +65,7 @@ export const FIELDS = {
   ARRAY: {
     name: "Multi-select",
     type: FieldType.ARRAY,
-    icon: "Duplicate",
+    icon: TypeIconMap[FieldType.ARRAY],
     constraints: {
       type: "array",
       presence: false,
@@ -72,7 +75,7 @@ export const FIELDS = {
   NUMBER: {
     name: "Number",
     type: FieldType.NUMBER,
-    icon: "123",
+    icon: TypeIconMap[FieldType.NUMBER],
     constraints: {
       type: "number",
       presence: false,
@@ -82,12 +85,12 @@ export const FIELDS = {
   BIGINT: {
     name: "BigInt",
     type: FieldType.BIGINT,
-    icon: "TagBold",
+    icon: TypeIconMap[FieldType.BIGINT],
   },
   BOOLEAN: {
     name: "Boolean",
     type: FieldType.BOOLEAN,
-    icon: "Boolean",
+    icon: TypeIconMap[FieldType.BOOLEAN],
     constraints: {
       type: "boolean",
       presence: false,
@@ -96,7 +99,7 @@ export const FIELDS = {
   DATETIME: {
     name: "Date/Time",
     type: FieldType.DATETIME,
-    icon: "Calendar",
+    icon: TypeIconMap[FieldType.DATETIME],
     constraints: {
       type: "string",
       length: {},
@@ -107,10 +110,18 @@ export const FIELDS = {
       },
     },
   },
-  ATTACHMENT: {
+  ATTACHMENT_SINGLE: {
     name: "Attachment",
-    type: FieldType.ATTACHMENT,
-    icon: "Folder",
+    type: FieldType.ATTACHMENT_SINGLE,
+    icon: TypeIconMap[FieldType.ATTACHMENT_SINGLE],
+    constraints: {
+      presence: false,
+    },
+  },
+  ATTACHMENTS: {
+    name: "Attachment List",
+    type: FieldType.ATTACHMENTS,
+    icon: TypeIconMap[FieldType.ATTACHMENTS],
     constraints: {
       type: "array",
       presence: false,
@@ -119,7 +130,7 @@ export const FIELDS = {
   LINK: {
     name: "Relationship",
     type: FieldType.LINK,
-    icon: "Link",
+    icon: TypeIconMap[FieldType.LINK],
     constraints: {
       type: "array",
       presence: false,
@@ -128,19 +139,19 @@ export const FIELDS = {
   AUTO: {
     name: "Auto Column",
     type: FieldType.AUTO,
-    icon: "MagicWand",
+    icon: TypeIconMap[FieldType.AUTO],
     constraints: {},
   },
   FORMULA: {
     name: "Formula",
     type: FieldType.FORMULA,
-    icon: "Calculator",
+    icon: TypeIconMap[FieldType.FORMULA],
     constraints: {},
   },
   JSON: {
     name: "JSON",
     type: FieldType.JSON,
-    icon: "Brackets",
+    icon: TypeIconMap[FieldType.JSON],
     constraints: {
       type: "object",
       presence: false,
@@ -150,13 +161,13 @@ export const FIELDS = {
     name: "User",
     type: FieldType.BB_REFERENCE,
     subtype: FieldSubtype.USER,
-    icon: "User",
+    icon: TypeIconMap[FieldType.USER],
   },
   USERS: {
     name: "Users",
     type: FieldType.BB_REFERENCE,
     subtype: FieldSubtype.USERS,
-    icon: "User",
+    icon: TypeIconMap[FieldType.USERS],
     constraints: {
       type: "array",
     },
@@ -299,7 +310,7 @@ export const PaginationLocations = [
 
 export const BannedSearchTypes = [
   FieldType.LINK,
-  FieldType.ATTACHMENT,
+  FieldType.ATTACHMENTS,
   FieldType.FORMULA,
   FieldType.JSON,
   "jsonarray",

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte

@@ -183,6 +183,7 @@
               props={{
                 // Generic settings
                 placeholder: setting.placeholder || null,
+                license: setting.license,
 
                 // Select settings
                 options: setting.options || [],

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Navigation/EditNavItemPopover.svelte

@@ -0,0 +1,141 @@
+<script>
+  import { Icon, Popover, RadioGroup } from "@budibase/bbui"
+  import { createEventDispatcher, getContext } from "svelte"
+  import PropertyControl from "components/design/settings/controls/PropertyControl.svelte"
+  import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
+  import DrawerBindableCombobox from "components/common/bindings/DrawerBindableCombobox.svelte"
+  import RoleSelect from "components/common/RoleSelect.svelte"
+  import SubLinksDrawer from "./SubLinksDrawer.svelte"
+  import { screenStore } from "stores/builder"
+
+  export let anchor
+  export let navItem
+  export let bindings
+
+  const draggable = getContext("draggable")
+  const dispatch = createEventDispatcher()
+  const typeOptions = [
+    { label: "Inline link", value: "link" },
+    { label: "Open sub links", value: "sublinks" },
+  ]
+
+  let popover
+  let open = false
+  let drawerCount = 0
+
+  $: urlOptions = $screenStore.screens
+    .map(screen => screen.routing?.route)
+    .filter(x => x != null)
+    .sort()
+
+  // Auto hide the component when another item is selected
+  $: if (open && $draggable.selected !== navItem.id) {
+    popover.hide()
+  }
+
+  // Open automatically if the component is marked as selected
+  $: if (!open && $draggable.selected === navItem.id && popover) {
+    popover.show()
+    open = true
+  }
+
+  const update = setting => async value => {
+    dispatch("change", {
+      ...navItem,
+      [setting]: value,
+    })
+  }
+</script>
+
+<Icon name={navItem.type === "sublinks" ? "Dropdown" : "Link"} size="S" />
+
+<Popover
+  bind:this={popover}
+  on:open={() => {
+    open = true
+    $draggable.actions.select(navItem.id)
+  }}
+  on:close={() => {
+    open = false
+    if ($draggable.selected === navItem.id) {
+      $draggable.actions.select()
+    }
+  }}
+  {anchor}
+  align="left-outside"
+  showPopover={drawerCount === 0}
+  clickOutsideOverride={drawerCount > 0}
+  maxHeight={600}
+  offset={18}
+>
+  <div class="settings">
+    <PropertyControl
+      label="Nav item"
+      control={RadioGroup}
+      value={navItem.type}
+      onChange={update("type")}
+      props={{
+        options: typeOptions,
+      }}
+    />
+    <PropertyControl
+      label="Label"
+      control={DrawerBindableInput}
+      value={navItem.text}
+      onChange={update("text")}
+      {bindings}
+      props={{
+        updateOnChange: false,
+      }}
+      on:drawerShow={() => drawerCount++}
+      on:drawerHide={() => drawerCount--}
+    />
+    {#if navItem.type === "sublinks"}
+      <PropertyControl
+        label="Sub links"
+        control={SubLinksDrawer}
+        value={navItem.subLinks}
+        onChange={update("subLinks")}
+        {bindings}
+        props={{
+          navItem,
+        }}
+        on:drawerShow={() => drawerCount++}
+        on:drawerHide={() => drawerCount--}
+      />
+    {:else}
+      <PropertyControl
+        label="Link"
+        control={DrawerBindableCombobox}
+        value={navItem.url}
+        onChange={update("url")}
+        {bindings}
+        props={{
+          options: urlOptions,
+          appendBindingsAsOptions: false,
+          placeholder: null,
+        }}
+        on:drawerShow={() => drawerCount++}
+        on:drawerHide={() => drawerCount--}
+      />
+    {/if}
+    <PropertyControl
+      label="Access"
+      control={RoleSelect}
+      value={navItem.roleId}
+      onChange={update("roleId")}
+    />
+  </div>
+</Popover>
+
+<style>
+  .settings {
+    background: var(--spectrum-alias-background-color-primary);
+    display: flex;
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: stretch;
+    gap: 8px;
+    padding: var(--spacing-xl);
+  }
+</style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Navigation/LinksDrawer.svelte

@@ -1,131 +0,0 @@
-<script>
-  import {
-    Button,
-    Icon,
-    DrawerContent,
-    Layout,
-    Input,
-    Combobox,
-  } from "@budibase/bbui"
-  import { flip } from "svelte/animate"
-  import { dndzone } from "svelte-dnd-action"
-  import { generate } from "shortid"
-  import { screenStore } from "stores/builder"
-  import RoleSelect from "components/design/settings/controls/RoleSelect.svelte"
-
-  export let links = []
-
-  const flipDurationMs = 150
-  let dragDisabled = true
-
-  $: links.forEach(link => {
-    if (!link.id) {
-      link.id = generate()
-    }
-  })
-  $: urlOptions = $screenStore.screens
-    .map(screen => screen.routing?.route)
-    .filter(x => x != null)
-
-  const addLink = () => {
-    links = [...links, {}]
-  }
-
-  const removeLink = id => {
-    links = links.filter(link => link.id !== id)
-  }
-
-  const updateLinks = e => {
-    links = e.detail.items
-  }
-
-  const handleFinalize = e => {
-    updateLinks(e)
-    dragDisabled = true
-  }
-</script>
-
-<!-- svelte-ignore a11y-no-static-element-interactions -->
-<DrawerContent>
-  <div class="container">
-    <Layout noPadding gap="S">
-      {#if links?.length}
-        <div
-          class="links"
-          use:dndzone={{
-            items: links,
-            flipDurationMs,
-            dropTargetStyle: { outline: "none" },
-            dragDisabled,
-          }}
-          on:finalize={handleFinalize}
-          on:consider={updateLinks}
-        >
-          {#each links as link (link.id)}
-            <div class="link" animate:flip={{ duration: flipDurationMs }}>
-              <div
-                class="handle"
-                aria-label="drag-handle"
-                style={dragDisabled ? "cursor: grab" : "cursor: grabbing"}
-                on:mousedown={() => (dragDisabled = false)}
-              >
-                <Icon name="DragHandle" size="XL" />
-              </div>
-              <Input bind:value={link.text} placeholder="Text" />
-              <Combobox
-                bind:value={link.url}
-                placeholder="URL"
-                options={urlOptions}
-              />
-              <RoleSelect bind:value={link.roleId} placeholder="Minimum role" />
-              <Icon
-                name="Close"
-                hoverable
-                size="S"
-                on:click={() => removeLink(link.id)}
-              />
-            </div>
-          {/each}
-        </div>
-      {/if}
-      <div>
-        <Button secondary icon="Add" on:click={addLink}>Add Link</Button>
-      </div>
-    </Layout>
-  </div>
-</DrawerContent>
-
-<style>
-  .container {
-    width: 100%;
-    max-width: 800px;
-    margin: 0 auto;
-  }
-  .links {
-    display: flex;
-    flex-direction: column;
-    justify-content: flex-start;
-    align-items: stretch;
-    gap: var(--spacing-m);
-  }
-  .link {
-    gap: var(--spacing-l);
-    display: flex;
-    flex-direction: row;
-    justify-content: space-between;
-    align-items: center;
-    border-radius: var(--border-radius-s);
-    transition: background-color ease-in-out 130ms;
-  }
-  .link:hover {
-    background-color: var(--spectrum-global-color-gray-100);
-  }
-  .link > :global(.spectrum-Form-item) {
-    flex: 1 1 auto;
-    width: 0;
-  }
-  .handle {
-    display: grid;
-    place-items: center;
-  }
-</style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Navigation/LinksEditor.svelte

@@ -1,30 +0,0 @@
-<script>
-  import { Button, Drawer } from "@budibase/bbui"
-  import NavigationLinksDrawer from "./LinksDrawer.svelte"
-  import { cloneDeep } from "lodash/fp"
-  import { navigationStore } from "stores/builder"
-
-  let drawer
-  let links
-
-  const openDrawer = () => {
-    links = cloneDeep($navigationStore.links || [])
-    drawer.show()
-  }
-
-  const save = async () => {
-    let navigation = $navigationStore
-    navigation.links = links
-    await navigationStore.save(navigation)
-    drawer.hide()
-  }
-</script>
-
-<Button cta on:click={openDrawer}>Configure Links</Button>
-<Drawer bind:this={drawer} title={"Navigation Links"}>
-  <svelte:fragment slot="description">
-    Configure the links in your navigation bar.
-  </svelte:fragment>
-  <Button cta slot="buttons" on:click={save}>Save</Button>
-  <NavigationLinksDrawer slot="body" bind:links />
-</Drawer>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Navigation/NavItem.svelte

@@ -0,0 +1,55 @@
+<script>
+  import { runtimeToReadableBinding } from "dataBinding"
+  import EditNavItemPopover from "./EditNavItemPopover.svelte"
+  import { Icon } from "@budibase/bbui"
+
+  export let item
+  export let removeNavItem
+  export let anchor
+  export let bindings
+
+  $: text = runtimeToReadableBinding(bindings, item.text)
+</script>
+
+<div class="list-item-body">
+  <div class="list-item-left">
+    <EditNavItemPopover {anchor} {bindings} navItem={item} on:change />
+    <div class="field-label">{text}</div>
+  </div>
+  <div class="list-item-right">
+    <Icon
+      size="S"
+      name="Close"
+      hoverable
+      on:click={e => {
+        e.stopPropagation()
+        removeNavItem(item.id)
+      }}
+    />
+  </div>
+</div>
+
+<style>
+  .field-label {
+    text-overflow: ellipsis;
+    white-space: nowrap;
+    overflow: hidden;
+  }
+  .list-item-body,
+  .list-item-left {
+    display: flex;
+    align-items: center;
+    gap: var(--spacing-m);
+    min-width: 0;
+  }
+  .list-item-body {
+    margin-top: 8px;
+    margin-bottom: 8px;
+  }
+  .list-item-right :global(div.spectrum-Switch) {
+    margin: 0px;
+  }
+  .list-item-body {
+    justify-content: space-between;
+  }
+</style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Navigation/NavItemConfiguration.svelte

@@ -0,0 +1,118 @@
+<script>
+  import { navigationStore } from "stores/builder"
+  import DraggableList from "components/design/settings/controls/DraggableList/DraggableList.svelte"
+  import NavItem from "./NavItem.svelte"
+  import { generate } from "shortid"
+  import { getSequentialName } from "helpers/duplicate"
+  import { Constants } from "@budibase/frontend-core"
+
+  export let bindings
+
+  $: navItems = enrichNavItems($navigationStore.links)
+  $: navItemProps = {
+    removeNavItem,
+    bindings,
+  }
+
+  const enrichNavItems = links => {
+    return (links || []).map(link => ({
+      ...link,
+      id: link.id || generate(),
+    }))
+  }
+
+  const save = async links => {
+    await navigationStore.save({ ...$navigationStore, links })
+  }
+
+  const handleNavItemUpdate = async e => {
+    const newNavItem = e.detail
+    const newLinks = [...navItems]
+    const idx = newLinks.findIndex(link => {
+      return link.id === newNavItem?.id
+    })
+    if (idx === -1) {
+      newLinks.push(newNavItem)
+    } else {
+      newLinks[idx] = newNavItem
+    }
+    await save(newLinks)
+  }
+
+  const handleListUpdate = async e => {
+    await save([...e.detail])
+  }
+
+  const addNavItem = async () => {
+    await save([
+      ...navItems,
+      {
+        id: generate(),
+        text: getSequentialName(navItems, "Nav Item ", x => x.text),
+        url: "",
+        roleId: Constants.Roles.BASIC,
+        type: "link",
+      },
+    ])
+  }
+
+  const removeNavItem = async id => {
+    await save(navItems.filter(navItem => navItem.id !== id))
+  }
+</script>
+
+<!-- svelte-ignore a11y-click-events-have-key-events -->
+<!-- svelte-ignore a11y-no-static-element-interactions -->
+<div class="link-configuration">
+  {#if navItems.length}
+    <DraggableList
+      on:change={handleListUpdate}
+      on:itemChange={handleNavItemUpdate}
+      items={navItems}
+      listItemKey="id"
+      listType={NavItem}
+      listTypeProps={navItemProps}
+      draggable={navItems.length > 1}
+    />
+  {/if}
+
+  <div class="list-footer" on:click={addNavItem} class:empty={!navItems.length}>
+    <div class="add-button">Add nav item</div>
+  </div>
+</div>
+
+<style>
+  .link-configuration :global(.list-wrap > li:last-child),
+  .link-configuration :global(.list-wrap) {
+    border-bottom-left-radius: unset;
+    border-bottom-right-radius: unset;
+    border-bottom: none;
+  }
+  .list-footer {
+    width: 100%;
+    border-bottom-left-radius: 4px;
+    border-bottom-right-radius: 4px;
+    background-color: var(
+      --spectrum-table-background-color,
+      var(--spectrum-global-color-gray-50)
+    );
+    transition: background-color ease-in-out 130ms;
+    display: flex;
+    justify-content: center;
+    border: 1px solid var(--spectrum-alias-border-color-mid);
+    cursor: pointer;
+  }
+  .list-footer.empty {
+    border-radius: 4px;
+  }
+  .list-footer:hover {
+    background-color: var(
+      --spectrum-table-row-background-color-hover,
+      var(--spectrum-alias-highlight-hover)
+    );
+  }
+
+  .add-button {
+    margin: var(--spacing-s);
+  }
+</style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Navigation/SubLinksDrawer.svelte

@@ -0,0 +1,142 @@
+<script>
+  import {
+    ActionButton,
+    Button,
+    Icon,
+    DrawerContent,
+    Layout,
+    Input,
+    Drawer,
+  } from "@budibase/bbui"
+  import { flip } from "svelte/animate"
+  import { dndzone } from "svelte-dnd-action"
+  import { generate } from "shortid"
+  import { screenStore } from "stores/builder"
+  import DrawerBindableCombobox from "components/common/bindings/DrawerBindableCombobox.svelte"
+
+  export let value = []
+  export let onChange
+  export let navItem
+  export let bindings
+
+  const flipDurationMs = 150
+
+  let drawer
+  let subLinks = value?.slice() || []
+
+  $: count = value?.length ?? 0
+  $: buttonText = `${count || "No"} sub link${count === 1 ? "" : "s"}`
+  $: drawerTitle = navItem.text ? `${navItem.text} sub links` : "Sub links"
+  $: subLinks.forEach(subLink => {
+    if (!subLink.id) {
+      subLink.id = generate()
+    }
+  })
+  $: urlOptions = $screenStore.screens
+    .map(screen => screen.routing?.route)
+    .filter(x => x != null)
+    .sort()
+
+  const addSubLink = () => {
+    subLinks = [...subLinks, {}]
+  }
+
+  const removeSubLink = id => {
+    subLinks = subLinks.filter(link => link.id !== id)
+  }
+
+  const saveSubLinks = () => {
+    onChange(subLinks)
+    drawer.hide()
+  }
+
+  const updateSubLinks = e => {
+    subLinks = e.detail.items
+  }
+</script>
+
+<Drawer bind:this={drawer} title={drawerTitle} on:drawerShow on:drawerHide>
+  <Button cta slot="buttons" on:click={saveSubLinks}>Save</Button>
+  <DrawerContent slot="body">
+    <div class="container">
+      <Layout noPadding gap="S">
+        {#if subLinks?.length}
+          <div
+            class="subLinks"
+            use:dndzone={{
+              items: subLinks,
+              flipDurationMs,
+              dropTargetStyle: { outline: "none" },
+            }}
+            on:consider={updateSubLinks}
+            on:finalize={updateSubLinks}
+          >
+            {#each subLinks as subLink (subLink.id)}
+              <div class="subLink" animate:flip={{ duration: flipDurationMs }}>
+                <Icon name="DragHandle" size="XL" />
+                <Input bind:value={subLink.text} placeholder="Text" />
+                <DrawerBindableCombobox
+                  value={subLink.url}
+                  on:change={e => (subLink.url = e.detail)}
+                  placeholder="Link"
+                  options={urlOptions}
+                  {bindings}
+                  appendBindingsAsOptions={false}
+                />
+                <Icon
+                  name="Close"
+                  hoverable
+                  size="S"
+                  on:click={() => removeSubLink(subLink.id)}
+                />
+              </div>
+            {/each}
+          </div>
+        {/if}
+        <div>
+          <ActionButton quiet icon="Add" on:click={addSubLink}>
+            Add link
+          </ActionButton>
+        </div>
+      </Layout>
+    </div>
+  </DrawerContent>
+</Drawer>
+
+<div class="button">
+  <ActionButton on:click={drawer.show}>{buttonText}</ActionButton>
+</div>
+
+<style>
+  .button :global(.spectrum-ActionButton) {
+    width: 100%;
+  }
+  .container {
+    width: 100%;
+    max-width: 800px;
+    margin: 0 auto;
+  }
+  .subLinks {
+    display: flex;
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: stretch;
+    gap: var(--spacing-m);
+  }
+  .subLink {
+    gap: var(--spacing-l);
+    display: flex;
+    flex-direction: row;
+    justify-content: space-between;
+    align-items: center;
+    border-radius: var(--border-radius-s);
+    transition: background-color ease-in-out 130ms;
+  }
+  .subLink:hover {
+    background-color: var(--spectrum-global-color-gray-100);
+  }
+  .subLink > :global(.spectrum-Form-item) {
+    flex: 1 1 auto;
+    width: 0;
+  }
+</style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Navigation/index.svelte

@@ -1,62 +1,61 @@
 <script>
-  import LinksEditor from "./LinksEditor.svelte"
+  import NavItemConfiguration from "./NavItemConfiguration.svelte"
   import { get } from "svelte/store"
   import Panel from "components/design/Panel.svelte"
   import {
-    Detail,
     Toggle,
-    Body,
-    Icon,
-    ColorPicker,
-    Input,
-    Label,
-    ActionGroup,
-    ActionButton,
+    DetailSummary,
     Checkbox,
     notifications,
     Select,
-    Combobox,
   } from "@budibase/bbui"
   import {
     themeStore,
     selectedScreen,
     screenStore,
-    navigationStore,
+    componentStore,
+    navigationStore as nav,
   } from "stores/builder"
   import { DefaultAppTheme } from "constants"
-  import BarButtonList from "/src/components/design/settings/controls/BarButtonList.svelte"
+  import PropertyControl from "components/design/settings/controls/PropertyControl.svelte"
+  import BarButtonList from "components/design/settings/controls/BarButtonList.svelte"
+  import ColorPicker from "components/design/settings/controls/ColorPicker.svelte"
+  import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
+  import DrawerBindableCombobox from "components/common/bindings/DrawerBindableCombobox.svelte"
+  import { getBindableProperties } from "dataBinding"
 
-  $: alignmentOptions = [
+  const positionOptions = [
+    { value: "Top", barIcon: "PaddingTop" },
+    { value: "Left", barIcon: "PaddingLeft" },
+  ]
+  const alignmentOptions = [
     { value: "Left", barIcon: "TextAlignLeft" },
     { value: "Center", barIcon: "TextAlignCenter" },
     { value: "Right", barIcon: "TextAlignRight" },
   ]
+  const widthOptions = ["Max", "Large", "Medium", "Small"]
 
+  $: bindings = getBindableProperties(
+    $selectedScreen,
+    $componentStore.selectedComponentId
+  )
   $: screenRouteOptions = $screenStore.screens
     .map(screen => screen.routing?.route)
     .filter(x => x != null)
 
-  const updateShowNavigation = async e => {
-    await screenStore.updateSetting(
-      get(selectedScreen),
-      "showNavigation",
-      e.detail
-    )
+  const updateShowNavigation = async show => {
+    await screenStore.updateSetting(get(selectedScreen), "showNavigation", show)
   }
 
   const update = async (key, value) => {
     try {
-      let navigation = $navigationStore
+      let navigation = $nav
       navigation[key] = value
-      await navigationStore.save(navigation)
+      await nav.save(navigation)
     } catch (error) {
       notifications.error("Error updating navigation settings")
     }
   }
-
-  const updateTextAlign = textAlignValue => {
-    navigationStore.syncAppNavigation({ textAlign: textAlignValue })
-  }
 </script>
 
 <Panel
@@ -65,215 +64,142 @@
   borderLeft
   wide
 >
-  <div class="generalSection">
-    <div class="subheading">
-      <Detail>General</Detail>
-    </div>
-    <div class="toggle">
-      <Toggle
-        on:change={updateShowNavigation}
-        value={$selectedScreen?.showNavigation}
-      />
-      <Body size="S">Show nav on this screen</Body>
-    </div>
-  </div>
+  <DetailSummary name="General" initiallyShow collapsible={false}>
+    <PropertyControl
+      control={Toggle}
+      props={{ text: "Show nav on this screen" }}
+      onChange={updateShowNavigation}
+      value={$selectedScreen?.showNavigation}
+    />
+  </DetailSummary>
 
   {#if $selectedScreen?.showNavigation}
-    <div class="divider" />
-    <div class="customizeSection">
-      <div class="subheading">
-        <Detail>Customize</Detail>
-      </div>
-      <div class="info">
-        <Icon name="InfoOutline" size="S" />
-        <Body size="S">These settings apply to all screens</Body>
-      </div>
-      <div class="configureLinks">
-        <LinksEditor />
-      </div>
-      <div class="controls">
-        <div class="label">
-          <Label size="M">Position</Label>
-        </div>
-        <ActionGroup quiet>
-          <ActionButton
-            selected={$navigationStore.navigation === "Top"}
-            quiet={$navigationStore.navigation !== "Top"}
-            icon="PaddingTop"
-            on:click={() => update("navigation", "Top")}
+    <DetailSummary name="Customize" initiallyShow collapsible={false}>
+      <NavItemConfiguration {bindings} />
+      <div class="settings">
+        <PropertyControl
+          label="Position"
+          control={BarButtonList}
+          onChange={position => update("navigation", position)}
+          value={$nav.navigation}
+          props={{
+            options: positionOptions,
+          }}
+        />
+        {#if $nav.navigation === "Top"}
+          <PropertyControl
+            label="Sticky header"
+            control={Checkbox}
+            value={$nav.sticky}
+            onChange={sticky => update("sticky", sticky)}
           />
-          <ActionButton
-            selected={$navigationStore.navigation === "Left"}
-            quiet={$navigationStore.navigation !== "Left"}
-            icon="PaddingLeft"
-            on:click={() => update("navigation", "Left")}
-          />
-        </ActionGroup>
-
-        {#if $navigationStore.navigation === "Top"}
-          <div class="label">
-            <Label size="M">Sticky header</Label>
-          </div>
-          <Checkbox
-            value={$navigationStore.sticky}
-            on:change={e => update("sticky", e.detail)}
-          />
-          <div class="label">
-            <Label size="M">Width</Label>
-          </div>
-          <Select
-            options={["Max", "Large", "Medium", "Small"]}
-            plaveholder={null}
-            value={$navigationStore.navWidth}
-            on:change={e => update("navWidth", e.detail)}
+          <PropertyControl
+            label="Width"
+            control={Select}
+            onChange={position => update("navWidth", position)}
+            value={$nav.navWidth}
+            props={{
+              placeholder: null,
+              options: widthOptions,
+            }}
           />
         {/if}
-        <div class="label">
-          <Label size="M">Show title</Label>
-        </div>
-        <Checkbox
-          value={!$navigationStore.hideTitle}
-          on:change={e => update("hideTitle", !e.detail)}
+        <PropertyControl
+          label="Show title"
+          control={Checkbox}
+          value={!$nav.hideTitle}
+          onChange={show => update("hideTitle", !show)}
         />
-        {#if !$navigationStore.hideTitle}
-          <div class="label">
-            <Label size="M">Title</Label>
-          </div>
-          <Input
-            value={$navigationStore.title}
-            on:change={e => update("title", e.detail)}
-            updateOnChange={false}
+        {#if !$nav.hideTitle}
+          <PropertyControl
+            label="Title"
+            control={DrawerBindableInput}
+            value={$nav.title}
+            onChange={title => update("title", title)}
+            {bindings}
+            props={{
+              updateOnChange: false,
+            }}
           />
-
-          <div class="label">
-            <Label size="M">Text align</Label>
-          </div>
-          <BarButtonList
-            options={alignmentOptions}
-            value={$navigationStore.textAlign}
-            onChange={updateTextAlign}
+          <PropertyControl
+            label="Text align"
+            control={BarButtonList}
+            onChange={align => nav.syncAppNavigation({ textAlign: align })}
+            value={$nav.textAlign}
+            props={{
+              options: alignmentOptions,
+            }}
           />
         {/if}
-        <div class="label">
-          <Label>Background</Label>
-        </div>
-        <ColorPicker
-          spectrumTheme={$themeStore.theme}
-          value={$navigationStore.navBackground ||
-            DefaultAppTheme.navBackground}
-          on:change={e => update("navBackground", e.detail)}
+        <PropertyControl
+          label="Background"
+          control={ColorPicker}
+          onChange={color => update("navBackground", color)}
+          value={$nav.navBackground || DefaultAppTheme.navBackground}
+          props={{
+            spectrumTheme: $themeStore.theme,
+          }}
         />
-        <div class="label">
-          <Label>Text</Label>
-        </div>
-        <ColorPicker
-          spectrumTheme={$themeStore.theme}
-          value={$navigationStore.navTextColor || DefaultAppTheme.navTextColor}
-          on:change={e => update("navTextColor", e.detail)}
+        <PropertyControl
+          label="Text"
+          control={ColorPicker}
+          onChange={color => update("navTextColor", color)}
+          value={$nav.navTextColor || DefaultAppTheme.navTextColor}
+          props={{
+            spectrumTheme: $themeStore.theme,
+          }}
         />
       </div>
-    </div>
+    </DetailSummary>
 
-    <div class="divider" />
-    <div class="customizeSection">
-      <div class="subheading">
-        <Detail>Logo</Detail>
-      </div>
-      <div class="controls">
-        <div class="label">
-          <Label size="M">Show logo</Label>
-        </div>
-        <Checkbox
-          value={!$navigationStore.hideLogo}
-          on:change={e => update("hideLogo", !e.detail)}
+    <DetailSummary name="Logo" initiallyShow collapsible={false}>
+      <div class="settings">
+        <PropertyControl
+          label="Show logo"
+          control={Checkbox}
+          value={!$nav.hideLogo}
+          onChange={show => update("hideLogo", !show)}
         />
-        {#if !$navigationStore.hideLogo}
-          <div class="label">
-            <Label size="M">Logo image URL</Label>
-          </div>
-          <Input
-            value={$navigationStore.logoUrl}
-            on:change={e => update("logoUrl", e.detail)}
-            updateOnChange={false}
+        {#if !$nav.hideLogo}
+          <PropertyControl
+            label="Logo image URL"
+            control={DrawerBindableInput}
+            value={$nav.logoUrl}
+            onChange={url => update("logoUrl", url)}
+            {bindings}
+            props={{
+              updateOnChange: false,
+            }}
           />
-          <div class="label">
-            <Label size="M">Logo link URL</Label>
-          </div>
-          <Combobox
-            value={$navigationStore.logoLinkUrl}
-            on:change={e => update("logoLinkUrl", e.detail)}
-            options={screenRouteOptions}
+          <PropertyControl
+            label="Logo link URL"
+            control={DrawerBindableCombobox}
+            value={$nav.logoLinkUrl}
+            onChange={url => update("logoLinkUrl", url)}
+            {bindings}
+            props={{
+              appendBindingsAsOptions: false,
+              options: screenRouteOptions,
+            }}
           />
-          <div class="label">
-            <Label size="M">New tab</Label>
-          </div>
-          <Checkbox
-            value={!!$navigationStore.openLogoLinkInNewTab}
-            on:change={e => update("openLogoLinkInNewTab", !!e.detail)}
+          <PropertyControl
+            label="New tab"
+            control={Checkbox}
+            value={$nav.openLogoLinkInNewTab}
+            onChange={show => update("openLogoLinkInNewTab", show)}
           />
         {/if}
       </div>
-    </div>
+    </DetailSummary>
   {/if}
 </Panel>
 
 <style>
-  .generalSection {
-    padding: 13px 13px 25px;
-  }
-
-  .customizeSection {
-    padding: 13px 13px 25px;
-  }
-  .subheading {
-    margin-bottom: 10px;
-  }
-
-  .subheading :global(p) {
-    color: var(--grey-6);
-  }
-
-  .toggle {
+  .settings {
     display: flex;
-    align-items: center;
-  }
-
-  .divider {
-    border-top: 1px solid var(--grey-3);
-  }
-
-  .controls {
-    position: relative;
-    display: grid;
-    grid-template-columns: 90px 1fr;
-    align-items: start;
-    transition: background 130ms ease-out, border-color 130ms ease-out;
-    border-left: 4px solid transparent;
-    margin: 0 calc(-1 * var(--spacing-xl));
-    padding: 0 var(--spacing-xl) 0 calc(var(--spacing-xl) - 4px);
-    gap: 12px;
-  }
-
-  .label {
-    margin-top: 16px;
-    transform: translateY(-50%);
-  }
-
-  .info {
-    background-color: var(--background-alt);
-    padding: 12px;
-    display: flex;
-    border-radius: 4px;
-    gap: 4px;
-    margin-bottom: 16px;
-  }
-  .info :global(svg) {
-    margin-right: 5px;
-    color: var(--spectrum-global-color-gray-600);
-  }
-
-  .configureLinks :global(button) {
-    margin-bottom: 20px;
-    width: 100%;
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: stretch;
+    gap: 8px;
   }
 </style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json

@@ -63,6 +63,7 @@
       "optionsfield",
       "booleanfield",
       "longformfield",
+      "attachmentsinglefield",
       "attachmentfield",
       "jsonfield",
       "relationshipfield",

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentKeyHandler.svelte

@@ -137,8 +137,12 @@
     const activeTag = document.activeElement?.tagName.toLowerCase()
     const inCodeEditor =
       document.activeElement?.classList?.contains("cm-content")
+    const inPosthogSurvey =
+      document.activeElement?.classList?.[0]?.startsWith("PostHogSurvey")
     if (
-      (inCodeEditor || ["input", "textarea"].indexOf(activeTag) !== -1) &&
+      (inCodeEditor ||
+        inPosthogSurvey ||
+        ["input", "textarea"].indexOf(activeTag) !== -1) &&
       e.key !== "Escape"
     ) {
       return

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte

@@ -79,7 +79,8 @@
         // for autoscreens, so it's always safe to do this.
         await navigationStore.saveLink(
           screen.routing.route,
-          capitalise(screen.routing.route.split("/")[1])
+          capitalise(screen.routing.route.split("/")[1]),
+          screenAccessRole
         )
       }
 

packages/builder/src/stores/builder/navigation.js

@@ -42,7 +42,7 @@ export class NavigationStore extends BudiStore {
     this.syncAppNavigation(app.navigation)
   }
 
-  async saveLink(url, title) {
+  async saveLink(url, title, roleId) {
     const navigation = get(this.store)
     let links = [...(navigation?.links ?? [])]
 
@@ -54,6 +54,8 @@ export class NavigationStore extends BudiStore {
     links.push({
       text: title,
       url,
+      type: "link",
+      roleId,
     })
     await this.save({
       ...navigation,
@@ -67,11 +69,20 @@ export class NavigationStore extends BudiStore {
     if (!links?.length) {
       return
     }
-
-    // Filter out the URLs to delete
     urls = Array.isArray(urls) ? urls : [urls]
+
+    // Filter out top level links pointing to these URLs
     links = links.filter(link => !urls.includes(link.url))
 
+    // Filter out nested links pointing to these URLs
+    links.forEach(link => {
+      if (link.type === "sublinks" && link.subLinks?.length) {
+        link.subLinks = link.subLinks.filter(
+          subLink => !urls.includes(subLink.url)
+        )
+      }
+    })
+
     await this.save({
       ...navigation,
       links,

packages/builder/src/stores/builder/tests/navigation.test.js

@@ -50,10 +50,18 @@ describe("Navigation store", () => {
       {
         url: "/home",
         text: "Home",
+        type: "link",
       },
       {
         url: "/test",
         text: "Test",
+        type: "sublinks",
+        subLinks: [
+          {
+            text: "Foo",
+            url: "/bar",
+          },
+        ],
       },
     ]
 
@@ -66,7 +74,7 @@ describe("Navigation store", () => {
       .spyOn(ctx.test.navigationStore, "save")
       .mockImplementation(() => {})
 
-    await ctx.test.navigationStore.saveLink("/test-url", "Testing")
+    await ctx.test.navigationStore.saveLink("/test-url", "Testing", "BASIC")
 
     expect(saveSpy).toBeCalledWith({
       ...INITIAL_NAVIGATION_STATE,
@@ -75,6 +83,8 @@ describe("Navigation store", () => {
         {
           url: "/test-url",
           text: "Testing",
+          type: "link",
+          roleId: "BASIC",
         },
       ],
     })
@@ -87,6 +97,7 @@ describe("Navigation store", () => {
         {
           url: "/home",
           text: "Home",
+          type: "link",
         },
       ],
     }))
@@ -94,7 +105,7 @@ describe("Navigation store", () => {
       .spyOn(ctx.test.navigationStore, "save")
       .mockImplementation(() => {})
 
-    await ctx.test.navigationStore.saveLink("/home", "Home")
+    await ctx.test.navigationStore.saveLink("/home", "Home", "BASIC")
 
     expect(saveSpy).not.toHaveBeenCalled()
   })
@@ -106,14 +117,23 @@ describe("Navigation store", () => {
         {
           url: "/home",
           text: "Home",
+          type: "link",
         },
         {
           url: "/test",
           text: "Test",
+          type: "link",
         },
         {
           url: "/last",
           text: "Last Link",
+          type: "sublinks",
+          subLinks: [
+            {
+              text: "Foo",
+              url: "/home",
+            },
+          ],
         },
       ],
     }))
@@ -130,6 +150,8 @@ describe("Navigation store", () => {
         {
           text: "Last Link",
           url: "/last",
+          type: "sublinks",
+          subLinks: [],
         },
       ],
     })
@@ -140,14 +162,17 @@ describe("Navigation store", () => {
       {
         url: "/home",
         text: "Home",
+        type: "link",
       },
       {
         url: "/test",
         text: "Test",
+        type: "link",
       },
       {
         url: "/last",
         text: "Last Link",
+        type: "link",
       },
     ]
 
@@ -168,10 +193,12 @@ describe("Navigation store", () => {
         {
           url: "/home",
           text: "Home",
+          type: "link",
         },
         {
           url: "/last",
           text: "Last Link",
+          type: "link",
         },
       ],
     })
@@ -180,10 +207,7 @@ describe("Navigation store", () => {
   it("Should ignore a request to delete if there are no links", async ctx => {
     const saveSpy = vi.spyOn(ctx.test.navigationStore, "save")
 
-    await ctx.test.navigationStore.deleteLink({
-      url: "/some-link",
-      text: "Some Link",
-    })
+    await ctx.test.navigationStore.deleteLink("/some-link")
 
     expect(saveSpy).not.toBeCalled()
   })
@@ -201,10 +225,18 @@ describe("Navigation store", () => {
         {
           url: "/home",
           text: "Home",
+          type: "link",
         },
         {
           url: "/last",
           text: "Last Link",
+          type: "sublinks",
+          subLinks: [
+            {
+              text: "Foo",
+              url: "/bar",
+            },
+          ],
         },
       ],
     }))
@@ -217,6 +249,7 @@ describe("Navigation store", () => {
         {
           url: "/new-link",
           text: "New Link",
+          type: "link",
         },
       ],
     }

packages/builder/src/stores/builder/tests/sortedIntegrations.test.js

@@ -6,7 +6,10 @@ import { derived } from "svelte/store"
 import { integrations } from "stores/builder/integrations"
 
 vi.mock("svelte/store", () => ({
-  derived: vi.fn(() => {}),
+  derived: vi.fn(),
+  writable: vi.fn(() => ({
+    subscribe: vi.fn(),
+  })),
 }))
 
 vi.mock("stores/builder/integrations", () => ({ integrations: vi.fn() }))

packages/builder/src/stores/portal/auth.js

@@ -2,7 +2,6 @@ import { derived, writable, get } from "svelte/store"
 import { API } from "api"
 import { admin } from "stores/portal"
 import analytics from "analytics"
-import { sdk } from "@budibase/shared-core"
 
 export function createAuthStore() {
   const auth = writable({
@@ -42,20 +41,6 @@ export function createAuthStore() {
         .activate()
         .then(() => {
           analytics.identify(user._id)
-          analytics.showChat(
-            {
-              email: user.email,
-              created_at: (user.createdAt || Date.now()) / 1000,
-              name: user.account?.name,
-              user_id: user._id,
-              tenant: user.tenantId,
-              admin: sdk.users.isAdmin(user),
-              builder: sdk.users.isBuilder(user),
-              "Company size": user.account?.size,
-              "Job role": user.account?.profession,
-            },
-            !!user?.account
-          )
         })
         .catch(() => {
           // This request may fail due to browser extensions blocking requests

packages/builder/tsconfig.json

@@ -1,9 +1,6 @@
 {
   "extends": "./tsconfig.build.json",
   "compilerOptions": {
-    "composite": true,
-    "declaration": true,
-    "sourceMap": true,
     "baseUrl": ".",
     "paths": {
       "assets/*": ["./assets/*"],

packages/builder/vite.config.js

@@ -77,9 +77,6 @@ export default defineConfig(({ mode }) => {
           isProduction ? "production" : "development"
         ),
         "process.env.POSTHOG_TOKEN": JSON.stringify(process.env.POSTHOG_TOKEN),
-        "process.env.INTERCOM_TOKEN": JSON.stringify(
-          process.env.INTERCOM_TOKEN
-        ),
       }),
       copyFonts("fonts"),
       ...(isProduction ? [] : devOnlyPlugins),

packages/client/manifest.json

@@ -4226,7 +4226,7 @@
     ]
   },
   "attachmentfield": {
-    "name": "Attachment",
+    "name": "Attachment list",
     "icon": "Attach",
     "styles": ["size"],
     "requiredAncestors": ["form"],
@@ -4322,6 +4322,97 @@
       }
     ]
   },
+  "attachmentsinglefield": {
+    "name": "Single Attachment",
+    "icon": "Attach",
+    "styles": ["size"],
+    "requiredAncestors": ["form"],
+    "editable": true,
+    "size": {
+      "width": 400,
+      "height": 200
+    },
+    "settings": [
+      {
+        "type": "field/attachment_single",
+        "label": "Field",
+        "key": "field",
+        "required": true
+      },
+      {
+        "type": "text",
+        "label": "Label",
+        "key": "label"
+      },
+      {
+        "type": "text",
+        "label": "Help text",
+        "key": "helpText"
+      },
+      {
+        "type": "text",
+        "label": "Extensions",
+        "key": "extensions"
+      },
+      {
+        "type": "event",
+        "label": "On change",
+        "key": "onChange",
+        "context": [
+          {
+            "label": "Field Value",
+            "key": "value"
+          }
+        ]
+      },
+      {
+        "type": "boolean",
+        "label": "Compact",
+        "key": "compact",
+        "defaultValue": false
+      },
+      {
+        "type": "boolean",
+        "label": "Read only",
+        "key": "disabled",
+        "defaultValue": false
+      },
+      {
+        "type": "validation/attachment",
+        "label": "Validation",
+        "key": "validation"
+      },
+      {
+        "type": "select",
+        "label": "Layout",
+        "key": "span",
+        "defaultValue": 6,
+        "hidden": true,
+        "showInBar": true,
+        "barStyle": "buttons",
+        "options": [
+          {
+            "label": "1 column",
+            "value": 6,
+            "barIcon": "Stop",
+            "barTitle": "1 column"
+          },
+          {
+            "label": "2 columns",
+            "value": 3,
+            "barIcon": "ColumnTwoA",
+            "barTitle": "2 columns"
+          },
+          {
+            "label": "3 columns",
+            "value": 2,
+            "barIcon": "ViewColumn",
+            "barTitle": "3 columns"
+          }
+        ]
+      }
+    ]
+  },
   "relationshipfield": {
     "name": "Relationship Picker",
     "icon": "TaskList",
@@ -4610,6 +4701,35 @@
         "key": "dataSource",
         "required": true
       },
+      {
+        "type": "select",
+        "label": "Auto-refresh",
+        "key": "autoRefresh",
+        "license": "premium",
+        "placeholder": "Never",
+        "options": [
+          {
+            "label": "10 seconds",
+            "value": 10
+          },
+          {
+            "label": "30 seconds",
+            "value": 30
+          },
+          {
+            "label": "1 minute",
+            "value": 60
+          },
+          {
+            "label": "5 minutes",
+            "value": 300
+          },
+          {
+            "label": "10 minutes",
+            "value": 600
+          }
+        ]
+      },
       {
         "type": "filter",
         "label": "Filtering",
@@ -4977,6 +5097,35 @@
         "key": "dataSource",
         "required": true
       },
+      {
+        "type": "select",
+        "label": "Auto-refresh",
+        "key": "autoRefresh",
+        "license": "premium",
+        "placeholder": "Never",
+        "options": [
+          {
+            "label": "10 seconds",
+            "value": 10
+          },
+          {
+            "label": "30 seconds",
+            "value": 30
+          },
+          {
+            "label": "1 minute",
+            "value": 60
+          },
+          {
+            "label": "5 minutes",
+            "value": 300
+          },
+          {
+            "label": "10 minutes",
+            "value": 600
+          }
+        ]
+      },
       {
         "type": "text",
         "label": "Title",
@@ -5445,6 +5594,35 @@
             "key": "dataSource",
             "required": true
           },
+          {
+            "type": "select",
+            "label": "Auto-refresh",
+            "key": "autoRefresh",
+            "license": "premium",
+            "placeholder": "Never",
+            "options": [
+              {
+                "label": "10 seconds",
+                "value": 10
+              },
+              {
+                "label": "30 seconds",
+                "value": 30
+              },
+              {
+                "label": "1 minute",
+                "value": 60
+              },
+              {
+                "label": "5 minutes",
+                "value": 300
+              },
+              {
+                "label": "10 minutes",
+                "value": 600
+              }
+            ]
+          },
           {
             "type": "columns",
             "label": "Columns",
@@ -5731,6 +5909,35 @@
         "key": "dataSource",
         "required": true
       },
+      {
+        "type": "select",
+        "label": "Auto-refresh",
+        "key": "autoRefresh",
+        "license": "premium",
+        "placeholder": "Never",
+        "options": [
+          {
+            "label": "10 seconds",
+            "value": 10
+          },
+          {
+            "label": "30 seconds",
+            "value": 30
+          },
+          {
+            "label": "1 minute",
+            "value": 60
+          },
+          {
+            "label": "5 minutes",
+            "value": 300
+          },
+          {
+            "label": "10 minutes",
+            "value": 600
+          }
+        ]
+      },
       {
         "type": "searchfield",
         "label": "Search columns",
@@ -5895,7 +6102,7 @@
     "block": true,
     "name": "Repeater Block",
     "icon": "ViewList",
-    "illegalChildren": ["section"],
+    "illegalChildren": ["section", "rowexplorer"],
     "hasChildren": true,
     "size": {
       "width": 400,
@@ -5908,6 +6115,35 @@
         "key": "dataSource",
         "required": true
       },
+      {
+        "type": "select",
+        "label": "Auto-refresh",
+        "key": "autoRefresh",
+        "license": "premium",
+        "placeholder": "Never",
+        "options": [
+          {
+            "label": "10 seconds",
+            "value": 10
+          },
+          {
+            "label": "30 seconds",
+            "value": 30
+          },
+          {
+            "label": "1 minute",
+            "value": 60
+          },
+          {
+            "label": "5 minutes",
+            "value": 300
+          },
+          {
+            "label": "10 minutes",
+            "value": 600
+          }
+        ]
+      },
       {
         "type": "filter",
         "label": "Filtering",
@@ -6504,6 +6740,35 @@
         "key": "dataSource",
         "required": true
       },
+      {
+        "type": "select",
+        "label": "Auto-refresh",
+        "key": "autoRefresh",
+        "license": "premium",
+        "placeholder": "Never",
+        "options": [
+          {
+            "label": "10 seconds",
+            "value": 10
+          },
+          {
+            "label": "30 seconds",
+            "value": 30
+          },
+          {
+            "label": "1 minute",
+            "value": 60
+          },
+          {
+            "label": "5 minutes",
+            "value": 300
+          },
+          {
+            "label": "10 minutes",
+            "value": 600
+          }
+        ]
+      },
       {
         "type": "text",
         "label": "Height",

packages/client/src/components/app/DataProvider.svelte

@@ -9,17 +9,18 @@
   export let sortOrder
   export let limit
   export let paginate
+  export let autoRefresh
 
   const { styleable, Provider, ActionTypes, API } = getContext("sdk")
   const component = getContext("component")
 
+  let interval
+  let queryExtensions = {}
+
   // We need to manage our lucene query manually as we want to allow components
   // to extend it
-  let queryExtensions = {}
   $: defaultQuery = LuceneUtils.buildLuceneQuery(filter)
   $: query = extendQuery(defaultQuery, queryExtensions)
-
-  // Fetch data and refresh when needed
   $: fetch = createFetch(dataSource)
   $: fetch.update({
     query,
@@ -28,11 +29,8 @@
     limit,
     paginate,
   })
-
-  // Sanitize schema to remove hidden fields
   $: schema = sanitizeSchema($fetch.schema)
-
-  // Build our action context
+  $: setUpAutoRefresh(autoRefresh)
   $: actions = [
     {
       type: ActionTypes.RefreshDatasource,
@@ -63,8 +61,6 @@
       },
     },
   ]
-
-  // Build our data context
   $: dataContext = {
     rows: $fetch.rows,
     info: $fetch.info,
@@ -140,6 +136,13 @@
     })
     return extendedQuery
   }
+
+  const setUpAutoRefresh = autoRefresh => {
+    clearInterval(interval)
+    if (autoRefresh) {
+      interval = setInterval(fetch.refresh, Math.max(10000, autoRefresh * 1000))
+    }
+  }
 </script>
 
 <div use:styleable={$component.styles} class="container">

packages/client/src/components/app/GridBlock.svelte

@@ -19,6 +19,7 @@
   export let columns = null
   export let onRowClick = null
   export let buttons = null
+  export let repeat = null
 
   const context = getContext("context")
   const component = getContext("component")
@@ -124,6 +125,7 @@
       {fixedRowHeight}
       {columnWhitelist}
       {schemaOverrides}
+      {repeat}
       canAddRows={allowAddRows}
       canEditRows={allowEditRows}
       canDeleteRows={allowDeleteRows}
@@ -147,7 +149,8 @@
     border: 1px solid var(--spectrum-global-color-gray-300);
     border-radius: 4px;
     overflow: hidden;
-    min-height: 410px;
+    min-height: 230px;
+    height: 410px;
   }
   div.in-builder :global(*) {
     pointer-events: none;

packages/client/src/components/app/Layout.svelte

@@ -2,22 +2,20 @@
   import { getContext, setContext } from "svelte"
   import { writable } from "svelte/store"
   import { Heading, Icon, clickOutside } from "@budibase/bbui"
-  import { FieldTypes } from "constants"
   import { Constants } from "@budibase/frontend-core"
-  import active from "svelte-spa-router/active"
+  import NavItem from "./NavItem.svelte"
 
   const sdk = getContext("sdk")
   const {
     routeStore,
     roleStore,
-    styleable,
     linkable,
     builderStore,
     sidePanelStore,
     appStore,
   } = sdk
-  const component = getContext("component")
   const context = getContext("context")
+  const navStateStore = writable({})
 
   // Legacy props which must remain unchanged for backwards compatibility
   export let title
@@ -65,7 +63,7 @@
   })
   setContext("layout", store)
 
-  $: validLinks = getValidLinks(links, $roleStore)
+  $: enrichedNavItems = enrichNavItems(links, $roleStore)
   $: typeClass = NavigationClasses[navigation] || NavigationClasses.None
   $: navWidthClass = WidthClasses[navWidth || width] || WidthClasses.Large
   $: pageWidthClass = WidthClasses[pageWidth || width] || WidthClasses.Large
@@ -103,28 +101,57 @@
     }
   }
 
-  const getValidLinks = (allLinks, userRoleHierarchy) => {
-    // Strip links missing required info
-    let validLinks = (allLinks || []).filter(link => link.text && link.url)
-    // Filter to only links allowed by the current role
-    return validLinks.filter(link => {
-      const role = link.roleId || Constants.Roles.BASIC
-      return userRoleHierarchy?.find(roleId => roleId === role)
-    })
+  const enrichNavItem = navItem => {
+    const internalLink = isInternal(navItem.url)
+    return {
+      ...navItem,
+      internalLink,
+      url: internalLink ? navItem.url : ensureExternal(navItem.url),
+    }
+  }
+
+  const enrichNavItems = (navItems, userRoleHierarchy) => {
+    if (!navItems?.length) {
+      return []
+    }
+    return navItems
+      .filter(navItem => {
+        // Strip nav items without text
+        if (!navItem.text) {
+          return false
+        }
+
+        // Strip out links without URLs
+        if (navItem.type !== "sublinks" && !navItem.url) {
+          return false
+        }
+
+        // Filter to only links allowed by the current role
+        const role = navItem.roleId || Constants.Roles.BASIC
+        return userRoleHierarchy?.find(roleId => roleId === role)
+      })
+      .map(navItem => {
+        const enrichedNavItem = enrichNavItem(navItem)
+        if (navItem.type === "sublinks" && navItem.subLinks?.length) {
+          enrichedNavItem.subLinks = navItem.subLinks
+            .filter(subLink => subLink.text && subLink.url)
+            .map(enrichNavItem)
+        }
+        return enrichedNavItem
+      })
   }
 
   const isInternal = url => {
-    return url.startsWith("/")
+    return url?.startsWith("/")
   }
 
   const ensureExternal = url => {
+    if (!url?.length) {
+      return url
+    }
     return !url.startsWith("http") ? `http://${url}` : url
   }
 
-  const close = () => {
-    mobileOpen = false
-  }
-
   const navigateToPortal = () => {
     if ($builderStore.inBuilder) return
     window.location.href = "/builder/apps"
@@ -169,15 +196,14 @@
 <!-- svelte-ignore a11y-no-static-element-interactions -->
 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <div
-  class="component {screenId} layout layout--{typeClass}"
-  use:styleable={$component.styles}
+  class="component layout layout--{typeClass}"
   class:desktop={!mobile}
   class:mobile={!!mobile}
   data-id={screenId}
   data-name="Screen"
   data-icon="WebPage"
 >
-  <div class="{screenId}-dom screen-wrapper layout-body">
+  <div class="screen-wrapper layout-body">
     {#if typeClass !== "none"}
       <div
         class="interactive component {navigationId}"
@@ -197,7 +223,7 @@
         >
           <div class="nav nav--{typeClass} size--{navWidthClass}">
             <div class="nav-header">
-              {#if validLinks?.length}
+              {#if enrichedNavItems.length}
                 <div class="burger">
                   <Icon
                     hoverable
@@ -246,28 +272,20 @@
               class:visible={mobileOpen}
               on:click={() => (mobileOpen = false)}
             />
-            {#if validLinks?.length}
+            {#if enrichedNavItems.length}
               <div class="links" class:visible={mobileOpen}>
-                {#each validLinks as { text, url }}
-                  {#if isInternal(url)}
-                    <a
-                      class={FieldTypes.LINK}
-                      href={url}
-                      use:linkable
-                      on:click={close}
-                      use:active={url}
-                    >
-                      {text}
-                    </a>
-                  {:else}
-                    <a
-                      class={FieldTypes.LINK}
-                      href={ensureExternal(url)}
-                      on:click={close}
-                    >
-                      {text}
-                    </a>
-                  {/if}
+                {#each enrichedNavItems as navItem}
+                  <NavItem
+                    type={navItem.type}
+                    text={navItem.text}
+                    url={navItem.url}
+                    subLinks={navItem.subLinks}
+                    internalLink={navItem.internalLink}
+                    on:clickLink={() => (mobileOpen = false)}
+                    leftNav={navigation === "Left"}
+                    {mobile}
+                    {navStateStore}
+                  />
                 {/each}
                 <div class="close">
                   <Icon
@@ -282,7 +300,14 @@
         </div>
       </div>
     {/if}
-    <div class="main-wrapper">
+    <div
+      class="main-wrapper"
+      on:click={() => {
+        if ($builderStore.inBuilder) {
+          builderStore.actions.selectComponent(screenId)
+        }
+      }}
+    >
       <div class="main size--{pageWidthClass}">
         <slot />
       </div>
@@ -505,21 +530,6 @@
     gap: var(--spacing-xl);
     margin-top: var(--spacing-xl);
   }
-  .link {
-    opacity: 0.75;
-    color: var(--navTextColor);
-    font-size: var(--spectrum-global-dimension-font-size-200);
-    font-weight: 600;
-    transition: color 130ms ease-out;
-  }
-  .link.active {
-    opacity: 1;
-  }
-  .link:hover {
-    opacity: 1;
-    text-decoration: underline;
-    text-underline-position: under;
-  }
   .close {
     display: none;
     position: absolute;

packages/client/src/components/app/NavItem.svelte

@@ -0,0 +1,173 @@
+<script>
+  import { createEventDispatcher, getContext } from "svelte"
+  import active from "svelte-spa-router/active"
+  import { Icon } from "@budibase/bbui"
+
+  export let type
+  export let url
+  export let text
+  export let subLinks
+  export let internalLink
+  export let leftNav = false
+  export let mobile = false
+  export let navStateStore
+
+  const dispatch = createEventDispatcher()
+  const sdk = getContext("sdk")
+  const { linkable } = sdk
+
+  let renderKey
+
+  $: expanded = !!$navStateStore[text]
+  $: renderLeftNav = leftNav || mobile
+  $: icon = !renderLeftNav || expanded ? "ChevronDown" : "ChevronRight"
+
+  const onClickLink = () => {
+    dispatch("clickLink")
+    renderKey = Math.random()
+  }
+
+  const onClickDropdown = () => {
+    if (!renderLeftNav) {
+      return
+    }
+    navStateStore.update(state => ({
+      ...state,
+      [text]: !state[text],
+    }))
+  }
+</script>
+
+{#if !type || type === "link"}
+  {#if internalLink}
+    <!--
+      It's stupid that we have to add class:active={false} here, but if we don't
+      then svelte will strip out the CSS selector and active links won't be
+      styled
+    -->
+    <a
+      href={url}
+      on:click={onClickLink}
+      use:active={url}
+      use:linkable
+      class:active={false}
+    >
+      {text}
+    </a>
+  {:else}
+    <a href={url} on:click={onClickLink}>
+      {text}
+    </a>
+  {/if}
+{:else}
+  <!-- svelte-ignore a11y-no-static-element-interactions -->
+  <!-- svelte-ignore a11y-click-events-have-key-events -->
+  {#key renderKey}
+    <div class="dropdown" class:left={renderLeftNav} class:expanded>
+      <div class="text" on:click={onClickDropdown}>
+        <span>{text}</span>
+        <Icon name={icon} />
+      </div>
+      <div class="sublinks-wrapper">
+        <div class="sublinks">
+          {#each subLinks || [] as subLink}
+            {#if subLink.internalLink}
+              <a
+                href={subLink.url}
+                on:click={onClickLink}
+                use:active={subLink.url}
+                use:linkable
+              >
+                {subLink.text}
+              </a>
+            {:else}
+              <a href={subLink.url} on:click={onClickLink}>
+                {subLink.text}
+              </a>
+            {/if}
+          {/each}
+        </div>
+      </div>
+    </div>
+  {/key}
+{/if}
+
+<style>
+  /* Generic styles */
+  a,
+  .text span {
+    opacity: 0.75;
+    color: var(--navTextColor);
+    font-size: var(--spectrum-global-dimension-font-size-200);
+    transition: opacity 130ms ease-out;
+    font-weight: 600;
+    user-select: none;
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
+  a.active {
+    opacity: 1;
+  }
+  a:hover,
+  .dropdown:not(.left.expanded):hover .text,
+  .text:hover {
+    cursor: pointer;
+    opacity: 1;
+  }
+
+  /* Top dropdowns */
+  .dropdown {
+    position: relative;
+  }
+  .text {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+    gap: var(--spacing-xs);
+  }
+  .sublinks-wrapper {
+    position: absolute;
+    top: 100%;
+    display: none;
+    padding-top: var(--spacing-s);
+  }
+  .dropdown:hover .sublinks-wrapper {
+    display: block;
+  }
+  .sublinks {
+    display: flex;
+    flex-direction: column;
+    justify-content: stretch;
+    align-items: flex-start;
+    background: var(--spectrum-global-color-gray-50);
+    border-radius: 6px;
+    border: 1px solid var(--spectrum-global-color-gray-300);
+    min-width: 150px;
+    max-width: 250px;
+    padding: 10px 0;
+    overflow: hidden;
+  }
+  .sublinks a {
+    padding: 6px var(--spacing-l);
+    font-weight: 400;
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    width: 100%;
+  }
+
+  /* Left dropdowns */
+  .dropdown.left .sublinks-wrapper {
+    display: none;
+  }
+  .dropdown.left,
+  .dropdown.left.expanded .sublinks-wrapper,
+  .dropdown.dropdown.left.expanded .sublinks {
+    display: contents;
+  }
+  .dropdown.left a {
+    padding-top: 0;
+    padding-bottom: 0;
+  }
+</style>

packages/client/src/components/app/blocks/CardsBlock.svelte

@@ -31,6 +31,7 @@
   export let cardButtonOnClick
   export let linkColumn
   export let noRowsMessage
+  export let autoRefresh
 
   const context = getContext("context")
   const { fetchDatasourceSchema, generateGoldenSample } = getContext("sdk")
@@ -184,6 +185,7 @@
           sortOrder,
           paginate,
           limit,
+          autoRefresh,
         }}
         order={1}
       >

packages/client/src/components/app/blocks/ChartBlock.svelte

@@ -8,6 +8,7 @@
   export let sortColumn
   export let sortOrder
   export let limit
+  export let autoRefresh
 
   // Block
   export let chartTitle
@@ -65,6 +66,7 @@
       sortColumn,
       sortOrder,
       limit,
+      autoRefresh,
     }}
   >
     {#if dataProviderId && chartType}

packages/client/src/components/app/blocks/FormBlockComponent.svelte

@@ -15,7 +15,8 @@
     [FieldType.BOOLEAN]: "booleanfield",
     [FieldType.LONGFORM]: "longformfield",
     [FieldType.DATETIME]: "datetimefield",
-    [FieldType.ATTACHMENT]: "attachmentfield",
+    [FieldType.ATTACHMENTS]: "attachmentfield",
+    [FieldType.ATTACHMENT_SINGLE]: "attachmentsinglefield",
     [FieldType.LINK]: "relationshipfield",
     [FieldType.JSON]: "jsonfield",
     [FieldType.BARCODEQR]: "codescanner",
@@ -60,7 +61,7 @@
 
   function getPropsByType(field) {
     const propsMapByType = {
-      [FieldType.ATTACHMENT]: (_field, schema) => {
+      [FieldType.ATTACHMENTS]: (_field, schema) => {
         return {
           maximum: schema?.constraints?.length?.maximum,
         }

packages/client/src/components/app/blocks/RepeaterBlock.svelte

@@ -17,6 +17,7 @@
   export let hAlign
   export let vAlign
   export let gap
+  export let autoRefresh
 
   const component = getContext("component")
   const context = getContext("context")
@@ -47,6 +48,7 @@
       sortOrder,
       limit,
       paginate,
+      autoRefresh,
     }}
   >
     {#if $component.empty}

packages/client/src/components/app/blocks/RowExplorer.svelte

@@ -16,6 +16,7 @@
   export let detailFields
   export let detailTitle
   export let noRowsMessage
+  export let autoRefresh
 
   const stateKey = generate()
   const context = getContext("context")
@@ -66,6 +67,7 @@
             noValue: false,
           },
         ],
+        autoRefresh,
       }}
       styles={{
         custom: `

packages/client/src/components/app/deprecated/TableBlock.svelte

@@ -33,6 +33,7 @@
   export let sidePanelSaveLabel
   export let sidePanelDeleteLabel
   export let notificationOverride
+  export let autoRefresh
 
   const { fetchDatasourceSchema, API, generateGoldenSample } = getContext("sdk")
   const component = getContext("component")
@@ -243,6 +244,7 @@
           sortOrder,
           paginate,
           limit: rowCount,
+          autoRefresh,
         }}
         context="provider"
         order={1}

packages/client/src/components/app/forms/AttachmentField.svelte

@@ -1,6 +1,7 @@
 <script>
   import Field from "./Field.svelte"
   import { CoreDropzone } from "@budibase/bbui"
+  import { FieldType } from "@budibase/types"
   import { getContext } from "svelte"
 
   export let field
@@ -14,6 +15,12 @@
   export let maximum = undefined
   export let span
   export let helpText = null
+  export let type = FieldType.ATTACHMENTS
+  export let fieldApiMapper = {
+    get: value => value,
+    set: value => value,
+  }
+  export let defaultValue = []
 
   let fieldState
   let fieldApi
@@ -63,9 +70,10 @@
   }
 
   const handleChange = e => {
-    const changed = fieldApi.setValue(e.detail)
+    const value = fieldApiMapper.set(e.detail)
+    const changed = fieldApi.setValue(value)
     if (onChange && changed) {
-      onChange({ value: e.detail })
+      onChange({ value })
     }
   }
 </script>
@@ -78,14 +86,14 @@
   {validation}
   {span}
   {helpText}
-  type="attachment"
+  {type}
   bind:fieldState
   bind:fieldApi
-  defaultValue={[]}
+  {defaultValue}
 >
   {#if fieldState}
     <CoreDropzone
-      value={fieldState.value}
+      value={fieldApiMapper.get(fieldState.value)}
       disabled={fieldState.disabled || fieldState.readonly}
       error={fieldState.error}
       on:change={handleChange}

packages/client/src/components/app/forms/AttachmentSingleField.svelte

@@ -0,0 +1,17 @@
+<script>
+  import { FieldType } from "@budibase/types"
+  import AttachmentField from "./AttachmentField.svelte"
+
+  const fieldApiMapper = {
+    get: value => (!Array.isArray(value) && value ? [value] : value) || [],
+    set: value => value[0] || null,
+  }
+</script>
+
+<AttachmentField
+  {...$$restProps}
+  type={FieldType.ATTACHMENT_SINGLE}
+  maximum={1}
+  defaultValue={null}
+  {fieldApiMapper}
+/>

packages/client/src/components/app/forms/index.js

@@ -9,6 +9,7 @@ export { default as booleanfield } from "./BooleanField.svelte"
 export { default as longformfield } from "./LongFormField.svelte"
 export { default as datetimefield } from "./DateTimeField.svelte"
 export { default as attachmentfield } from "./AttachmentField.svelte"
+export { default as attachmentsinglefield } from "./AttachmentSingleField.svelte"
 export { default as relationshipfield } from "./RelationshipField.svelte"
 export { default as passwordfield } from "./PasswordField.svelte"
 export { default as formstep } from "./FormStep.svelte"

packages/client/src/components/app/forms/validation.js

@@ -192,7 +192,7 @@ const parseType = (value, type) => {
   }
 
   // Parse attachments, treating no elements as null
-  if (type === FieldTypes.ATTACHMENT) {
+  if (type === FieldTypes.ATTACHMENTS) {
     if (!Array.isArray(value) || !value.length) {
       return null
     }

packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte

@@ -10,6 +10,7 @@
   export let invertX = false
   export let invertY = false
   export let schema
+  export let maximum
 
   const { API, notifications } = getContext("grid")
   const imageExtensions = ["png", "tiff", "gif", "raw", "jpg", "jpeg"]
@@ -98,7 +99,7 @@
       {value}
       compact
       on:change={e => onChange(e.detail)}
-      maximum={schema.constraints?.length?.maximum}
+      maximum={maximum || schema.constraints?.length?.maximum}
       {processFiles}
       {deleteAttachments}
       {handleFileTooLarge}

packages/frontend-core/src/components/grid/cells/AttachmentSingleCell.svelte

@@ -0,0 +1,22 @@
+<script>
+  import AttachmentCell from "./AttachmentCell.svelte"
+
+  export let value
+  export let onChange
+  export let api
+
+  $: arrayValue = (!Array.isArray(value) && value ? [value] : value) || []
+
+  $: onFileChange = value => {
+    value = value[0] || null
+    onChange(value)
+  }
+</script>
+
+<AttachmentCell
+  bind:api
+  {...$$restProps}
+  maximum={1}
+  value={arrayValue}
+  onChange={onFileChange}
+/>

packages/frontend-core/src/components/grid/lib/renderers.js

@@ -11,6 +11,7 @@ import BooleanCell from "../cells/BooleanCell.svelte"
 import FormulaCell from "../cells/FormulaCell.svelte"
 import JSONCell from "../cells/JSONCell.svelte"
 import AttachmentCell from "../cells/AttachmentCell.svelte"
+import AttachmentSingleCell from "../cells/AttachmentSingleCell.svelte"
 import BBReferenceCell from "../cells/BBReferenceCell.svelte"
 
 const TypeComponentMap = {
@@ -22,7 +23,8 @@ const TypeComponentMap = {
   [FieldType.ARRAY]: MultiSelectCell,
   [FieldType.NUMBER]: NumberCell,
   [FieldType.BOOLEAN]: BooleanCell,
-  [FieldType.ATTACHMENT]: AttachmentCell,
+  [FieldType.ATTACHMENTS]: AttachmentCell,
+  [FieldType.ATTACHMENT_SINGLE]: AttachmentSingleCell,
   [FieldType.LINK]: RelationshipCell,
   [FieldType.FORMULA]: FormulaCell,
   [FieldType.JSON]: JSONCell,

packages/frontend-core/src/components/grid/lib/utils.js

@@ -1,4 +1,4 @@
-import { FieldType, FieldTypeSubtypes } from "@budibase/types"
+import { TypeIconMap } from "../../../constants"
 
 export const getColor = (idx, opacity = 0.3) => {
   if (idx == null || idx === -1) {
@@ -7,26 +7,6 @@ export const getColor = (idx, opacity = 0.3) => {
   return `hsla(${((idx + 1) * 222) % 360}, 90%, 75%, ${opacity})`
 }
 
-const TypeIconMap = {
-  [FieldType.STRING]: "Text",
-  [FieldType.OPTIONS]: "Dropdown",
-  [FieldType.DATETIME]: "Date",
-  [FieldType.BARCODEQR]: "Camera",
-  [FieldType.LONGFORM]: "TextAlignLeft",
-  [FieldType.ARRAY]: "Dropdown",
-  [FieldType.NUMBER]: "123",
-  [FieldType.BOOLEAN]: "Boolean",
-  [FieldType.ATTACHMENT]: "AppleFiles",
-  [FieldType.LINK]: "DataCorrelated",
-  [FieldType.FORMULA]: "Calculator",
-  [FieldType.JSON]: "Brackets",
-  [FieldType.BIGINT]: "TagBold",
-  [FieldType.BB_REFERENCE]: {
-    [FieldTypeSubtypes.BB_REFERENCE.USER]: "User",
-    [FieldTypeSubtypes.BB_REFERENCE.USERS]: "UserGroup",
-  },
-}
-
 export const getColumnIcon = column => {
   if (column.schema.autocolumn) {
     return "MagicWand"

packages/frontend-core/src/constants.js

@@ -4,6 +4,7 @@
 export { OperatorOptions, SqlNumberTypeRangeMap } from "@budibase/shared-core"
 export { Feature as Features } from "@budibase/types"
 import { BpmCorrelationKey } from "@budibase/shared-core"
+import { FieldType, FieldTypeSubtypes } from "@budibase/types"
 
 // Cookie names
 export const Cookies = {
@@ -113,3 +114,27 @@ export const ContextScopes = {
   Local: "local",
   Global: "global",
 }
+
+export const TypeIconMap = {
+  [FieldType.STRING]: "Text",
+  [FieldType.OPTIONS]: "Dropdown",
+  [FieldType.DATETIME]: "Calendar",
+  [FieldType.BARCODEQR]: "Camera",
+  [FieldType.LONGFORM]: "TextAlignLeft",
+  [FieldType.ARRAY]: "Duplicate",
+  [FieldType.NUMBER]: "123",
+  [FieldType.BOOLEAN]: "Boolean",
+  [FieldType.ATTACHMENTS]: "Attach",
+  [FieldType.ATTACHMENT_SINGLE]: "Attach",
+  [FieldType.LINK]: "DataCorrelated",
+  [FieldType.FORMULA]: "Calculator",
+  [FieldType.JSON]: "Brackets",
+  [FieldType.BIGINT]: "TagBold",
+  [FieldType.AUTO]: "MagicWand",
+  [FieldType.USER]: "User",
+  [FieldType.USERS]: "UserGroup",
+  [FieldType.BB_REFERENCE]: {
+    [FieldTypeSubtypes.BB_REFERENCE.USER]: "User",
+    [FieldTypeSubtypes.BB_REFERENCE.USERS]: "UserGroup",
+  },
+}

packages/pro

@@ -1 +1 @@
-Subproject commit f8e8f87bd52081e1303a5ae92c432ea5b38f3bb4
+Subproject commit ef186d00241f96037f9fd34d7a3826041977ab3a

packages/server/package.json

@@ -83,7 +83,7 @@
     "joi": "17.6.0",
     "js-yaml": "4.1.0",
     "jsonschema": "1.4.0",
-    "knex": "2.4.0",
+    "knex": "2.4.2",
     "koa": "2.13.4",
     "koa-body": "4.2.0",
     "koa-compress": "4.0.1",
@@ -109,6 +109,8 @@
     "server-destroy": "1.0.1",
     "snowflake-promise": "^4.5.0",
     "socket.io": "4.6.1",
+    "sqlite3": "5.1.6",
+    "swagger-parser": "10.0.3",
     "tar": "6.1.15",
     "to-json-schema": "0.2.5",
     "undici": "^6.0.1",

packages/server/specs/openapi.json

@@ -853,6 +853,7 @@
                         "array",
                         "datetime",
                         "attachment",
+                        "attachment_single",
                         "link",
                         "formula",
                         "auto",
@@ -1059,6 +1060,7 @@
                             "array",
                             "datetime",
                             "attachment",
+                            "attachment_single",
                             "link",
                             "formula",
                             "auto",
@@ -1276,6 +1278,7 @@
                               "array",
                               "datetime",
                               "attachment",
+                              "attachment_single",
                               "link",
                               "formula",
                               "auto",
@@ -1752,7 +1755,7 @@
               },
               "fuzzy": {
                 "type": "object",
-                "description": "A fuzzy search, only supported by internal tables."
+                "description": "Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'."
               },
               "range": {
                 "type": "object",
@@ -1786,6 +1789,36 @@
               "oneOf": {
                 "type": "object",
                 "description": "Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2]."
+              },
+              "contains": {
+                "type": "object",
+                "description": "Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all.",
+                "example": {
+                  "arrayColumn": [
+                    "a",
+                    "b"
+                  ]
+                }
+              },
+              "notContains": {
+                "type": "object",
+                "description": "The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response.",
+                "example": {
+                  "arrayColumn": [
+                    "a",
+                    "b"
+                  ]
+                }
+              },
+              "containsAny": {
+                "type": "object",
+                "description": "As with the contains search, only works on array column types and searches for any of the provided values when given an array.",
+                "example": {
+                  "arrayColumn": [
+                    "a",
+                    "b"
+                  ]
+                }
               }
             }
           },

packages/server/specs/openapi.yaml

@@ -775,6 +775,7 @@ components:
                       - array
                       - datetime
                       - attachment
+                      - attachment_single
                       - link
                       - formula
                       - auto
@@ -940,6 +941,7 @@ components:
                           - array
                           - datetime
                           - attachment
+                          - attachment_single
                           - link
                           - formula
                           - auto
@@ -1112,6 +1114,7 @@ components:
                             - array
                             - datetime
                             - attachment
+                            - attachment_single
                             - link
                             - formula
                             - auto
@@ -1492,7 +1495,8 @@ components:
                 description: The value to search for in the column.
             fuzzy:
               type: object
-              description: A fuzzy search, only supported by internal tables.
+              description: Searches for a sub-string within a string column, e.g. searching
+                for 'dib' will match 'Budibase'.
             range:
               type: object
               description: Searches within a range, the format of this must be in the format
@@ -1524,6 +1528,32 @@ components:
               description: Searches for rows which have a column value that is any of the
                 specified values. The format of this must be columnName ->
                 [value1, value2].
+            contains:
+              type: object
+              description: Searches for a value, or set of values in array column types (such
+                as a multi-select). If an array of search options is provided
+                then it must match all.
+              example:
+                arrayColumn:
+                  - a
+                  - b
+            notContains:
+              type: object
+              description: The logical inverse of contains. Only works on array column types.
+                If an array of values is passed, the row must not match any of
+                them to be returned in the response.
+              example:
+                arrayColumn:
+                  - a
+                  - b
+            containsAny:
+              type: object
+              description: As with the contains search, only works on array column types and
+                searches for any of the provided values when given an array.
+              example:
+                arrayColumn:
+                  - a
+                  - b
         paginate:
           type: boolean
           description: Enables pagination, by default this is disabled.

packages/server/specs/resources/misc.ts

@@ -27,7 +27,8 @@ export default new Resource().setSchemas({
           },
           fuzzy: {
             type: "object",
-            description: "A fuzzy search, only supported by internal tables.",
+            description:
+              "Searches for a sub-string within a string column, e.g. searching for 'dib' will match 'Budibase'.",
           },
           range: {
             type: "object",
@@ -67,6 +68,30 @@ export default new Resource().setSchemas({
             description:
               "Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2].",
           },
+          contains: {
+            type: "object",
+            description:
+              "Searches for a value, or set of values in array column types (such as a multi-select). If an array of search options is provided then it must match all.",
+            example: {
+              arrayColumn: ["a", "b"],
+            },
+          },
+          notContains: {
+            type: "object",
+            description:
+              "The logical inverse of contains. Only works on array column types. If an array of values is passed, the row must not match any of them to be returned in the response.",
+            example: {
+              arrayColumn: ["a", "b"],
+            },
+          },
+          containsAny: {
+            type: "object",
+            description:
+              "As with the contains search, only works on array column types and searches for any of the provided values when given an array.",
+            example: {
+              arrayColumn: ["a", "b"],
+            },
+          },
         },
       },
       paginate: {

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -6,12 +6,10 @@ import {
   FieldType,
   FilterType,
   IncludeRelationship,
-  ManyToManyRelationshipFieldMetadata,
   OneToManyRelationshipFieldMetadata,
   Operation,
   PaginationJson,
   RelationshipFieldMetadata,
-  RelationshipsJson,
   Row,
   SearchFilters,
   SortJson,
@@ -23,14 +21,20 @@ import {
   breakExternalTableId,
   breakRowIdField,
   convertRowId,
-  generateRowIdField,
   isRowId,
   isSQL,
+  generateRowIdField,
 } from "../../../integrations/utils"
+import {
+  buildExternalRelationships,
+  buildSqlFieldList,
+  generateIdForRow,
+  sqlOutputProcessing,
+  isManyToMany,
+} from "./utils"
 import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
-import { processDates, processFormulas } from "../../../utilities/rowProcessor"
 import { db as dbCore } from "@budibase/backend-core"
 import AliasTables from "./alias"
 import sdk from "../../../sdk"
@@ -154,7 +158,8 @@ function cleanupConfig(config: RunConfig, table: Table): RunConfig {
   // filter out fields which cannot be keys
   const fieldNames = Object.entries(table.schema)
     .filter(schema => primaryOptions.find(val => val === schema[1].type))
-    .map(([fieldName]) => fieldName)
+    // map to fieldName
+    .map(entry => entry[0])
   const iterateObject = (obj: { [key: string]: any }) => {
     for (let [field, value] of Object.entries(obj)) {
       if (fieldNames.find(name => name === field) && isRowId(value)) {
@@ -183,34 +188,6 @@ function cleanupConfig(config: RunConfig, table: Table): RunConfig {
   return config
 }
 
-function generateIdForRow(
-  row: Row | undefined,
-  table: Table,
-  isLinked: boolean = false
-): string {
-  const primary = table.primary
-  if (!row || !primary) {
-    return ""
-  }
-  // build id array
-  let idParts = []
-  for (let field of primary) {
-    let fieldValue = extractFieldValue({
-      row,
-      tableName: table.name,
-      fieldName: field,
-      isLinked,
-    })
-    if (fieldValue != null) {
-      idParts.push(fieldValue)
-    }
-  }
-  if (idParts.length === 0) {
-    return ""
-  }
-  return generateRowIdField(idParts)
-}
-
 function getEndpoint(tableId: string | undefined, operation: string) {
   if (!tableId) {
     throw new Error("Cannot get endpoint information - no table ID specified")
@@ -223,71 +200,6 @@ function getEndpoint(tableId: string | undefined, operation: string) {
   }
 }
 
-// need to handle table name + field or just field, depending on if relationships used
-function extractFieldValue({
-  row,
-  tableName,
-  fieldName,
-  isLinked,
-}: {
-  row: Row
-  tableName: string
-  fieldName: string
-  isLinked: boolean
-}) {
-  let value = row[`${tableName}.${fieldName}`]
-  if (value == null && !isLinked) {
-    value = row[fieldName]
-  }
-  return value
-}
-
-function basicProcessing({
-  row,
-  table,
-  isLinked,
-}: {
-  row: Row
-  table: Table
-  isLinked: boolean
-}): Row {
-  const thisRow: Row = {}
-  // filter the row down to what is actually the row (not joined)
-  for (let field of Object.values(table.schema)) {
-    const fieldName = field.name
-
-    const value = extractFieldValue({
-      row,
-      tableName: table.name,
-      fieldName,
-      isLinked,
-    })
-
-    // all responses include "select col as table.col" so that overlaps are handled
-    if (value != null) {
-      thisRow[fieldName] = value
-    }
-  }
-  thisRow._id = generateIdForRow(row, table, isLinked)
-  thisRow.tableId = table._id
-  thisRow._rev = "rev"
-  return thisRow
-}
-
-function fixArrayTypes(row: Row, table: Table) {
-  for (let [fieldName, schema] of Object.entries(table.schema)) {
-    if (schema.type === FieldType.ARRAY && typeof row[fieldName] === "string") {
-      try {
-        row[fieldName] = JSON.parse(row[fieldName])
-      } catch (err) {
-        // couldn't convert back to array, ignore
-        delete row[fieldName]
-      }
-    }
-  }
-  return row
-}
-
 function isOneSide(
   field: RelationshipFieldMetadata
 ): field is OneToManyRelationshipFieldMetadata {
@@ -296,12 +208,6 @@ function isOneSide(
   )
 }
 
-function isManyToMany(
-  field: RelationshipFieldMetadata
-): field is ManyToManyRelationshipFieldMetadata {
-  return !!(field as ManyToManyRelationshipFieldMetadata).through
-}
-
 function isEditableColumn(column: FieldSchema) {
   const isExternalAutoColumn =
     column.autocolumn &&
@@ -435,187 +341,6 @@ export class ExternalRequest<T extends Operation> {
     return { row: newRow, manyRelationships }
   }
 
-  async processRelationshipFields(
-    table: Table,
-    row: Row,
-    relationships: RelationshipsJson[]
-  ): Promise<Row> {
-    for (let relationship of relationships) {
-      const linkedTable = this.tables[relationship.tableName]
-      if (!linkedTable || !row[relationship.column]) {
-        continue
-      }
-      for (let key of Object.keys(row[relationship.column])) {
-        let relatedRow: Row = row[relationship.column][key]
-        // add this row as context for the relationship
-        for (let col of Object.values(linkedTable.schema)) {
-          if (col.type === FieldType.LINK && col.tableId === table._id) {
-            relatedRow[col.name] = [row]
-          }
-        }
-        // process additional types
-        relatedRow = processDates(table, relatedRow)
-        relatedRow = await processFormulas(linkedTable, relatedRow)
-        row[relationship.column][key] = relatedRow
-      }
-    }
-    return row
-  }
-
-  /**
-   * This iterates through the returned rows and works out what elements of the rows
-   * actually match up to another row (based on primary keys) - this is pretty specific
-   * to SQL and the way that SQL relationships are returned based on joins.
-   * This is complicated, but the idea is that when a SQL query returns all the relations
-   * will be separate rows, with all of the data in each row. We have to decipher what comes
-   * from where (which tables) and how to convert that into budibase columns.
-   */
-  updateRelationshipColumns(
-    table: Table,
-    row: Row,
-    rows: { [key: string]: Row },
-    relationships: RelationshipsJson[]
-  ) {
-    const columns: { [key: string]: any } = {}
-    for (let relationship of relationships) {
-      const linkedTable = this.tables[relationship.tableName]
-      if (!linkedTable) {
-        continue
-      }
-      const fromColumn = `${table.name}.${relationship.from}`
-      const toColumn = `${linkedTable.name}.${relationship.to}`
-      // this is important when working with multiple relationships
-      // between the same tables, don't want to overlap/multiply the relations
-      if (
-        !relationship.through &&
-        row[fromColumn]?.toString() !== row[toColumn]?.toString()
-      ) {
-        continue
-      }
-
-      let linked = basicProcessing({ row, table: linkedTable, isLinked: true })
-      if (!linked._id) {
-        continue
-      }
-      columns[relationship.column] = linked
-    }
-    for (let [column, related] of Object.entries(columns)) {
-      if (!row._id) {
-        continue
-      }
-      const rowId: string = row._id
-      if (!Array.isArray(rows[rowId][column])) {
-        rows[rowId][column] = []
-      }
-      // make sure relationship hasn't been found already
-      if (
-        !rows[rowId][column].find(
-          (relation: Row) => relation._id === related._id
-        )
-      ) {
-        rows[rowId][column].push(related)
-      }
-    }
-    return rows
-  }
-
-  async outputProcessing(
-    rows: Row[] = [],
-    table: Table,
-    relationships: RelationshipsJson[]
-  ) {
-    if (!rows || rows.length === 0 || rows[0].read === true) {
-      return []
-    }
-    let finalRows: { [key: string]: Row } = {}
-    for (let row of rows) {
-      const rowId = generateIdForRow(row, table)
-      row._id = rowId
-      // this is a relationship of some sort
-      if (finalRows[rowId]) {
-        finalRows = this.updateRelationshipColumns(
-          table,
-          row,
-          finalRows,
-          relationships
-        )
-        continue
-      }
-      const thisRow = fixArrayTypes(
-        basicProcessing({ row, table, isLinked: false }),
-        table
-      )
-      if (thisRow._id == null) {
-        throw "Unable to generate row ID for SQL rows"
-      }
-      finalRows[thisRow._id] = thisRow
-      // do this at end once its been added to the final rows
-      finalRows = this.updateRelationshipColumns(
-        table,
-        row,
-        finalRows,
-        relationships
-      )
-    }
-
-    // make sure all related rows are correct
-    let finalRowArray = []
-    for (let row of Object.values(finalRows)) {
-      finalRowArray.push(
-        await this.processRelationshipFields(table, row, relationships)
-      )
-    }
-
-    // process some additional types
-    finalRowArray = processDates(table, finalRowArray)
-    return finalRowArray
-  }
-
-  /**
-   * Gets the list of relationship JSON structures based on the columns in the table,
-   * this will be used by the underlying library to build whatever relationship mechanism
-   * it has (e.g. SQL joins).
-   */
-  buildRelationships(table: Table): RelationshipsJson[] {
-    const relationships = []
-    for (let [fieldName, field] of Object.entries(table.schema)) {
-      if (field.type !== FieldType.LINK) {
-        continue
-      }
-      const { tableName: linkTableName } = breakExternalTableId(field.tableId)
-      // no table to link to, this is not a valid relationships
-      if (!linkTableName || !this.tables[linkTableName]) {
-        continue
-      }
-      const linkTable = this.tables[linkTableName]
-      if (!table.primary || !linkTable.primary) {
-        continue
-      }
-      const definition: RelationshipsJson = {
-        tableName: linkTableName,
-        // need to specify where to put this back into
-        column: fieldName,
-      }
-      if (isManyToMany(field)) {
-        const { tableName: throughTableName } = breakExternalTableId(
-          field.through
-        )
-        definition.through = throughTableName
-        // don't support composite keys for relationships
-        definition.from = field.throughTo || table.primary[0]
-        definition.to = field.throughFrom || linkTable.primary[0]
-        definition.fromPrimary = table.primary[0]
-        definition.toPrimary = linkTable.primary[0]
-      } else {
-        // if no foreign key specified then use the name of the field in other table
-        definition.from = field.foreignKey || table.primary[0]
-        definition.to = field.fieldName
-      }
-      relationships.push(definition)
-    }
-    return relationships
-  }
-
   /**
    * This is a cached lookup, of relationship records, this is mainly for creating/deleting junction
    * information.
@@ -801,41 +526,6 @@ export class ExternalRequest<T extends Operation> {
     }
   }
 
-  /**
-   * This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which
-   * you have column overlap in relationships, e.g. we join a few different tables and they all have the
-   * concept of an ID, but for some of them it will be null (if they say don't have a relationship).
-   * Creating the specific list of fields that we desire, and excluding the ones that are no use to us
-   * is more performant and has the added benefit of protecting against this scenario.
-   */
-  buildFields(table: Table, includeRelations: boolean) {
-    function extractRealFields(table: Table, existing: string[] = []) {
-      return Object.entries(table.schema)
-        .filter(
-          column =>
-            column[1].type !== FieldType.LINK &&
-            column[1].type !== FieldType.FORMULA &&
-            !existing.find((field: string) => field === column[0])
-        )
-        .map(column => `${table.name}.${column[0]}`)
-    }
-    let fields = extractRealFields(table)
-    for (let field of Object.values(table.schema)) {
-      if (field.type !== FieldType.LINK || !includeRelations) {
-        continue
-      }
-      const { tableName: linkTableName } = breakExternalTableId(field.tableId)
-      if (linkTableName) {
-        const linkTable = this.tables[linkTableName]
-        if (linkTable) {
-          const linkedFields = extractRealFields(linkTable, fields)
-          fields = fields.concat(linkedFields)
-        }
-      }
-    }
-    return fields
-  }
-
   async run(config: RunConfig): Promise<ExternalRequestReturnType<T>> {
     const { operation, tableId } = this
     let { datasourceId, tableName } = breakExternalTableId(tableId)
@@ -869,14 +559,16 @@ export class ExternalRequest<T extends Operation> {
           delete sort?.[sortColumn]
           break
         case FieldType.NUMBER:
-          sort[sortColumn].type = SortType.number
+          if (sort && sort[sortColumn]) {
+            sort[sortColumn].type = SortType.NUMBER
+          }
           break
       }
     }
     filters = buildFilters(id, filters || {}, table)
-    const relationships = this.buildRelationships(table)
+    const relationships = buildExternalRelationships(table, this.tables)
 
-    const includeSqlRelationships =
+    const incRelationships =
       config.includeSqlRelationships === IncludeRelationship.INCLUDE
 
     // clean up row on ingress using schema
@@ -896,7 +588,11 @@ export class ExternalRequest<T extends Operation> {
       },
       resource: {
         // have to specify the fields to avoid column overlap (for SQL)
-        fields: isSql ? this.buildFields(table, includeSqlRelationships) : [],
+        fields: isSql
+          ? buildSqlFieldList(table, this.tables, {
+              relationships: incRelationships,
+            })
+          : [],
       },
       filters,
       sort,
@@ -935,9 +631,10 @@ export class ExternalRequest<T extends Operation> {
         processed.manyRelationships
       )
     }
-    const output = await this.outputProcessing(
-      responseRows,
+    const output = await sqlOutputProcessing(
+      response,
       table,
+      this.tables,
       relationships
     )
     // if reading it'll just be an array of rows, return whole thing

packages/server/src/api/controllers/row/alias.ts

@@ -155,7 +155,9 @@ export default class AliasTables {
     return map
   }
 
-  async queryWithAliasing(json: QueryJson): DatasourcePlusQueryResponse {
+  async queryWithAliasing(
+    json: QueryJson
+  ): Promise<DatasourcePlusQueryResponse> {
     const datasourceId = json.endpoint.datasourceId
     const datasource = await sdk.datasources.get(datasourceId)
 

packages/server/src/api/controllers/row/external.ts

@@ -17,11 +17,9 @@ import {
   Row,
   Table,
   UserCtx,
-  EmptyFilterOption,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import * as utils from "./utils"
-import { dataFilters } from "@budibase/shared-core"
 import {
   inputProcessing,
   outputProcessing,
@@ -33,17 +31,6 @@ export async function handleRequest<T extends Operation>(
   tableId: string,
   opts?: RunConfig
 ): Promise<ExternalRequestReturnType<T>> {
-  // make sure the filters are cleaned up, no empty strings for equals, fuzzy or string
-  if (opts && opts.filters) {
-    opts.filters = sdk.rows.removeEmptyFilters(opts.filters)
-  }
-  if (
-    !dataFilters.hasFilters(opts?.filters) &&
-    opts?.filters?.onEmptyFilter === EmptyFilterOption.RETURN_NONE
-  ) {
-    return [] as any
-  }
-
   return new ExternalRequest<T>(operation, tableId, opts?.datasource).run(
     opts || {}
   )

packages/server/src/api/controllers/row/index.ts

@@ -13,7 +13,7 @@ import {
   PatchRowRequest,
   PatchRowResponse,
   Row,
-  SearchParams,
+  RowSearchParams,
   SearchRowRequest,
   SearchRowResponse,
   UserCtx,
@@ -192,7 +192,7 @@ export async function destroy(ctx: UserCtx<DeleteRowRequest>) {
 export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
   const tableId = utils.getTableId(ctx)
 
-  const searchParams: SearchParams = {
+  const searchParams: RowSearchParams = {
     ...ctx.request.body,
     tableId,
   }

packages/server/src/api/controllers/row/utils.ts

@@ -1,59 +0,0 @@
-import { InternalTables } from "../../../db/utils"
-import * as userController from "../user"
-import { context } from "@budibase/backend-core"
-import { Ctx, Row, UserCtx } from "@budibase/types"
-
-import validateJs from "validate.js"
-
-validateJs.extend(validateJs.validators.datetime, {
-  parse: function (value: string) {
-    return new Date(value).getTime()
-  },
-  // Input is a unix timestamp
-  format: function (value: string) {
-    return new Date(value).toISOString()
-  },
-})
-
-export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
-  const db = context.getAppDB()
-  let row: Row
-  // TODO remove special user case in future
-  if (tableId === InternalTables.USER_METADATA) {
-    ctx.params = {
-      id: rowId,
-    }
-    await userController.findMetadata(ctx)
-    row = ctx.body
-  } else {
-    row = await db.get(rowId)
-  }
-  if (row.tableId !== tableId) {
-    throw "Supplied tableId does not match the rows tableId"
-  }
-  return row
-}
-
-export function getTableId(ctx: Ctx): string {
-  // top priority, use the URL first
-  if (ctx.params?.sourceId) {
-    return ctx.params.sourceId
-  }
-  // now check for old way of specifying table ID
-  if (ctx.params?.tableId) {
-    return ctx.params.tableId
-  }
-  // check body for a table ID
-  if (ctx.request.body?.tableId) {
-    return ctx.request.body.tableId
-  }
-  // now check if a specific view name
-  if (ctx.params?.viewName) {
-    return ctx.params.viewName
-  }
-  throw new Error("Unable to find table ID in request")
-}
-
-export function isUserMetadataTable(tableId: string) {
-  return tableId === InternalTables.USER_METADATA
-}

packages/server/src/api/controllers/row/utils/basic.ts

@@ -0,0 +1,116 @@
+// need to handle table name + field or just field, depending on if relationships used
+import { FieldType, Row, Table } from "@budibase/types"
+import { generateRowIdField } from "../../../../integrations/utils"
+import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
+
+function extractFieldValue({
+  row,
+  tableName,
+  fieldName,
+  isLinked,
+}: {
+  row: Row
+  tableName: string
+  fieldName: string
+  isLinked: boolean
+}) {
+  let value = row[`${tableName}.${fieldName}`]
+  if (value == null && !isLinked) {
+    value = row[fieldName]
+  }
+  return value
+}
+
+export function getInternalRowId(row: Row, table: Table): string {
+  return extractFieldValue({
+    row,
+    tableName: table._id!,
+    fieldName: "_id",
+    isLinked: false,
+  })
+}
+
+export function generateIdForRow(
+  row: Row | undefined,
+  table: Table,
+  isLinked: boolean = false
+): string {
+  const primary = table.primary
+  if (!row || !primary) {
+    return ""
+  }
+  // build id array
+  let idParts = []
+  for (let field of primary) {
+    let fieldValue = extractFieldValue({
+      row,
+      tableName: table.name,
+      fieldName: field,
+      isLinked,
+    })
+    if (fieldValue != null) {
+      idParts.push(fieldValue)
+    }
+  }
+  if (idParts.length === 0) {
+    return ""
+  }
+  return generateRowIdField(idParts)
+}
+
+export function basicProcessing({
+  row,
+  table,
+  isLinked,
+  internal,
+}: {
+  row: Row
+  table: Table
+  isLinked: boolean
+  internal?: boolean
+}): Row {
+  const thisRow: Row = {}
+  // filter the row down to what is actually the row (not joined)
+  for (let field of Object.values(table.schema)) {
+    const fieldName = field.name
+    const value = extractFieldValue({
+      row,
+      tableName: table.name,
+      fieldName,
+      isLinked,
+    })
+    // all responses include "select col as table.col" so that overlaps are handled
+    if (value != null) {
+      thisRow[fieldName] = value
+    }
+  }
+  if (!internal) {
+    thisRow._id = generateIdForRow(row, table, isLinked)
+    thisRow.tableId = table._id
+    thisRow._rev = "rev"
+  } else {
+    for (let internalColumn of CONSTANT_INTERNAL_ROW_COLS) {
+      thisRow[internalColumn] = extractFieldValue({
+        row,
+        tableName: table._id!,
+        fieldName: internalColumn,
+        isLinked: false,
+      })
+    }
+  }
+  return thisRow
+}
+
+export function fixArrayTypes(row: Row, table: Table) {
+  for (let [fieldName, schema] of Object.entries(table.schema)) {
+    if (schema.type === FieldType.ARRAY && typeof row[fieldName] === "string") {
+      try {
+        row[fieldName] = JSON.parse(row[fieldName])
+      } catch (err) {
+        // couldn't convert back to array, ignore
+        delete row[fieldName]
+      }
+    }
+  }
+  return row
+}

packages/server/src/api/controllers/row/utils/index.ts

@@ -0,0 +1,3 @@
+export * from "./basic"
+export * from "./sqlUtils"
+export * from "./utils"

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -0,0 +1,194 @@
+import {
+  FieldType,
+  ManyToManyRelationshipFieldMetadata,
+  RelationshipFieldMetadata,
+  RelationshipsJson,
+  Row,
+  Table,
+} from "@budibase/types"
+import { breakExternalTableId } from "../../../../integrations/utils"
+import { basicProcessing } from "./basic"
+import { generateJunctionTableID } from "../../../../db/utils"
+
+type TableMap = Record<string, Table>
+
+export function isManyToMany(
+  field: RelationshipFieldMetadata
+): field is ManyToManyRelationshipFieldMetadata {
+  return !!(field as ManyToManyRelationshipFieldMetadata).through
+}
+
+/**
+ * This iterates through the returned rows and works out what elements of the rows
+ * actually match up to another row (based on primary keys) - this is pretty specific
+ * to SQL and the way that SQL relationships are returned based on joins.
+ * This is complicated, but the idea is that when a SQL query returns all the relations
+ * will be separate rows, with all of the data in each row. We have to decipher what comes
+ * from where (which tables) and how to convert that into budibase columns.
+ */
+export async function updateRelationshipColumns(
+  table: Table,
+  tables: TableMap,
+  row: Row,
+  rows: { [key: string]: Row },
+  relationships: RelationshipsJson[],
+  opts?: { sqs?: boolean }
+) {
+  const columns: { [key: string]: any } = {}
+  for (let relationship of relationships) {
+    const linkedTable = tables[relationship.tableName]
+    if (!linkedTable) {
+      continue
+    }
+    const fromColumn = `${table.name}.${relationship.from}`
+    const toColumn = `${linkedTable.name}.${relationship.to}`
+    // this is important when working with multiple relationships
+    // between the same tables, don't want to overlap/multiply the relations
+    if (
+      !relationship.through &&
+      row[fromColumn]?.toString() !== row[toColumn]?.toString()
+    ) {
+      continue
+    }
+
+    let linked = await basicProcessing({
+      row,
+      table: linkedTable,
+      isLinked: true,
+      internal: opts?.sqs,
+    })
+    if (!linked._id) {
+      continue
+    }
+    columns[relationship.column] = linked
+  }
+  for (let [column, related] of Object.entries(columns)) {
+    if (!row._id) {
+      continue
+    }
+    const rowId: string = row._id
+    if (!Array.isArray(rows[rowId][column])) {
+      rows[rowId][column] = []
+    }
+    // make sure relationship hasn't been found already
+    if (
+      !rows[rowId][column].find((relation: Row) => relation._id === related._id)
+    ) {
+      rows[rowId][column].push(related)
+    }
+  }
+  return rows
+}
+
+/**
+ * Gets the list of relationship JSON structures based on the columns in the table,
+ * this will be used by the underlying library to build whatever relationship mechanism
+ * it has (e.g. SQL joins).
+ */
+export function buildExternalRelationships(
+  table: Table,
+  tables: TableMap
+): RelationshipsJson[] {
+  const relationships = []
+  for (let [fieldName, field] of Object.entries(table.schema)) {
+    if (field.type !== FieldType.LINK) {
+      continue
+    }
+    const { tableName: linkTableName } = breakExternalTableId(field.tableId)
+    // no table to link to, this is not a valid relationships
+    if (!linkTableName || !tables[linkTableName]) {
+      continue
+    }
+    const linkTable = tables[linkTableName]
+    if (!table.primary || !linkTable.primary) {
+      continue
+    }
+    const definition: RelationshipsJson = {
+      tableName: linkTableName,
+      // need to specify where to put this back into
+      column: fieldName,
+    }
+    if (isManyToMany(field)) {
+      const { tableName: throughTableName } = breakExternalTableId(
+        field.through
+      )
+      definition.through = throughTableName
+      // don't support composite keys for relationships
+      definition.from = field.throughTo || table.primary[0]
+      definition.to = field.throughFrom || linkTable.primary[0]
+      definition.fromPrimary = table.primary[0]
+      definition.toPrimary = linkTable.primary[0]
+    } else {
+      // if no foreign key specified then use the name of the field in other table
+      definition.from = field.foreignKey || table.primary[0]
+      definition.to = field.fieldName
+    }
+    relationships.push(definition)
+  }
+  return relationships
+}
+
+export function buildInternalRelationships(table: Table): RelationshipsJson[] {
+  const relationships: RelationshipsJson[] = []
+  const links = Object.values(table.schema).filter(
+    column => column.type === FieldType.LINK
+  )
+  const tableId = table._id!
+  for (let link of links) {
+    if (link.type !== FieldType.LINK) {
+      continue
+    }
+    const linkTableId = link.tableId!
+    const junctionTableId = generateJunctionTableID(tableId, linkTableId)
+    const isFirstTable = tableId > linkTableId
+    relationships.push({
+      through: junctionTableId,
+      column: link.name,
+      tableName: linkTableId,
+      fromPrimary: "_id",
+      to: isFirstTable ? "doc2.rowId" : "doc1.rowId",
+      from: isFirstTable ? "doc1.rowId" : "doc2.rowId",
+      toPrimary: "_id",
+    })
+  }
+  return relationships
+}
+
+/**
+ * This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which
+ * you have column overlap in relationships, e.g. we join a few different tables and they all have the
+ * concept of an ID, but for some of them it will be null (if they say don't have a relationship).
+ * Creating the specific list of fields that we desire, and excluding the ones that are no use to us
+ * is more performant and has the added benefit of protecting against this scenario.
+ */
+export function buildSqlFieldList(
+  table: Table,
+  tables: TableMap,
+  opts?: { relationships: boolean }
+) {
+  function extractRealFields(table: Table, existing: string[] = []) {
+    return Object.entries(table.schema)
+      .filter(
+        column =>
+          column[1].type !== FieldType.LINK &&
+          column[1].type !== FieldType.FORMULA &&
+          !existing.find((field: string) => field === column[0])
+      )
+      .map(column => `${table.name}.${column[0]}`)
+  }
+  let fields = extractRealFields(table)
+  for (let field of Object.values(table.schema)) {
+    if (field.type !== FieldType.LINK || !opts?.relationships) {
+      continue
+    }
+    const { tableName: linkTableName } = breakExternalTableId(field.tableId)
+    if (linkTableName) {
+      const linkTable = tables[linkTableName]
+      if (linkTable) {
+        const linkedFields = extractRealFields(linkTable, fields)
+        fields = fields.concat(linkedFields)
+      }
+    }
+  }
+  return fields
+}

packages/server/src/api/controllers/row/utils/utils.ts

@@ -0,0 +1,189 @@
+import { InternalTables } from "../../../../db/utils"
+import * as userController from "../../user"
+import { context } from "@budibase/backend-core"
+import {
+  Ctx,
+  DatasourcePlusQueryResponse,
+  FieldType,
+  RelationshipsJson,
+  Row,
+  Table,
+  UserCtx,
+} from "@budibase/types"
+import {
+  processDates,
+  processFormulas,
+} from "../../../../utilities/rowProcessor"
+import { updateRelationshipColumns } from "./sqlUtils"
+import {
+  basicProcessing,
+  generateIdForRow,
+  fixArrayTypes,
+  getInternalRowId,
+} from "./basic"
+import sdk from "../../../../sdk"
+
+import validateJs from "validate.js"
+
+validateJs.extend(validateJs.validators.datetime, {
+  parse: function (value: string) {
+    return new Date(value).getTime()
+  },
+  // Input is a unix timestamp
+  format: function (value: string) {
+    return new Date(value).toISOString()
+  },
+})
+
+export async function processRelationshipFields(
+  table: Table,
+  tables: Record<string, Table>,
+  row: Row,
+  relationships: RelationshipsJson[]
+): Promise<Row> {
+  for (let relationship of relationships) {
+    const linkedTable = tables[relationship.tableName]
+    if (!linkedTable || !row[relationship.column]) {
+      continue
+    }
+    for (let key of Object.keys(row[relationship.column])) {
+      let relatedRow: Row = row[relationship.column][key]
+      // add this row as context for the relationship
+      for (let col of Object.values(linkedTable.schema)) {
+        if (col.type === FieldType.LINK && col.tableId === table._id) {
+          relatedRow[col.name] = [row]
+        }
+      }
+      // process additional types
+      relatedRow = processDates(table, relatedRow)
+      relatedRow = await processFormulas(linkedTable, relatedRow)
+      row[relationship.column][key] = relatedRow
+    }
+  }
+  return row
+}
+
+export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
+  const db = context.getAppDB()
+  let row: Row
+  // TODO remove special user case in future
+  if (tableId === InternalTables.USER_METADATA) {
+    ctx.params = {
+      id: rowId,
+    }
+    await userController.findMetadata(ctx)
+    row = ctx.body
+  } else {
+    row = await db.get(rowId)
+  }
+  if (row.tableId !== tableId) {
+    throw "Supplied tableId does not match the rows tableId"
+  }
+  return row
+}
+
+export function getTableId(ctx: Ctx): string {
+  // top priority, use the URL first
+  if (ctx.params?.sourceId) {
+    return ctx.params.sourceId
+  }
+  // now check for old way of specifying table ID
+  if (ctx.params?.tableId) {
+    return ctx.params.tableId
+  }
+  // check body for a table ID
+  if (ctx.request.body?.tableId) {
+    return ctx.request.body.tableId
+  }
+  // now check if a specific view name
+  if (ctx.params?.viewName) {
+    return ctx.params.viewName
+  }
+  throw new Error("Unable to find table ID in request")
+}
+
+export async function validate(
+  opts: { row: Row } & ({ tableId: string } | { table: Table })
+) {
+  let fetchedTable: Table
+  if ("tableId" in opts) {
+    fetchedTable = await sdk.tables.getTable(opts.tableId)
+  } else {
+    fetchedTable = opts.table
+  }
+  return sdk.rows.utils.validate({
+    ...opts,
+    table: fetchedTable,
+  })
+}
+
+export async function sqlOutputProcessing(
+  rows: DatasourcePlusQueryResponse,
+  table: Table,
+  tables: Record<string, Table>,
+  relationships: RelationshipsJson[],
+  opts?: { sqs?: boolean }
+): Promise<Row[]> {
+  if (!Array.isArray(rows) || rows.length === 0 || rows[0].read === true) {
+    return []
+  }
+  let finalRows: { [key: string]: Row } = {}
+  for (let row of rows as Row[]) {
+    let rowId = row._id
+    if (opts?.sqs) {
+      rowId = getInternalRowId(row, table)
+    } else if (!rowId) {
+      rowId = generateIdForRow(row, table)
+      row._id = rowId
+    }
+    // this is a relationship of some sort
+    if (finalRows[rowId]) {
+      finalRows = await updateRelationshipColumns(
+        table,
+        tables,
+        row,
+        finalRows,
+        relationships,
+        opts
+      )
+      continue
+    }
+    const thisRow = fixArrayTypes(
+      basicProcessing({
+        row,
+        table,
+        isLinked: false,
+        internal: opts?.sqs,
+      }),
+      table
+    )
+    if (thisRow._id == null) {
+      throw new Error("Unable to generate row ID for SQL rows")
+    }
+    finalRows[thisRow._id] = thisRow
+    // do this at end once its been added to the final rows
+    finalRows = await updateRelationshipColumns(
+      table,
+      tables,
+      row,
+      finalRows,
+      relationships
+    )
+  }
+
+  // make sure all related rows are correct
+  let finalRowArray = []
+  for (let row of Object.values(finalRows)) {
+    finalRowArray.push(
+      await processRelationshipFields(table, tables, row, relationships)
+    )
+  }
+
+  // process some additional types
+  finalRowArray = processDates(table, finalRowArray)
+  return finalRowArray
+}
+
+export function isUserMetadataTable(tableId: string) {
+  return tableId === InternalTables.USER_METADATA
+}

packages/server/src/api/controllers/row/views.ts

@@ -4,8 +4,8 @@ import {
   SearchRowResponse,
   SearchViewRowRequest,
   RequiredKeys,
-  SearchParams,
   SearchFilters,
+  RowSearchParams,
 } from "@budibase/types"
 import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../../sdk"
@@ -57,7 +57,7 @@ export async function searchView(
   }
 
   const searchOptions: RequiredKeys<SearchViewRowRequest> &
-    RequiredKeys<Pick<SearchParams, "tableId" | "query" | "fields">> = {
+    RequiredKeys<Pick<RowSearchParams, "tableId" | "query" | "fields">> = {
     tableId: view.tableId,
     query,
     fields: viewFields,

packages/server/src/api/controllers/static/index.ts

@@ -2,7 +2,7 @@ import { InvalidFileExtensions } from "@budibase/shared-core"
 import AppComponent from "./templates/BudibaseApp.svelte"
 import { join } from "../../../utilities/centralPath"
 import * as uuid from "uuid"
-import { ObjectStoreBuckets, devClientVersion } from "../../../constants"
+import { devClientVersion, ObjectStoreBuckets } from "../../../constants"
 import { processString } from "@budibase/string-templates"
 import {
   loadHandlebarsFile,
@@ -10,24 +10,24 @@ import {
   TOP_LEVEL_PATH,
 } from "../../../utilities/fileSystem"
 import env from "../../../environment"
-import { DocumentType } from "../../../db/utils"
 import {
+  BadRequestError,
+  configs,
   context,
   objectStore,
   utils,
-  configs,
-  BadRequestError,
 } from "@budibase/backend-core"
 import AWS from "aws-sdk"
 import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
 import {
-  UserCtx,
   App,
   Ctx,
-  ProcessAttachmentResponse,
+  DocumentType,
   Feature,
+  ProcessAttachmentResponse,
+  UserCtx,
 } from "@budibase/types"
 import {
   getAppMigrationVersion,
@@ -147,8 +147,7 @@ const requiresMigration = async (ctx: Ctx) => {
 
   const latestMigrationApplied = await getAppMigrationVersion(appId)
 
-  const requiresMigrations = latestMigrationApplied !== latestMigration
-  return requiresMigrations
+  return latestMigrationApplied !== latestMigration
 }
 
 export const serveApp = async function (ctx: UserCtx) {

packages/server/src/api/controllers/table/utils.ts

@@ -30,9 +30,9 @@ import {
   View,
   RelationshipFieldMetadata,
   FieldType,
-  FieldTypeSubtypes,
-  AttachmentFieldMetadata,
 } from "@budibase/types"
+import sdk from "../../../sdk"
+import env from "../../../environment"
 
 export async function clearColumns(table: Table, columnNames: string[]) {
   const db = context.getAppDB()
@@ -91,26 +91,6 @@ export async function checkForColumnUpdates(
     await checkForViewUpdates(updatedTable, deletedColumns, columnRename)
   }
 
-  const changedAttachmentSubtypeColumns = Object.values(
-    updatedTable.schema
-  ).filter(
-    (column): column is AttachmentFieldMetadata =>
-      column.type === FieldType.ATTACHMENT &&
-      column.subtype !== oldTable?.schema[column.name]?.subtype
-  )
-  for (const attachmentColumn of changedAttachmentSubtypeColumns) {
-    if (attachmentColumn.subtype === FieldTypeSubtypes.ATTACHMENT.SINGLE) {
-      attachmentColumn.constraints ??= { length: {} }
-      attachmentColumn.constraints.length ??= {}
-      attachmentColumn.constraints.length.maximum = 1
-      attachmentColumn.constraints.length.message =
-        "cannot contain multiple files"
-    } else {
-      delete attachmentColumn.constraints?.length?.maximum
-      delete attachmentColumn.constraints?.length?.message
-    }
-  }
-
   return { rows: updatedRows, table: updatedTable }
 }
 
@@ -342,6 +322,9 @@ class TableSaveFunctions {
       importRows: this.importRows,
       user: this.user,
     })
+    if (env.SQS_SEARCH_ENABLE) {
+      await sdk.tables.sqs.addTableToSqlite(table)
+    }
     return table
   }
 

packages/server/src/api/controllers/view/utils.ts

@@ -2,13 +2,18 @@ import {
   ViewName,
   generateMemoryViewID,
   getMemoryViewParams,
-  DocumentType,
   SEPARATOR,
 } from "../../../db/utils"
 import env from "../../../environment"
 import { context } from "@budibase/backend-core"
 import viewBuilder from "./viewBuilder"
-import { Database, DBView, DesignDocument, InMemoryView } from "@budibase/types"
+import {
+  Database,
+  DBView,
+  DocumentType,
+  DesignDocument,
+  InMemoryView,
+} from "@budibase/types"
 
 export async function getView(viewName: string) {
   const db = context.getAppDB()

packages/server/src/api/controllers/view/views.ts

@@ -4,7 +4,6 @@ import { csv, json, jsonWithSchema, Format, isFormat } from "./exporters"
 import { deleteView, getView, getViews, saveView } from "./utils"
 import { fetchView } from "../row"
 import { context, events } from "@budibase/backend-core"
-import { DocumentType } from "../../../db/utils"
 import sdk from "../../../sdk"
 import {
   FieldType,
@@ -14,6 +13,7 @@ import {
   TableExportFormat,
   TableSchema,
   View,
+  DocumentType,
 } from "@budibase/types"
 import { builderSocket } from "../../../websockets"
 

packages/server/src/api/routes/tests/row.spec.ts

@@ -6,14 +6,17 @@ import * as setup from "./utilities"
 import { context, InternalTable, tenancy } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
+  AttachmentFieldMetadata,
   AutoFieldSubType,
   Datasource,
+  DateFieldMetadata,
   DeleteRow,
   FieldSchema,
   FieldType,
   FieldTypeSubtypes,
   FormulaType,
   INTERNAL_TABLE_SOURCE_ID,
+  NumberFieldMetadata,
   QuotaUsageType,
   RelationshipType,
   Row,
@@ -232,9 +235,14 @@ describe.each([
           name: "str",
           constraints: { type: "string", presence: false },
         }
-        const attachment: FieldSchema = {
-          type: FieldType.ATTACHMENT,
-          name: "attachment",
+        const singleAttachment: FieldSchema = {
+          type: FieldType.ATTACHMENT_SINGLE,
+          name: "single attachment",
+          constraints: { presence: false },
+        }
+        const attachmentList: AttachmentFieldMetadata = {
+          type: FieldType.ATTACHMENTS,
+          name: "attachments",
           constraints: { type: "array", presence: false },
         }
         const bool: FieldSchema = {
@@ -242,12 +250,12 @@ describe.each([
           name: "boolean",
           constraints: { type: "boolean", presence: false },
         }
-        const number: FieldSchema = {
+        const number: NumberFieldMetadata = {
           type: FieldType.NUMBER,
           name: "str",
           constraints: { type: "number", presence: false },
         }
-        const datetime: FieldSchema = {
+        const datetime: DateFieldMetadata = {
           type: FieldType.DATETIME,
           name: "datetime",
           constraints: {
@@ -297,10 +305,12 @@ describe.each([
               boolUndefined: bool,
               boolString: bool,
               boolBool: bool,
-              attachmentNull: attachment,
-              attachmentUndefined: attachment,
-              attachmentEmpty: attachment,
-              attachmentEmptyArrayStr: attachment,
+              singleAttachmentNull: singleAttachment,
+              singleAttachmentUndefined: singleAttachment,
+              attachmentListNull: attachmentList,
+              attachmentListUndefined: attachmentList,
+              attachmentListEmpty: attachmentList,
+              attachmentListEmptyArrayStr: attachmentList,
               arrayFieldEmptyArrayStr: arrayField,
               arrayFieldArrayStrKnown: arrayField,
               arrayFieldNull: arrayField,
@@ -336,10 +346,12 @@ describe.each([
           boolString: "true",
           boolBool: true,
           tableId: table._id,
-          attachmentNull: null,
-          attachmentUndefined: undefined,
-          attachmentEmpty: "",
-          attachmentEmptyArrayStr: "[]",
+          singleAttachmentNull: null,
+          singleAttachmentUndefined: undefined,
+          attachmentListNull: null,
+          attachmentListUndefined: undefined,
+          attachmentListEmpty: "",
+          attachmentListEmptyArrayStr: "[]",
           arrayFieldEmptyArrayStr: "[]",
           arrayFieldUndefined: undefined,
           arrayFieldNull: null,
@@ -368,10 +380,12 @@ describe.each([
         expect(row.boolUndefined).toBe(undefined)
         expect(row.boolString).toBe(true)
         expect(row.boolBool).toBe(true)
-        expect(row.attachmentNull).toEqual([])
-        expect(row.attachmentUndefined).toBe(undefined)
-        expect(row.attachmentEmpty).toEqual([])
-        expect(row.attachmentEmptyArrayStr).toEqual([])
+        expect(row.singleAttachmentNull).toEqual(null)
+        expect(row.singleAttachmentUndefined).toBe(undefined)
+        expect(row.attachmentListNull).toEqual([])
+        expect(row.attachmentListUndefined).toBe(undefined)
+        expect(row.attachmentListEmpty).toEqual([])
+        expect(row.attachmentListEmptyArrayStr).toEqual([])
         expect(row.arrayFieldEmptyArrayStr).toEqual([])
         expect(row.arrayFieldNull).toEqual([])
         expect(row.arrayFieldUndefined).toEqual(undefined)
@@ -817,12 +831,44 @@ describe.each([
 
   isInternal &&
     describe("attachments", () => {
-      it("should allow enriching attachment rows", async () => {
+      it("should allow enriching single attachment rows", async () => {
         const table = await config.api.table.save(
           defaultTable({
             schema: {
               attachment: {
-                type: FieldType.ATTACHMENT,
+                type: FieldType.ATTACHMENT_SINGLE,
+                name: "attachment",
+                constraints: { presence: false },
+              },
+            },
+          })
+        )
+        const attachmentId = `${uuid.v4()}.csv`
+        const row = await config.api.row.save(table._id!, {
+          name: "test",
+          description: "test",
+          attachment: {
+            key: `${config.getAppId()}/attachments/${attachmentId}`,
+          },
+
+          tableId: table._id,
+        })
+        await config.withEnv({ SELF_HOSTED: "true" }, async () => {
+          return context.doInAppContext(config.getAppId(), async () => {
+            const enriched = await outputProcessing(table, [row])
+            expect((enriched as Row[])[0].attachment.url).toBe(
+              `/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
+            )
+          })
+        })
+      })
+
+      it("should allow enriching attachment list rows", async () => {
+        const table = await config.api.table.save(
+          defaultTable({
+            schema: {
+              attachment: {
+                type: FieldType.ATTACHMENTS,
                 name: "attachment",
                 constraints: { type: "array", presence: false },
               },
@@ -1272,7 +1318,6 @@ describe.each([
           ? {}
           : {
               hasNextPage: false,
-              bookmark: null,
             }),
       })
     })

packages/server/src/api/routes/tests/search.spec.ts

@@ -0,0 +1,277 @@
+import { tableForDatasource } from "../../../tests/utilities/structures"
+import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
+
+import * as setup from "./utilities"
+import {
+  Datasource,
+  EmptyFilterOption,
+  FieldType,
+  Row,
+  SearchFilters,
+  Table,
+} from "@budibase/types"
+
+jest.unmock("mssql")
+
+describe.each([
+  ["internal", undefined],
+  ["internal-sqs", undefined],
+  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+])("/api/:sourceId/search (%s)", (name, dsProvider) => {
+  const isSqs = name === "internal-sqs"
+  const config = setup.getConfig()
+
+  let envCleanup: (() => void) | undefined
+  let table: Table
+  let datasource: Datasource | undefined
+
+  beforeAll(async () => {
+    if (isSqs) {
+      envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" })
+    }
+    await config.init()
+    if (dsProvider) {
+      datasource = await config.createDatasource({
+        datasource: await dsProvider,
+      })
+    }
+  })
+
+  afterAll(async () => {
+    setup.afterAll()
+    if (envCleanup) {
+      envCleanup()
+    }
+  })
+
+  describe("strings", () => {
+    beforeAll(async () => {
+      table = await config.api.table.save(
+        tableForDatasource(datasource, {
+          schema: {
+            name: {
+              name: "name",
+              type: FieldType.STRING,
+            },
+          },
+        })
+      )
+    })
+
+    const rows = [{ name: "foo" }, { name: "bar" }]
+    let savedRows: Row[]
+
+    beforeAll(async () => {
+      savedRows = await Promise.all(
+        rows.map(r => config.api.row.save(table._id!, r))
+      )
+    })
+
+    interface StringSearchTest {
+      query: SearchFilters
+      expected: (typeof rows)[number][]
+    }
+
+    const stringSearchTests: StringSearchTest[] = [
+      // These three test cases are generic and don't really need
+      // to be repeated for all data types, so we just do them here.
+      { query: {}, expected: rows },
+      {
+        query: { onEmptyFilter: EmptyFilterOption.RETURN_ALL },
+        expected: rows,
+      },
+      {
+        query: { onEmptyFilter: EmptyFilterOption.RETURN_NONE },
+        expected: [],
+      },
+      // The rest of these tests are specific to strings.
+      { query: { string: { name: "foo" } }, expected: [rows[0]] },
+      { query: { string: { name: "none" } }, expected: [] },
+      { query: { fuzzy: { name: "oo" } }, expected: [rows[0]] },
+      { query: { equal: { name: "foo" } }, expected: [rows[0]] },
+      { query: { notEqual: { name: "foo" } }, expected: [rows[1]] },
+      { query: { oneOf: { name: ["foo"] } }, expected: [rows[0]] },
+    ]
+
+    it.each(stringSearchTests)(
+      `should be able to run query: $query`,
+      async ({ query, expected }) => {
+        const { rows: foundRows } = await config.api.row.search(table._id!, {
+          tableId: table._id!,
+          query,
+        })
+        expect(foundRows).toHaveLength(expected.length)
+        expect(foundRows).toEqual(
+          expect.arrayContaining(
+            expected.map(r =>
+              expect.objectContaining(savedRows.find(sr => sr.name === r.name)!)
+            )
+          )
+        )
+      }
+    )
+  })
+
+  describe("number", () => {
+    beforeAll(async () => {
+      table = await config.api.table.save(
+        tableForDatasource(datasource, {
+          schema: {
+            age: {
+              name: "age",
+              type: FieldType.NUMBER,
+            },
+          },
+        })
+      )
+    })
+
+    const rows = [{ age: 1 }, { age: 10 }]
+    let savedRows: Row[]
+
+    beforeAll(async () => {
+      savedRows = await Promise.all(
+        rows.map(r => config.api.row.save(table._id!, r))
+      )
+    })
+
+    interface NumberSearchTest {
+      query: SearchFilters
+      expected: (typeof rows)[number][]
+    }
+
+    const numberSearchTests: NumberSearchTest[] = [
+      { query: { equal: { age: 1 } }, expected: [rows[0]] },
+      { query: { equal: { age: 2 } }, expected: [] },
+      { query: { notEqual: { age: 1 } }, expected: [rows[1]] },
+      { query: { oneOf: { age: [1] } }, expected: [rows[0]] },
+      { query: { range: { age: { low: 1, high: 5 } } }, expected: [rows[0]] },
+      { query: { range: { age: { low: 0, high: 1 } } }, expected: [rows[0]] },
+      { query: { range: { age: { low: 3, high: 4 } } }, expected: [] },
+      { query: { range: { age: { low: 0, high: 11 } } }, expected: rows },
+    ]
+
+    it.each(numberSearchTests)(
+      `should be able to run query: $query`,
+      async ({ query, expected }) => {
+        const { rows: foundRows } = await config.api.row.search(table._id!, {
+          tableId: table._id!,
+          query,
+        })
+        expect(foundRows).toHaveLength(expected.length)
+        expect(foundRows).toEqual(
+          expect.arrayContaining(
+            expected.map(r =>
+              expect.objectContaining(savedRows.find(sr => sr.age === r.age)!)
+            )
+          )
+        )
+      }
+    )
+  })
+
+  describe("dates", () => {
+    beforeEach(async () => {
+      table = await config.api.table.save(
+        tableForDatasource(datasource, {
+          schema: {
+            dob: {
+              name: "dob",
+              type: FieldType.DATETIME,
+            },
+          },
+        })
+      )
+    })
+
+    const rows = [
+      { dob: new Date("2020-01-01").toISOString() },
+      { dob: new Date("2020-01-10").toISOString() },
+    ]
+    let savedRows: Row[]
+
+    beforeEach(async () => {
+      savedRows = await Promise.all(
+        rows.map(r => config.api.row.save(table._id!, r))
+      )
+    })
+
+    interface DateSearchTest {
+      query: SearchFilters
+      expected: (typeof rows)[number][]
+    }
+
+    const dateSearchTests: DateSearchTest[] = [
+      {
+        query: { equal: { dob: new Date("2020-01-01").toISOString() } },
+        expected: [rows[0]],
+      },
+      {
+        query: { equal: { dob: new Date("2020-01-02").toISOString() } },
+        expected: [],
+      },
+      {
+        query: { notEqual: { dob: new Date("2020-01-01").toISOString() } },
+        expected: [rows[1]],
+      },
+      {
+        query: { oneOf: { dob: [new Date("2020-01-01").toISOString()] } },
+        expected: [rows[0]],
+      },
+      {
+        query: {
+          range: {
+            dob: {
+              low: new Date("2020-01-01").toISOString(),
+              high: new Date("2020-01-05").toISOString(),
+            },
+          },
+        },
+        expected: [rows[0]],
+      },
+      {
+        query: {
+          range: {
+            dob: {
+              low: new Date("2020-01-01").toISOString(),
+              high: new Date("2020-01-10").toISOString(),
+            },
+          },
+        },
+        expected: rows,
+      },
+      {
+        query: {
+          range: {
+            dob: {
+              low: new Date("2020-01-05").toISOString(),
+              high: new Date("2020-01-10").toISOString(),
+            },
+          },
+        },
+        expected: [rows[1]],
+      },
+    ]
+
+    it.each(dateSearchTests)(
+      `should be able to run query: $query`,
+      async ({ query, expected }) => {
+        const { rows: foundRows } = await config.api.row.search(table._id!, {
+          tableId: table._id!,
+          query,
+        })
+        expect(foundRows).toHaveLength(expected.length)
+        expect(foundRows).toEqual(
+          expect.arrayContaining(
+            expected.map(r =>
+              expect.objectContaining(savedRows.find(sr => sr.dob === r.dob)!)
+            )
+          )
+        )
+      }
+    )
+  })
+})