Merge branch 'master' into nested-nav-links

.github/workflows/budibase_ci.yml

@@ -92,7 +92,6 @@ jobs:
   test-libraries:
     runs-on: ubuntu-latest
     env:
-      DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull
       REUSE_CONTAINERS: true
     steps:
       - name: Checkout repo
@@ -110,7 +109,7 @@ jobs:
       - name: Pull testcontainers images
         run: |
           docker pull testcontainers/ryuk:0.5.1 &
-          docker pull budibase/couchdb &
+          docker pull budibase/couchdb:v3.2.1-sql &
           docker pull redis &
 
           wait $(jobs -p)
@@ -151,7 +150,6 @@ jobs:
   test-server:
     runs-on: budi-tubby-tornado-quad-core-150gb
     env:
-      DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull
       REUSE_CONTAINERS: true
     steps:
       - name: Checkout repo
@@ -175,7 +173,7 @@ jobs:
           docker pull mongo:7.0-jammy &
           docker pull mariadb:lts &
           docker pull testcontainers/ryuk:0.5.1 &
-          docker pull budibase/couchdb &
+          docker pull budibase/couchdb:v3.2.1-sql &
           docker pull redis &
 
           wait $(jobs -p)

globalSetup.ts

@@ -13,8 +13,8 @@ export default async function setup() {
   }
 
   try {
-    let couchdb = new GenericContainer("budibase/couchdb")
+    let couchdb = new GenericContainer("budibase/couchdb:v3.2.1-sqs")
-      .withExposedPorts(5984)
+      .withExposedPorts(5984, 4984)
       .withEnvironment({
         COUCHDB_PASSWORD: "budibase",
         COUCHDB_USER: "budibase",

hosting/couchdb/Dockerfile

@@ -128,4 +128,4 @@ ADD couch/vm.args couch/local.ini ./etc/
 WORKDIR /
 ADD runner.sh ./bbcouch-runner.sh
 RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau
-CMD ["./bbcouch-runner.sh"]
+CMD ["./bbcouch-runner.sh"]

hosting/couchdb/Dockerfile.v2

@@ -0,0 +1,135 @@
+# Modified from https://github.com/apache/couchdb-docker/blob/main/3.3.3/Dockerfile
+#
+# Everything in this `base` image is adapted from the official `couchdb` image's
+# Dockerfile. Only modifications related to upgrading from Debian bullseye to
+# bookworm have been included. The `runner` image contains Budibase's
+# customisations to the image, e.g. adding Clouseau.
+FROM node:20-slim AS base
+
+# Add CouchDB user account to make sure the IDs are assigned consistently
+RUN groupadd -g 5984 -r couchdb && useradd -u 5984 -d /opt/couchdb -g couchdb couchdb
+
+# be sure GPG and apt-transport-https are available and functional
+RUN set -ex; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+        apt-transport-https \
+        ca-certificates \
+        dirmngr \
+        gnupg \
+     ; \
+    rm -rf /var/lib/apt/lists/*
+
+# grab tini for signal handling and zombie reaping
+# see https://github.com/apache/couchdb-docker/pull/28#discussion_r141112407
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends tini; \
+    rm -rf /var/lib/apt/lists/*; \
+    tini --version
+
+# http://docs.couchdb.org/en/latest/install/unix.html#installing-the-apache-couchdb-packages
+ENV GPG_COUCH_KEY \
+# gpg: rsa8192 205-01-19 The Apache Software Foundation (Package repository signing key) <root@apache.org>
+    390EF70BB1EA12B2773962950EE62FB37A00258D
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y curl; \
+    export GNUPGHOME="$(mktemp -d)"; \
+    curl -fL -o keys.asc https://couchdb.apache.org/repo/keys.asc; \
+    gpg --batch --import keys.asc; \
+    gpg --batch --export "${GPG_COUCH_KEY}" > /usr/share/keyrings/couchdb-archive-keyring.gpg; \
+    command -v gpgconf && gpgconf --kill all || :; \
+    rm -rf "$GNUPGHOME"; \
+    apt-key list; \
+    apt purge -y --autoremove curl; \
+    rm -rf /var/lib/apt/lists/*
+
+ENV COUCHDB_VERSION 3.3.3
+
+RUN . /etc/os-release; \
+    echo "deb [signed-by=/usr/share/keyrings/couchdb-archive-keyring.gpg] https://apache.jfrog.io/artifactory/couchdb-deb/ ${VERSION_CODENAME} main" | \
+        tee /etc/apt/sources.list.d/couchdb.list >/dev/null
+
+# https://github.com/apache/couchdb-pkg/blob/master/debian/README.Debian
+RUN set -eux; \
+    apt-get update; \
+    \
+    echo "couchdb couchdb/mode select none" | debconf-set-selections; \
+# we DO want recommends this time
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \
+            couchdb="$COUCHDB_VERSION"~bookworm \
+    ; \
+# Undo symlinks to /var/log and /var/lib
+    rmdir /var/lib/couchdb /var/log/couchdb; \
+    rm /opt/couchdb/data /opt/couchdb/var/log; \
+    mkdir -p /opt/couchdb/data /opt/couchdb/var/log; \
+    chown couchdb:couchdb /opt/couchdb/data /opt/couchdb/var/log; \
+    chmod 777 /opt/couchdb/data /opt/couchdb/var/log; \
+# Remove file that sets logging to a file
+    rm /opt/couchdb/etc/default.d/10-filelog.ini; \
+# Check we own everything in /opt/couchdb. Matches the command in dockerfile_entrypoint.sh
+    find /opt/couchdb \! \( -user couchdb -group couchdb \) -exec chown -f couchdb:couchdb '{}' +; \
+# Setup directories and permissions for config. Technically these could be 555 and 444 respectively
+# but we keep them as 755 and 644 for consistency with CouchDB defaults and the dockerfile_entrypoint.sh.
+    find /opt/couchdb/etc -type d ! -perm 0755 -exec chmod -f 0755 '{}' +; \
+    find /opt/couchdb/etc -type f ! -perm 0644 -exec chmod -f 0644 '{}' +; \
+# only local.d needs to be writable for the docker_entrypoint.sh
+    chmod -f 0777 /opt/couchdb/etc/local.d; \
+# apt clean-up
+    rm -rf /var/lib/apt/lists/*;
+
+# Add configuration
+COPY --chown=couchdb:couchdb couch/10-docker-default.ini /opt/couchdb/etc/default.d/
+# COPY --chown=couchdb:couchdb vm.args /opt/couchdb/etc/
+
+COPY docker-entrypoint.sh /usr/local/bin
+RUN ln -s usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh # backwards compat
+ENTRYPOINT ["tini", "--", "/docker-entrypoint.sh"]
+
+VOLUME /opt/couchdb/data
+
+# 5984: Main CouchDB endpoint
+# 4369: Erlang portmap daemon (epmd)
+# 9100: CouchDB cluster communication port
+EXPOSE 5984 4369 9100
+CMD ["/opt/couchdb/bin/couchdb"]
+
+FROM base as runner
+
+ENV COUCHDB_USER admin
+ENV COUCHDB_PASSWORD admin
+EXPOSE 5984
+EXPOSE 4984
+
+RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \
+    wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - && \
+    apt-add-repository 'deb http://security.debian.org/debian-security bookworm-security/updates main' && \
+    apt-add-repository 'deb http://archive.debian.org/debian stretch-backports main' && \
+    apt-add-repository 'deb https://packages.adoptium.net/artifactory/deb bookworm main' && \
+    apt-get update && apt-get install -y --no-install-recommends temurin-8-jdk && \
+    rm -rf /var/lib/apt/lists/
+
+# setup clouseau
+WORKDIR /
+RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clouseau-2.21.0-dist.zip && \
+  unzip clouseau-2.21.0-dist.zip && \
+  mv clouseau-2.21.0 /opt/clouseau && \
+  rm clouseau-2.21.0-dist.zip
+
+WORKDIR /opt/clouseau
+RUN mkdir ./bin
+ADD clouseau/clouseau ./bin/
+ADD clouseau/log4j.properties clouseau/clouseau.ini ./
+
+# setup CouchDB
+WORKDIR /opt/couchdb
+ADD couch/vm.args couch/local.ini ./etc/
+
+WORKDIR /opt/sqs
+ADD sqs/sqs sqs/better_sqlite3.node ./
+
+WORKDIR /
+ADD runner.v2.sh ./bbcouch-runner.sh
+RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau /opt/sqs/sqs
+CMD ["./bbcouch-runner.sh"]

hosting/couchdb/runner.v2.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+DATA_DIR=${DATA_DIR:-/data}
+COUCHDB_ERLANG_COOKIE=${COUCHDB_ERLANG_COOKIE:-B9CFC32C-3458-4A86-8448-B3C753991CA7}
+
+mkdir -p ${DATA_DIR}
+mkdir -p ${DATA_DIR}/couch/{dbs,views}
+mkdir -p ${DATA_DIR}/search
+chown -R couchdb:couchdb ${DATA_DIR}/couch
+
+echo ${TARGETBUILD} > /buildtarget.txt
+if [[ "${TARGETBUILD}" = "aas" ]]; then
+    # Azure AppService uses /home for persistent data & SSH on port 2222
+    DATA_DIR="${DATA_DIR:-/home}"
+    WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
+    mkdir -p $DATA_DIR/{search,minio,couch}
+    mkdir -p $DATA_DIR/couch/{dbs,views}
+    chown -R couchdb:couchdb $DATA_DIR/couch/
+    apt update
+    apt-get install -y openssh-server
+    echo "root:Docker!" | chpasswd
+    mkdir -p /tmp
+    chmod +x /tmp/ssh_setup.sh \
+        && (sleep 1;/tmp/ssh_setup.sh 2>&1 > /dev/null)
+    cp /etc/sshd_config /etc/ssh/sshd_config
+    /etc/init.d/ssh restart
+    sed -i "s#DATA_DIR#/home#g" /opt/clouseau/clouseau.ini
+    sed -i "s#DATA_DIR#/home#g" /opt/couchdb/etc/local.ini
+elif [[ "${TARGETBUILD}" = "single" ]]; then
+    # In the single image build, the Dockerfile specifies /data as a volume
+    # mount, so we use that for all persistent data.
+    sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
+    sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
+elif [[ "${TARGETBUILD}" = "docker-compose" ]]; then
+    # We remove the database_dir and view_index_dir settings from the local.ini
+    # in docker-compose because it will default to /opt/couchdb/data which is what
+    # our docker-compose was using prior to us switching to using our own CouchDB
+    # image.
+    sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini
+    sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini
+    sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini
+elif [[ -n $KUBERNETES_SERVICE_HOST ]]; then
+    # In Kubernetes the directory /opt/couchdb/data has a persistent volume
+    # mount for storing database data.
+    sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini
+
+    # We remove the database_dir and view_index_dir settings from the local.ini
+    # in Kubernetes because it will default to /opt/couchdb/data which is what
+    # our Helm chart was using prior to us switching to using our own CouchDB
+    # image.
+    sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini
+    sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini
+
+    # We remove the -name setting from the vm.args file in Kubernetes because
+    # it will default to the pod FQDN, which is what's required for clustering
+    # to work.
+    sed -i "s/^-name .*$//g" /opt/couchdb/etc/vm.args
+else
+    # For all other builds, we use /data for persistent data.
+    sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
+    sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
+fi
+
+sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/couchdb/etc/vm.args
+sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/clouseau/clouseau.ini
+
+# Start Clouseau. Budibase won't function correctly without Clouseau running, it
+# powers the search API endpoints which are used to do all sorts, including
+# populating app grids.
+/opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
+
+# Start CouchDB.
+/docker-entrypoint.sh /opt/couchdb/bin/couchdb &
+
+# Start SQS.
+/opt/sqs/sqs --server "http://localhost:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 &
+
+# Wait for CouchDB to start up.
+while [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; do
+    echo 'Waiting for CouchDB to start...';
+    sleep 5;
+done
+
+# CouchDB needs the `_users` and `_replicator` databases to exist before it will
+# function correctly, so we create them here.
+curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_users
+curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_replicator
+sleep infinity

hosting/docker-compose.dev.yaml

@@ -40,7 +40,6 @@ services:
       - PROXY_ADDRESS=host.docker.internal
 
   couchdb-service:
-    # platform: linux/amd64
     container_name: budi-couchdb3-dev
     restart: on-failure
     image: budibase/couchdb

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.22.18",
+  "version": "2.23.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

package.json

@@ -59,7 +59,7 @@
     "dev:all": "yarn run kill-all && lerna run --stream dev",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
     "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
-    "test": "lerna run --stream test --stream",
+    "test": "REUSE_CONTAINERS=1 lerna run --concurrency 1 --stream test --stream",
     "lint:eslint": "eslint packages --max-warnings=0",
     "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"",
     "lint": "yarn run lint:eslint && yarn run lint:prettier",
@@ -74,6 +74,7 @@
     "build:docker:single": "./scripts/build-single-image.sh",
     "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting",
     "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.2.1 --push ./hosting/couchdb",
+    "publish:docker:couch-sqs": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile.v2 -t budibase/couchdb:v3.2.1-sqs --push ./hosting/couchdb",
     "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting",
     "release:helm": "node scripts/releaseHelmChart",
     "env:multi:enable": "lerna run --stream env:multi:enable",

packages/backend-core/src/constants/db.ts

@@ -66,3 +66,4 @@ export const APP_PREFIX = prefixed(DocumentType.APP)
 export const APP_DEV = prefixed(DocumentType.APP_DEV)
 export const APP_DEV_PREFIX = APP_DEV
 export const BUDIBASE_DATASOURCE_TYPE = "budibase"
+export const SQLITE_DESIGN_DOC_ID = "_design/sqlite"

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -18,6 +18,7 @@ import { directCouchUrlCall } from "./utils"
 import { getPouchDB } from "./pouchDB"
 import { WriteStream, ReadStream } from "fs"
 import { newid } from "../../docIds/newid"
+import { SQLITE_DESIGN_DOC_ID } from "../../constants"
 import { DDInstrumentedDatabase } from "../instrumentation"
 
 const DATABASE_NOT_FOUND = "Database does not exist."
@@ -247,6 +248,21 @@ export class DatabaseImpl implements Database {
     })
   }
 
+  async sql<T extends Document>(sql: string): Promise<T[]> {
+    const dbName = this.name
+    const url = `/${dbName}/${SQLITE_DESIGN_DOC_ID}`
+    const response = await directCouchUrlCall({
+      url: `${this.couchInfo.sqlUrl}/${url}`,
+      method: "POST",
+      cookie: this.couchInfo.cookie,
+      body: sql,
+    })
+    if (response.status > 300) {
+      throw new Error(await response.text())
+    }
+    return (await response.json()) as T[]
+  }
+
   async query<T extends Document>(
     viewName: string,
     params: DatabaseQueryOpts

packages/backend-core/src/db/couch/connections.ts

@@ -25,6 +25,7 @@ export const getCouchInfo = (connection?: string) => {
   const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
   return {
     url: urlInfo.url!,
+    sqlUrl: env.COUCH_DB_SQL_URL,
     auth: {
       username: username,
       password: password,

packages/backend-core/src/db/couch/utils.ts

@@ -30,8 +30,13 @@ export async function directCouchUrlCall({
     },
   }
   if (body && method !== "GET") {
-    params.body = JSON.stringify(body)
+    if (typeof body === "string") {
-    params.headers["Content-Type"] = "application/json"
+      params.body = body
+      params.headers["Content-Type"] = "text/plain"
+    } else {
+      params.body = JSON.stringify(body)
+      params.headers["Content-Type"] = "application/json"
+    }
   }
   return await fetch(checkSlashesInUrl(encodeURI(url)), params)
 }

packages/backend-core/src/db/instrumentation.ts

@@ -149,4 +149,11 @@ export class DDInstrumentedDatabase implements Database {
       return this.db.getIndexes(...args)
     })
   }
+
+  sql<T extends Document>(sql: string): Promise<T[]> {
+    return tracer.trace("db.sql", span => {
+      span?.addTags({ db_name: this.name })
+      return this.db.sql(sql)
+    })
+  }
 }

packages/backend-core/src/db/lucene.ts

@@ -1,28 +1,16 @@
 import fetch from "node-fetch"
 import { getCouchInfo } from "./couch"
-import { SearchFilters, Row, EmptyFilterOption } from "@budibase/types"
+import {
+  SearchFilters,
+  Row,
+  EmptyFilterOption,
+  SearchResponse,
+  SearchParams,
+  WithRequired,
+} from "@budibase/types"
 
 const QUERY_START_REGEX = /\d[0-9]*:/g
 
-interface SearchResponse<T> {
-  rows: T[] | any[]
-  bookmark?: string
-  totalRows: number
-}
-
-export type SearchParams<T> = {
-  tableId?: string
-  sort?: string
-  sortOrder?: string
-  sortType?: string
-  limit?: number
-  bookmark?: string
-  version?: string
-  indexer?: () => Promise<any>
-  disableEscaping?: boolean
-  rows?: T | Row[]
-}
-
 export function removeKeyNumbering(key: any): string {
   if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) {
     const parts = key.split(":")
@@ -44,7 +32,7 @@ export class QueryBuilder<T> {
   #query: SearchFilters
   #limit: number
   #sort?: string
-  #bookmark?: string
+  #bookmark?: string | number
   #sortOrder: string
   #sortType: string
   #includeDocs: boolean
@@ -130,7 +118,7 @@ export class QueryBuilder<T> {
     return this
   }
 
-  setBookmark(bookmark?: string) {
+  setBookmark(bookmark?: string | number) {
     if (bookmark != null) {
       this.#bookmark = bookmark
     }
@@ -226,14 +214,20 @@ export class QueryBuilder<T> {
     }
   }
 
-  /**
+  preprocess(
-   * Preprocesses a value before going into a lucene search.
+    value: any,
-   * Transforms strings to lowercase and wraps strings and bools in quotes.
+    {
-   * @param value The value to process
+      escape,
-   * @param options The preprocess options
+      lowercase,
-   * @returns {string|*}
+      wrap,
-   */
+      type,
-  preprocess(value: any, { escape, lowercase, wrap, type }: any = {}) {
+    }: {
+      escape?: boolean
+      lowercase?: boolean
+      wrap?: boolean
+      type?: string
+    } = {}
+  ): string | any {
     const hasVersion = !!this.#version
     // Determine if type needs wrapped
     const originalType = typeof value
@@ -561,7 +555,7 @@ async function runQuery<T>(
   url: string,
   body: any,
   cookie: string
-): Promise<SearchResponse<T>> {
+): Promise<WithRequired<SearchResponse<T>, "totalRows">> {
   const response = await fetch(url, {
     body: JSON.stringify(body),
     method: "POST",
@@ -575,7 +569,7 @@ async function runQuery<T>(
   }
   const json = await response.json()
 
-  let output: SearchResponse<T> = {
+  let output: WithRequired<SearchResponse<T>, "totalRows"> = {
     rows: [],
     totalRows: 0,
   }
@@ -613,63 +607,51 @@ async function recursiveSearch<T>(
   dbName: string,
   index: string,
   query: any,
-  params: any
+  params: SearchParams
 ): Promise<any> {
   const bookmark = params.bookmark
   const rows = params.rows || []
-  if (rows.length >= params.limit) {
+  if (params.limit && rows.length >= params.limit) {
     return rows
   }
   let pageSize = QueryBuilder.maxLimit
-  if (rows.length > params.limit - QueryBuilder.maxLimit) {
+  if (params.limit && rows.length > params.limit - QueryBuilder.maxLimit) {
     pageSize = params.limit - rows.length
   }
-  const page = await new QueryBuilder<T>(dbName, index, query)
+  const queryBuilder = new QueryBuilder<T>(dbName, index, query)
+  queryBuilder
     .setVersion(params.version)
-    .setTable(params.tableId)
     .setBookmark(bookmark)
     .setLimit(pageSize)
     .setSort(params.sort)
     .setSortOrder(params.sortOrder)
     .setSortType(params.sortType)
-    .run()
+
+  if (params.tableId) {
+    queryBuilder.setTable(params.tableId)
+  }
+
+  const page = await queryBuilder.run()
   if (!page.rows.length) {
     return rows
   }
   if (page.rows.length < QueryBuilder.maxLimit) {
     return [...rows, ...page.rows]
   }
-  const newParams = {
+  const newParams: SearchParams = {
     ...params,
     bookmark: page.bookmark,
-    rows: [...rows, ...page.rows],
+    rows: [...rows, ...page.rows] as Row[],
   }
   return await recursiveSearch(dbName, index, query, newParams)
 }
 
-/**
- * Performs a paginated search. A bookmark will be returned to allow the next
- * page to be fetched. There is a max limit off 200 results per page in a
- * paginated search.
- * @param dbName Which database to run a lucene query on
- * @param index Which search index to utilise
- * @param query The JSON query structure
- * @param params The search params including:
- *   tableId {string} The table ID to search
- *   sort {string} The sort column
- *   sortOrder {string} The sort order ("ascending" or "descending")
- *   sortType {string} Whether to treat sortable values as strings or
- *     numbers. ("string" or "number")
- *   limit {number} The desired page size
- *   bookmark {string} The bookmark to resume from
- * @returns {Promise<{hasNextPage: boolean, rows: *[]}>}
- */
 export async function paginatedSearch<T>(
   dbName: string,
   index: string,
   query: SearchFilters,
-  params: SearchParams<T>
+  params: SearchParams
-) {
+): Promise<SearchResponse<T>> {
   let limit = params.limit
   if (limit == null || isNaN(limit) || limit < 0) {
     limit = 50
@@ -713,29 +695,12 @@ export async function paginatedSearch<T>(
   }
 }
 
-/**
- * Performs a full search, fetching multiple pages if required to return the
- * desired amount of results. There is a limit of 1000 results to avoid
- * heavy performance hits, and to avoid client components breaking from
- * handling too much data.
- * @param dbName Which database to run a lucene query on
- * @param index Which search index to utilise
- * @param query The JSON query structure
- * @param params The search params including:
- *   tableId {string} The table ID to search
- *   sort {string} The sort column
- *   sortOrder {string} The sort order ("ascending" or "descending")
- *   sortType {string} Whether to treat sortable values as strings or
- *     numbers. ("string" or "number")
- *   limit {number} The desired number of results
- * @returns {Promise<{rows: *}>}
- */
 export async function fullSearch<T>(
   dbName: string,
   index: string,
   query: SearchFilters,
-  params: SearchParams<T>
+  params: SearchParams
-) {
+): Promise<{ rows: Row[] }> {
   let limit = params.limit
   if (limit == null || isNaN(limit) || limit < 0) {
     limit = 1000

packages/backend-core/src/db/tests/lucene.spec.ts

@@ -1,23 +1,39 @@
 import { newid } from "../../docIds/newid"
 import { getDB } from "../db"
-import { Database, EmptyFilterOption } from "@budibase/types"
+import {
-import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene"
+  Database,
+  EmptyFilterOption,
+  SortOrder,
+  SortType,
+  DocumentType,
+  SEPARATOR,
+} from "@budibase/types"
+import { fullSearch, paginatedSearch, QueryBuilder } from "../lucene"
 
 const INDEX_NAME = "main"
+const TABLE_ID = DocumentType.TABLE + SEPARATOR + newid()
 
 const index = `function(doc) {
-  let props = ["property", "number", "array"]
+  if (!doc._id.startsWith("ro_")) {
-  for (let key of props) {
+    return
-    if (Array.isArray(doc[key])) {
+  }
-      for (let val of doc[key]) {
+  let keys = Object.keys(doc).filter(key => !key.startsWith("_"))
+  for (let key of keys) {
+    const value = doc[key]
+    if (Array.isArray(value)) {
+      for (let val of value) {
         index(key, val)
       }
-    } else if (doc[key]) {
+    } else if (value) {
-      index(key, doc[key])
+      index(key, value)
     }
   }
 }`
 
+function rowId(id?: string) {
+  return DocumentType.ROW + SEPARATOR + (id || newid())
+}
+
 describe("lucene", () => {
   let db: Database, dbName: string
 
@@ -25,10 +41,21 @@ describe("lucene", () => {
     dbName = `db-${newid()}`
     // create the DB for testing
     db = getDB(dbName)
-    await db.put({ _id: newid(), property: "word", array: ["1", "4"] })
-    await db.put({ _id: newid(), property: "word2", array: ["3", "1"] })
     await db.put({
-      _id: newid(),
+      _id: rowId(),
+      tableId: TABLE_ID,
+      property: "word",
+      array: ["1", "4"],
+    })
+    await db.put({
+      _id: rowId(),
+      tableId: TABLE_ID,
+      property: "word2",
+      array: ["3", "1"],
+    })
+    await db.put({
+      _id: rowId(),
+      tableId: TABLE_ID,
       property: "word3",
       number: 1,
       array: ["1", "2"],
@@ -240,7 +267,8 @@ describe("lucene", () => {
         docs = Array(QueryBuilder.maxLimit * 2.5)
           .fill(0)
           .map((_, i) => ({
-            _id: i.toString().padStart(3, "0"),
+            _id: rowId(i.toString().padStart(3, "0")),
+            tableId: TABLE_ID,
             property: `value_${i.toString().padStart(3, "0")}`,
             array: [],
           }))
@@ -338,10 +366,11 @@ describe("lucene", () => {
           },
         },
         {
+          tableId: TABLE_ID,
           limit: 1,
           sort: "property",
-          sortType: "string",
+          sortType: SortType.STRING,
-          sortOrder: "desc",
+          sortOrder: SortOrder.DESCENDING,
         }
       )
       expect(page.rows.length).toBe(1)
@@ -360,7 +389,10 @@ describe("lucene", () => {
             property: "wo",
           },
         },
-        {}
+        {
+          tableId: TABLE_ID,
+          query: {},
+        }
       )
       expect(page.rows.length).toBe(3)
     })

packages/backend-core/src/index.ts

@@ -32,7 +32,6 @@ export { default as env } from "./environment"
 export * as blacklist from "./blacklist"
 export * as docUpdates from "./docUpdates"
 export * from "./utils/Duration"
-export { SearchParams } from "./db"
 export * as docIds from "./docIds"
 export * as security from "./security"
 // Add context to tenancy for backwards compatibility

packages/backend-core/tests/core/utilities/testContainerUtils.ts

@@ -77,9 +77,15 @@ export function setupEnv(...envs: any[]) {
     throw new Error("CouchDB port not found")
   }
 
+  const couchSqlPort = getExposedV4Port(couch, 4984)
+  if (!couchSqlPort) {
+    throw new Error("CouchDB SQL port not found")
+  }
+
   const configs = [
     { key: "COUCH_DB_PORT", value: `${couchPort}` },
     { key: "COUCH_DB_URL", value: `http://127.0.0.1:${couchPort}` },
+    { key: "COUCH_DB_SQL_URL", value: `http://127.0.0.1:${couchSqlPort}` },
   ]
 
   for (const config of configs.filter(x => !!x.value)) {

packages/bbui/src/Form/Core/Dropzone.svelte

@@ -37,6 +37,7 @@
     "svg",
     "bmp",
     "jfif",
+    "webp",
   ]
 
   const fieldId = id || uuid()
@@ -67,7 +68,7 @@
   }
 
   $: showDropzone =
-    (!maximum || (maximum && value?.length < maximum)) && !disabled
+    (!maximum || (maximum && (value?.length || 0) < maximum)) && !disabled
 
   async function processFileList(fileList) {
     if (

packages/builder/src/analytics/IntercomClient.js

@@ -1,95 +0,0 @@
-export default class IntercomClient {
-  constructor(token) {
-    this.token = token
-  }
-
-  //
-  /**
-   * Instantiate intercom using their provided script.
-   */
-  init() {
-    if (!this.token) return
-
-    const token = this.token
-
-    var w = window
-    var ic = w.Intercom
-    if (typeof ic === "function") {
-      ic("reattach_activator")
-      ic("update", w.intercomSettings)
-    } else {
-      var d = document
-      var i = function () {
-        i.c(arguments)
-      }
-      i.q = []
-      i.c = function (args) {
-        i.q.push(args)
-      }
-      w.Intercom = i
-      var l = function () {
-        var s = d.createElement("script")
-        s.type = "text/javascript"
-        s.async = true
-        s.src = "https://widget.intercom.io/widget/" + token
-        var x = d.getElementsByTagName("script")[0]
-        x.parentNode.insertBefore(s, x)
-      }
-      if (document.readyState === "complete") {
-        l()
-      } else if (w.attachEvent) {
-        w.attachEvent("onload", l)
-      } else {
-        w.addEventListener("load", l, false)
-      }
-
-      this.initialised = true
-    }
-  }
-
-  /**
-   * Show the intercom chat bubble.
-   * @param {Object} user - user to identify
-   * @returns Intercom global object
-   */
-  show(user = {}, enabled) {
-    if (!this.initialised || !enabled) return
-
-    return window.Intercom("boot", {
-      app_id: this.token,
-      ...user,
-    })
-  }
-
-  /**
-   * Update intercom user details and messages.
-   * @returns Intercom global object
-   */
-  update() {
-    if (!this.initialised) return
-
-    return window.Intercom("update")
-  }
-
-  /**
-   * Capture analytics events and send them to intercom.
-   * @param {String} event - event identifier
-   * @param {Object} props - properties for the event
-   * @returns Intercom global object
-   */
-  captureEvent(event, props = {}) {
-    if (!this.initialised) return
-
-    return window.Intercom("trackEvent", event, props)
-  }
-
-  /**
-   * Disassociate the user from the current session.
-   * @returns Intercom global object
-   */
-  logout() {
-    if (!this.initialised) return
-
-    return window.Intercom("shutdown")
-  }
-}

packages/builder/src/analytics/index.js

@@ -1,14 +1,12 @@
 import { API } from "api"
 import PosthogClient from "./PosthogClient"
-import IntercomClient from "./IntercomClient"
 import { Events, EventSource } from "./constants"
 
 const posthog = new PosthogClient(process.env.POSTHOG_TOKEN)
-const intercom = new IntercomClient(process.env.INTERCOM_TOKEN)
 
 class AnalyticsHub {
   constructor() {
-    this.clients = [posthog, intercom]
+    this.clients = [posthog]
     this.initialised = false
   }
 
@@ -31,20 +29,10 @@ class AnalyticsHub {
 
   captureEvent(eventName, props = {}) {
     posthog.captureEvent(eventName, props)
-    intercom.captureEvent(eventName, props)
-  }
-
-  showChat(user) {
-    intercom.show(user)
-  }
-
-  initPosthog() {
-    posthog.init()
   }
 
   async logout() {
     posthog.logout()
-    intercom.logout()
   }
 }
 

packages/builder/src/components/backend/DataTable/formula.js

@@ -9,7 +9,7 @@ const MAX_DEPTH = 1
 const TYPES_TO_SKIP = [
   FieldType.FORMULA,
   FieldType.LONGFORM,
-  FieldType.ATTACHMENT,
+  FieldType.ATTACHMENTS,
   //https://github.com/Budibase/budibase/issues/3030
   FieldType.INTERNAL,
 ]

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -394,7 +394,8 @@
         FIELDS.BIGINT,
         FIELDS.BOOLEAN,
         FIELDS.DATETIME,
-        FIELDS.ATTACHMENT,
+        FIELDS.ATTACHMENT_SINGLE,
+        FIELDS.ATTACHMENTS,
         FIELDS.LINK,
         FIELDS.FORMULA,
         FIELDS.JSON,

packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte

@@ -1,6 +1,7 @@
 <script>
+  import { FieldType, FieldSubtype } from "@budibase/types"
   import { Select, Toggle, Multiselect } from "@budibase/bbui"
-  import { DB_TYPE_INTERNAL, FIELDS } from "constants/backend"
+  import { DB_TYPE_INTERNAL } from "constants/backend"
   import { API } from "api"
   import { parseFile } from "./utils"
 
@@ -23,43 +24,47 @@
   const typeOptions = [
     {
       label: "Text",
-      value: FIELDS.STRING.type,
+      value: FieldType.STRING,
     },
     {
       label: "Number",
-      value: FIELDS.NUMBER.type,
+      value: FieldType.NUMBER,
     },
     {
       label: "Date",
-      value: FIELDS.DATETIME.type,
+      value: FieldType.DATETIME,
     },
     {
       label: "Options",
-      value: FIELDS.OPTIONS.type,
+      value: FieldType.OPTIONS,
     },
     {
       label: "Multi-select",
-      value: FIELDS.ARRAY.type,
+      value: FieldType.ARRAY.type,
     },
     {
       label: "Barcode/QR",
-      value: FIELDS.BARCODEQR.type,
+      value: FieldType.BARCODEQR,
     },
     {
       label: "Long Form Text",
-      value: FIELDS.LONGFORM.type,
+      value: FieldType.LONGFORM,
     },
     {
       label: "Attachment",
-      value: FIELDS.ATTACHMENT.type,
+      value: FieldType.ATTACHMENT_SINGLE,
+    },
+    {
+      label: "Attachment list",
+      value: FieldType.ATTACHMENTS,
     },
     {
       label: "User",
-      value: `${FIELDS.USER.type}${FIELDS.USER.subtype}`,
+      value: `${FieldType.BB_REFERENCE}${FieldSubtype.USER}`,
     },
     {
       label: "Users",
-      value: `${FIELDS.USERS.type}${FIELDS.USERS.subtype}`,
+      value: `${FieldType.BB_REFERENCE}${FieldSubtype.USERS}`,
     },
   ]
 

packages/builder/src/components/deploy/AppActions.svelte

@@ -32,7 +32,6 @@
   import TourWrap from "components/portal/onboarding/TourWrap.svelte"
   import { TOUR_STEP_KEYS } from "components/portal/onboarding/tours.js"
   import { goto } from "@roxi/routify"
-  import { onMount } from "svelte"
 
   export let application
   export let loaded
@@ -151,10 +150,6 @@
       notifications.error("Error refreshing app")
     }
   }
-
-  onMount(() => {
-    analytics.initPosthog()
-  })
 </script>
 
 <!-- svelte-ignore a11y-click-events-have-key-events -->

packages/builder/src/components/design/settings/componentSettings.js

@@ -1,4 +1,6 @@
 import { Checkbox, Select, RadioGroup, Stepper, Input } from "@budibase/bbui"
+import { licensing } from "stores/portal"
+import { get } from "svelte/store"
 import DataSourceSelect from "./controls/DataSourceSelect/DataSourceSelect.svelte"
 import S3DataSourceSelect from "./controls/S3DataSourceSelect.svelte"
 import DataProviderSelect from "./controls/DataProviderSelect.svelte"
@@ -26,7 +28,8 @@ import FieldConfiguration from "./controls/FieldConfiguration/FieldConfiguration
 import ButtonConfiguration from "./controls/ButtonConfiguration/ButtonConfiguration.svelte"
 import RelationshipFilterEditor from "./controls/RelationshipFilterEditor.svelte"
 import FormStepConfiguration from "./controls/FormStepConfiguration.svelte"
-import FormStepControls from "components/design/settings/controls/FormStepControls.svelte"
+import FormStepControls from "./controls/FormStepControls.svelte"
+import PaywalledSetting from "./controls/PaywalledSetting.svelte"
 
 const componentMap = {
   text: DrawerBindableInput,
@@ -67,6 +70,7 @@ const componentMap = {
   "field/longform": FormFieldSelect,
   "field/datetime": FormFieldSelect,
   "field/attachment": FormFieldSelect,
+  "field/attachment_single": FormFieldSelect,
   "field/s3": Input,
   "field/link": FormFieldSelect,
   "field/array": FormFieldSelect,
@@ -86,11 +90,16 @@ const componentMap = {
 }
 
 export const getComponentForSetting = setting => {
-  const { type, showInBar, barStyle } = setting || {}
+  const { type, showInBar, barStyle, license } = setting || {}
   if (!type) {
     return null
   }
 
+  // Check for paywalled settings
+  if (license && get(licensing).isFreePlan) {
+    return PaywalledSetting
+  }
+
   // We can show a clone of the bar settings for certain select settings
   if (showInBar && type === "select" && barStyle === "buttons") {
     return BarButtonList

packages/builder/src/components/design/settings/controls/FieldConfiguration/utils.js

@@ -41,7 +41,8 @@ export const FieldTypeToComponentMap = {
   [FieldType.BOOLEAN]: "booleanfield",
   [FieldType.LONGFORM]: "longformfield",
   [FieldType.DATETIME]: "datetimefield",
-  [FieldType.ATTACHMENT]: "attachmentfield",
+  [FieldType.ATTACHMENTS]: "attachmentfield",
+  [FieldType.ATTACHMENT_SINGLE]: "attachmentsinglefield",
   [FieldType.LINK]: "relationshipfield",
   [FieldType.JSON]: "jsonfield",
   [FieldType.BARCODEQR]: "codescanner",

packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte

@@ -34,7 +34,7 @@
     $selectedScreen,
     datasource
   )?.table?.primaryDisplay
-  $: schema = getSchema(selectedScreen, datasource)
+  $: schema = getSchema($selectedScreen, datasource)
   $: columns = getColumns({
     columns: value,
     schema,

packages/builder/src/components/design/settings/controls/PaywalledSetting.svelte

@@ -0,0 +1,23 @@
+<script>
+  import { Tag, Tags } from "@budibase/bbui"
+  import { getFormattedPlanName } from "helpers/planTitle"
+
+  export let license
+
+  $: title = getFormattedPlanName(license)
+</script>
+
+<div>
+  <Tags>
+    <Tag icon="LockClosed">{title}</Tag>
+  </Tags>
+</div>
+
+<style>
+  div {
+    height: 32px;
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+  }
+</style>

packages/builder/src/constants/backend/index.js

@@ -5,6 +5,9 @@ import {
   AutoFieldSubType,
   Hosting,
 } from "@budibase/types"
+import { Constants } from "@budibase/frontend-core"
+
+const { TypeIconMap } = Constants
 
 export { RelationshipType } from "@budibase/types"
 
@@ -22,7 +25,7 @@ export const FIELDS = {
   STRING: {
     name: "Text",
     type: FieldType.STRING,
-    icon: "Text",
+    icon: TypeIconMap[FieldType.STRING],
     constraints: {
       type: "string",
       length: {},
@@ -32,7 +35,7 @@ export const FIELDS = {
   BARCODEQR: {
     name: "Barcode/QR",
     type: FieldType.BARCODEQR,
-    icon: "Camera",
+    icon: TypeIconMap[FieldType.BARCODEQR],
     constraints: {
       type: "string",
       length: {},
@@ -42,7 +45,7 @@ export const FIELDS = {
   LONGFORM: {
     name: "Long Form Text",
     type: FieldType.LONGFORM,
-    icon: "TextAlignLeft",
+    icon: TypeIconMap[FieldType.LONGFORM],
     constraints: {
       type: "string",
       length: {},
@@ -52,7 +55,7 @@ export const FIELDS = {
   OPTIONS: {
     name: "Options",
     type: FieldType.OPTIONS,
-    icon: "Dropdown",
+    icon: TypeIconMap[FieldType.OPTIONS],
     constraints: {
       type: "string",
       presence: false,
@@ -62,7 +65,7 @@ export const FIELDS = {
   ARRAY: {
     name: "Multi-select",
     type: FieldType.ARRAY,
-    icon: "Duplicate",
+    icon: TypeIconMap[FieldType.ARRAY],
     constraints: {
       type: "array",
       presence: false,
@@ -72,7 +75,7 @@ export const FIELDS = {
   NUMBER: {
     name: "Number",
     type: FieldType.NUMBER,
-    icon: "123",
+    icon: TypeIconMap[FieldType.NUMBER],
     constraints: {
       type: "number",
       presence: false,
@@ -82,12 +85,12 @@ export const FIELDS = {
   BIGINT: {
     name: "BigInt",
     type: FieldType.BIGINT,
-    icon: "TagBold",
+    icon: TypeIconMap[FieldType.BIGINT],
   },
   BOOLEAN: {
     name: "Boolean",
     type: FieldType.BOOLEAN,
-    icon: "Boolean",
+    icon: TypeIconMap[FieldType.BOOLEAN],
     constraints: {
       type: "boolean",
       presence: false,
@@ -96,7 +99,7 @@ export const FIELDS = {
   DATETIME: {
     name: "Date/Time",
     type: FieldType.DATETIME,
-    icon: "Calendar",
+    icon: TypeIconMap[FieldType.DATETIME],
     constraints: {
       type: "string",
       length: {},
@@ -107,10 +110,18 @@ export const FIELDS = {
       },
     },
   },
-  ATTACHMENT: {
+  ATTACHMENT_SINGLE: {
     name: "Attachment",
-    type: FieldType.ATTACHMENT,
+    type: FieldType.ATTACHMENT_SINGLE,
-    icon: "Folder",
+    icon: TypeIconMap[FieldType.ATTACHMENT_SINGLE],
+    constraints: {
+      presence: false,
+    },
+  },
+  ATTACHMENTS: {
+    name: "Attachment List",
+    type: FieldType.ATTACHMENTS,
+    icon: TypeIconMap[FieldType.ATTACHMENTS],
     constraints: {
       type: "array",
       presence: false,
@@ -119,7 +130,7 @@ export const FIELDS = {
   LINK: {
     name: "Relationship",
     type: FieldType.LINK,
-    icon: "Link",
+    icon: TypeIconMap[FieldType.LINK],
     constraints: {
       type: "array",
       presence: false,
@@ -128,19 +139,19 @@ export const FIELDS = {
   AUTO: {
     name: "Auto Column",
     type: FieldType.AUTO,
-    icon: "MagicWand",
+    icon: TypeIconMap[FieldType.AUTO],
     constraints: {},
   },
   FORMULA: {
     name: "Formula",
     type: FieldType.FORMULA,
-    icon: "Calculator",
+    icon: TypeIconMap[FieldType.FORMULA],
     constraints: {},
   },
   JSON: {
     name: "JSON",
     type: FieldType.JSON,
-    icon: "Brackets",
+    icon: TypeIconMap[FieldType.JSON],
     constraints: {
       type: "object",
       presence: false,
@@ -150,13 +161,13 @@ export const FIELDS = {
     name: "User",
     type: FieldType.BB_REFERENCE,
     subtype: FieldSubtype.USER,
-    icon: "User",
+    icon: TypeIconMap[FieldType.USER],
   },
   USERS: {
     name: "Users",
     type: FieldType.BB_REFERENCE,
     subtype: FieldSubtype.USERS,
-    icon: "User",
+    icon: TypeIconMap[FieldType.USERS],
     constraints: {
       type: "array",
     },
@@ -299,7 +310,7 @@ export const PaginationLocations = [
 
 export const BannedSearchTypes = [
   FieldType.LINK,
-  FieldType.ATTACHMENT,
+  FieldType.ATTACHMENTS,
   FieldType.FORMULA,
   FieldType.JSON,
   "jsonarray",

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte

@@ -183,6 +183,7 @@
               props={{
                 // Generic settings
                 placeholder: setting.placeholder || null,
+                license: setting.license,
 
                 // Select settings
                 options: setting.options || [],

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json

@@ -63,6 +63,7 @@
       "optionsfield",
       "booleanfield",
       "longformfield",
+      "attachmentsinglefield",
       "attachmentfield",
       "jsonfield",
       "relationshipfield",

packages/builder/src/stores/builder/tests/sortedIntegrations.test.js

@@ -6,7 +6,10 @@ import { derived } from "svelte/store"
 import { integrations } from "stores/builder/integrations"
 
 vi.mock("svelte/store", () => ({
-  derived: vi.fn(() => {}),
+  derived: vi.fn(),
+  writable: vi.fn(() => ({
+    subscribe: vi.fn(),
+  })),
 }))
 
 vi.mock("stores/builder/integrations", () => ({ integrations: vi.fn() }))

packages/builder/src/stores/portal/auth.js

@@ -2,7 +2,6 @@ import { derived, writable, get } from "svelte/store"
 import { API } from "api"
 import { admin } from "stores/portal"
 import analytics from "analytics"
-import { sdk } from "@budibase/shared-core"
 
 export function createAuthStore() {
   const auth = writable({
@@ -42,20 +41,6 @@ export function createAuthStore() {
         .activate()
         .then(() => {
           analytics.identify(user._id)
-          analytics.showChat(
-            {
-              email: user.email,
-              created_at: (user.createdAt || Date.now()) / 1000,
-              name: user.account?.name,
-              user_id: user._id,
-              tenant: user.tenantId,
-              admin: sdk.users.isAdmin(user),
-              builder: sdk.users.isBuilder(user),
-              "Company size": user.account?.size,
-              "Job role": user.account?.profession,
-            },
-            !!user?.account
-          )
         })
         .catch(() => {
           // This request may fail due to browser extensions blocking requests

packages/builder/tsconfig.json

@@ -1,9 +1,6 @@
 {
   "extends": "./tsconfig.build.json",
   "compilerOptions": {
-    "composite": true,
-    "declaration": true,
-    "sourceMap": true,
     "baseUrl": ".",
     "paths": {
       "assets/*": ["./assets/*"],

packages/builder/vite.config.js

@@ -77,9 +77,6 @@ export default defineConfig(({ mode }) => {
           isProduction ? "production" : "development"
         ),
         "process.env.POSTHOG_TOKEN": JSON.stringify(process.env.POSTHOG_TOKEN),
-        "process.env.INTERCOM_TOKEN": JSON.stringify(
-          process.env.INTERCOM_TOKEN
-        ),
       }),
       copyFonts("fonts"),
       ...(isProduction ? [] : devOnlyPlugins),

packages/client/manifest.json

@@ -4226,7 +4226,7 @@
     ]
   },
   "attachmentfield": {
-    "name": "Attachment",
+    "name": "Attachment list",
     "icon": "Attach",
     "styles": ["size"],
     "requiredAncestors": ["form"],
@@ -4322,6 +4322,97 @@
       }
     ]
   },
+  "attachmentsinglefield": {
+    "name": "Single Attachment",
+    "icon": "Attach",
+    "styles": ["size"],
+    "requiredAncestors": ["form"],
+    "editable": true,
+    "size": {
+      "width": 400,
+      "height": 200
+    },
+    "settings": [
+      {
+        "type": "field/attachment_single",
+        "label": "Field",
+        "key": "field",
+        "required": true
+      },
+      {
+        "type": "text",
+        "label": "Label",
+        "key": "label"
+      },
+      {
+        "type": "text",
+        "label": "Help text",
+        "key": "helpText"
+      },
+      {
+        "type": "text",
+        "label": "Extensions",
+        "key": "extensions"
+      },
+      {
+        "type": "event",
+        "label": "On change",
+        "key": "onChange",
+        "context": [
+          {
+            "label": "Field Value",
+            "key": "value"
+          }
+        ]
+      },
+      {
+        "type": "boolean",
+        "label": "Compact",
+        "key": "compact",
+        "defaultValue": false
+      },
+      {
+        "type": "boolean",
+        "label": "Read only",
+        "key": "disabled",
+        "defaultValue": false
+      },
+      {
+        "type": "validation/attachment",
+        "label": "Validation",
+        "key": "validation"
+      },
+      {
+        "type": "select",
+        "label": "Layout",
+        "key": "span",
+        "defaultValue": 6,
+        "hidden": true,
+        "showInBar": true,
+        "barStyle": "buttons",
+        "options": [
+          {
+            "label": "1 column",
+            "value": 6,
+            "barIcon": "Stop",
+            "barTitle": "1 column"
+          },
+          {
+            "label": "2 columns",
+            "value": 3,
+            "barIcon": "ColumnTwoA",
+            "barTitle": "2 columns"
+          },
+          {
+            "label": "3 columns",
+            "value": 2,
+            "barIcon": "ViewColumn",
+            "barTitle": "3 columns"
+          }
+        ]
+      }
+    ]
+  },
   "relationshipfield": {
     "name": "Relationship Picker",
     "icon": "TaskList",
@@ -4610,6 +4701,35 @@
         "key": "dataSource",
         "required": true
       },
+      {
+        "type": "select",
+        "label": "Auto-refresh",
+        "key": "autoRefresh",
+        "license": "premium",
+        "placeholder": "Never",
+        "options": [
+          {
+            "label": "10 seconds",
+            "value": 10
+          },
+          {
+            "label": "30 seconds",
+            "value": 30
+          },
+          {
+            "label": "1 minute",
+            "value": 60
+          },
+          {
+            "label": "5 minutes",
+            "value": 300
+          },
+          {
+            "label": "10 minutes",
+            "value": 600
+          }
+        ]
+      },
       {
         "type": "filter",
         "label": "Filtering",
@@ -4977,6 +5097,35 @@
         "key": "dataSource",
         "required": true
       },
+      {
+        "type": "select",
+        "label": "Auto-refresh",
+        "key": "autoRefresh",
+        "license": "premium",
+        "placeholder": "Never",
+        "options": [
+          {
+            "label": "10 seconds",
+            "value": 10
+          },
+          {
+            "label": "30 seconds",
+            "value": 30
+          },
+          {
+            "label": "1 minute",
+            "value": 60
+          },
+          {
+            "label": "5 minutes",
+            "value": 300
+          },
+          {
+            "label": "10 minutes",
+            "value": 600
+          }
+        ]
+      },
       {
         "type": "text",
         "label": "Title",
@@ -5445,6 +5594,35 @@
             "key": "dataSource",
             "required": true
           },
+          {
+            "type": "select",
+            "label": "Auto-refresh",
+            "key": "autoRefresh",
+            "license": "premium",
+            "placeholder": "Never",
+            "options": [
+              {
+                "label": "10 seconds",
+                "value": 10
+              },
+              {
+                "label": "30 seconds",
+                "value": 30
+              },
+              {
+                "label": "1 minute",
+                "value": 60
+              },
+              {
+                "label": "5 minutes",
+                "value": 300
+              },
+              {
+                "label": "10 minutes",
+                "value": 600
+              }
+            ]
+          },
           {
             "type": "columns",
             "label": "Columns",
@@ -5731,6 +5909,35 @@
         "key": "dataSource",
         "required": true
       },
+      {
+        "type": "select",
+        "label": "Auto-refresh",
+        "key": "autoRefresh",
+        "license": "premium",
+        "placeholder": "Never",
+        "options": [
+          {
+            "label": "10 seconds",
+            "value": 10
+          },
+          {
+            "label": "30 seconds",
+            "value": 30
+          },
+          {
+            "label": "1 minute",
+            "value": 60
+          },
+          {
+            "label": "5 minutes",
+            "value": 300
+          },
+          {
+            "label": "10 minutes",
+            "value": 600
+          }
+        ]
+      },
       {
         "type": "searchfield",
         "label": "Search columns",
@@ -5895,7 +6102,7 @@
     "block": true,
     "name": "Repeater Block",
     "icon": "ViewList",
-    "illegalChildren": ["section"],
+    "illegalChildren": ["section", "rowexplorer"],
     "hasChildren": true,
     "size": {
       "width": 400,
@@ -5908,6 +6115,35 @@
         "key": "dataSource",
         "required": true
       },
+      {
+        "type": "select",
+        "label": "Auto-refresh",
+        "key": "autoRefresh",
+        "license": "premium",
+        "placeholder": "Never",
+        "options": [
+          {
+            "label": "10 seconds",
+            "value": 10
+          },
+          {
+            "label": "30 seconds",
+            "value": 30
+          },
+          {
+            "label": "1 minute",
+            "value": 60
+          },
+          {
+            "label": "5 minutes",
+            "value": 300
+          },
+          {
+            "label": "10 minutes",
+            "value": 600
+          }
+        ]
+      },
       {
         "type": "filter",
         "label": "Filtering",
@@ -6504,6 +6740,35 @@
         "key": "dataSource",
         "required": true
       },
+      {
+        "type": "select",
+        "label": "Auto-refresh",
+        "key": "autoRefresh",
+        "license": "premium",
+        "placeholder": "Never",
+        "options": [
+          {
+            "label": "10 seconds",
+            "value": 10
+          },
+          {
+            "label": "30 seconds",
+            "value": 30
+          },
+          {
+            "label": "1 minute",
+            "value": 60
+          },
+          {
+            "label": "5 minutes",
+            "value": 300
+          },
+          {
+            "label": "10 minutes",
+            "value": 600
+          }
+        ]
+      },
       {
         "type": "text",
         "label": "Height",

packages/client/src/components/app/DataProvider.svelte

@@ -9,17 +9,18 @@
   export let sortOrder
   export let limit
   export let paginate
+  export let autoRefresh
 
   const { styleable, Provider, ActionTypes, API } = getContext("sdk")
   const component = getContext("component")
 
+  let interval
+  let queryExtensions = {}
+
   // We need to manage our lucene query manually as we want to allow components
   // to extend it
-  let queryExtensions = {}
   $: defaultQuery = LuceneUtils.buildLuceneQuery(filter)
   $: query = extendQuery(defaultQuery, queryExtensions)
-
-  // Fetch data and refresh when needed
   $: fetch = createFetch(dataSource)
   $: fetch.update({
     query,
@@ -28,11 +29,8 @@
     limit,
     paginate,
   })
-
-  // Sanitize schema to remove hidden fields
   $: schema = sanitizeSchema($fetch.schema)
-
+  $: setUpAutoRefresh(autoRefresh)
-  // Build our action context
   $: actions = [
     {
       type: ActionTypes.RefreshDatasource,
@@ -63,8 +61,6 @@
       },
     },
   ]
-
-  // Build our data context
   $: dataContext = {
     rows: $fetch.rows,
     info: $fetch.info,
@@ -140,6 +136,13 @@
     })
     return extendedQuery
   }
+
+  const setUpAutoRefresh = autoRefresh => {
+    clearInterval(interval)
+    if (autoRefresh) {
+      interval = setInterval(fetch.refresh, Math.max(10000, autoRefresh * 1000))
+    }
+  }
 </script>
 
 <div use:styleable={$component.styles} class="container">

packages/client/src/components/app/GridBlock.svelte

@@ -18,6 +18,7 @@
   export let columns = null
   export let onRowClick = null
   export let buttons = null
+  export let repeat = null
 
   const context = getContext("context")
   const component = getContext("component")
@@ -122,6 +123,7 @@
       {fixedRowHeight}
       {columnWhitelist}
       {schemaOverrides}
+      {repeat}
       canAddRows={allowAddRows}
       canEditRows={allowEditRows}
       canDeleteRows={allowDeleteRows}
@@ -145,7 +147,8 @@
     border: 1px solid var(--spectrum-global-color-gray-300);
     border-radius: 4px;
     overflow: hidden;
-    min-height: 410px;
+    min-height: 230px;
+    height: 410px;
   }
   div.in-builder :global(*) {
     pointer-events: none;

packages/client/src/components/app/Layout.svelte

@@ -9,13 +9,11 @@
   const {
     routeStore,
     roleStore,
-    styleable,
     linkable,
     builderStore,
     sidePanelStore,
     appStore,
   } = sdk
-  const component = getContext("component")
   const context = getContext("context")
   const navStateStore = writable({})
 
@@ -198,15 +196,14 @@
 <!-- svelte-ignore a11y-no-static-element-interactions -->
 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <div
-  class="component {screenId} layout layout--{typeClass}"
+  class="component layout layout--{typeClass}"
-  use:styleable={$component.styles}
   class:desktop={!mobile}
   class:mobile={!!mobile}
   data-id={screenId}
   data-name="Screen"
   data-icon="WebPage"
 >
-  <div class="{screenId}-dom screen-wrapper layout-body">
+  <div class="screen-wrapper layout-body">
     {#if typeClass !== "none"}
       <div
         class="interactive component {navigationId}"
@@ -303,7 +300,14 @@
         </div>
       </div>
     {/if}
-    <div class="main-wrapper">
+    <div
+      class="main-wrapper"
+      on:click={() => {
+        if ($builderStore.inBuilder) {
+          builderStore.actions.selectComponent(screenId)
+        }
+      }}
+    >
       <div class="main size--{pageWidthClass}">
         <slot />
       </div>

packages/client/src/components/app/blocks/CardsBlock.svelte

@@ -31,6 +31,7 @@
   export let cardButtonOnClick
   export let linkColumn
   export let noRowsMessage
+  export let autoRefresh
 
   const context = getContext("context")
   const { fetchDatasourceSchema, generateGoldenSample } = getContext("sdk")
@@ -184,6 +185,7 @@
           sortOrder,
           paginate,
           limit,
+          autoRefresh,
         }}
         order={1}
       >

packages/client/src/components/app/blocks/ChartBlock.svelte

@@ -8,6 +8,7 @@
   export let sortColumn
   export let sortOrder
   export let limit
+  export let autoRefresh
 
   // Block
   export let chartTitle
@@ -65,6 +66,7 @@
       sortColumn,
       sortOrder,
       limit,
+      autoRefresh,
     }}
   >
     {#if dataProviderId && chartType}

packages/client/src/components/app/blocks/FormBlockComponent.svelte

@@ -15,7 +15,8 @@
     [FieldType.BOOLEAN]: "booleanfield",
     [FieldType.LONGFORM]: "longformfield",
     [FieldType.DATETIME]: "datetimefield",
-    [FieldType.ATTACHMENT]: "attachmentfield",
+    [FieldType.ATTACHMENTS]: "attachmentfield",
+    [FieldType.ATTACHMENT_SINGLE]: "attachmentsinglefield",
     [FieldType.LINK]: "relationshipfield",
     [FieldType.JSON]: "jsonfield",
     [FieldType.BARCODEQR]: "codescanner",
@@ -60,7 +61,7 @@
 
   function getPropsByType(field) {
     const propsMapByType = {
-      [FieldType.ATTACHMENT]: (_field, schema) => {
+      [FieldType.ATTACHMENTS]: (_field, schema) => {
         return {
           maximum: schema?.constraints?.length?.maximum,
         }

packages/client/src/components/app/blocks/RepeaterBlock.svelte

@@ -17,6 +17,7 @@
   export let hAlign
   export let vAlign
   export let gap
+  export let autoRefresh
 
   const component = getContext("component")
   const context = getContext("context")
@@ -47,6 +48,7 @@
       sortOrder,
       limit,
       paginate,
+      autoRefresh,
     }}
   >
     {#if $component.empty}

packages/client/src/components/app/blocks/RowExplorer.svelte

@@ -16,6 +16,7 @@
   export let detailFields
   export let detailTitle
   export let noRowsMessage
+  export let autoRefresh
 
   const stateKey = generate()
   const context = getContext("context")
@@ -66,6 +67,7 @@
             noValue: false,
           },
         ],
+        autoRefresh,
       }}
       styles={{
         custom: `

packages/client/src/components/app/deprecated/TableBlock.svelte

@@ -33,6 +33,7 @@
   export let sidePanelSaveLabel
   export let sidePanelDeleteLabel
   export let notificationOverride
+  export let autoRefresh
 
   const { fetchDatasourceSchema, API, generateGoldenSample } = getContext("sdk")
   const component = getContext("component")
@@ -243,6 +244,7 @@
           sortOrder,
           paginate,
           limit: rowCount,
+          autoRefresh,
         }}
         context="provider"
         order={1}

packages/client/src/components/app/forms/AttachmentField.svelte

@@ -1,6 +1,7 @@
 <script>
   import Field from "./Field.svelte"
   import { CoreDropzone } from "@budibase/bbui"
+  import { FieldType } from "@budibase/types"
   import { getContext } from "svelte"
 
   export let field
@@ -14,6 +15,12 @@
   export let maximum = undefined
   export let span
   export let helpText = null
+  export let type = FieldType.ATTACHMENTS
+  export let fieldApiMapper = {
+    get: value => value,
+    set: value => value,
+  }
+  export let defaultValue = []
 
   let fieldState
   let fieldApi
@@ -63,9 +70,10 @@
   }
 
   const handleChange = e => {
-    const changed = fieldApi.setValue(e.detail)
+    const value = fieldApiMapper.set(e.detail)
+    const changed = fieldApi.setValue(value)
     if (onChange && changed) {
-      onChange({ value: e.detail })
+      onChange({ value })
     }
   }
 </script>
@@ -78,14 +86,14 @@
   {validation}
   {span}
   {helpText}
-  type="attachment"
+  {type}
   bind:fieldState
   bind:fieldApi
-  defaultValue={[]}
+  {defaultValue}
 >
   {#if fieldState}
     <CoreDropzone
-      value={fieldState.value}
+      value={fieldApiMapper.get(fieldState.value)}
       disabled={fieldState.disabled || fieldState.readonly}
       error={fieldState.error}
       on:change={handleChange}

packages/client/src/components/app/forms/AttachmentSingleField.svelte

@@ -0,0 +1,17 @@
+<script>
+  import { FieldType } from "@budibase/types"
+  import AttachmentField from "./AttachmentField.svelte"
+
+  const fieldApiMapper = {
+    get: value => (!Array.isArray(value) && value ? [value] : value) || [],
+    set: value => value[0] || null,
+  }
+</script>
+
+<AttachmentField
+  {...$$restProps}
+  type={FieldType.ATTACHMENT_SINGLE}
+  maximum={1}
+  defaultValue={null}
+  {fieldApiMapper}
+/>

packages/client/src/components/app/forms/index.js

@@ -9,6 +9,7 @@ export { default as booleanfield } from "./BooleanField.svelte"
 export { default as longformfield } from "./LongFormField.svelte"
 export { default as datetimefield } from "./DateTimeField.svelte"
 export { default as attachmentfield } from "./AttachmentField.svelte"
+export { default as attachmentsinglefield } from "./AttachmentSingleField.svelte"
 export { default as relationshipfield } from "./RelationshipField.svelte"
 export { default as passwordfield } from "./PasswordField.svelte"
 export { default as formstep } from "./FormStep.svelte"

packages/client/src/components/app/forms/validation.js

@@ -192,7 +192,7 @@ const parseType = (value, type) => {
   }
 
   // Parse attachments, treating no elements as null
-  if (type === FieldTypes.ATTACHMENT) {
+  if (type === FieldTypes.ATTACHMENTS) {
     if (!Array.isArray(value) || !value.length) {
       return null
     }

packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte

@@ -10,6 +10,7 @@
   export let invertX = false
   export let invertY = false
   export let schema
+  export let maximum
 
   const { API, notifications } = getContext("grid")
   const imageExtensions = ["png", "tiff", "gif", "raw", "jpg", "jpeg"]
@@ -98,7 +99,7 @@
       {value}
       compact
       on:change={e => onChange(e.detail)}
-      maximum={schema.constraints?.length?.maximum}
+      maximum={maximum || schema.constraints?.length?.maximum}
       {processFiles}
       {deleteAttachments}
       {handleFileTooLarge}

packages/frontend-core/src/components/grid/cells/AttachmentSingleCell.svelte

@@ -0,0 +1,22 @@
+<script>
+  import AttachmentCell from "./AttachmentCell.svelte"
+
+  export let value
+  export let onChange
+  export let api
+
+  $: arrayValue = (!Array.isArray(value) && value ? [value] : value) || []
+
+  $: onFileChange = value => {
+    value = value[0] || null
+    onChange(value)
+  }
+</script>
+
+<AttachmentCell
+  bind:api
+  {...$$restProps}
+  maximum={1}
+  value={arrayValue}
+  onChange={onFileChange}
+/>

packages/frontend-core/src/components/grid/lib/renderers.js

@@ -11,6 +11,7 @@ import BooleanCell from "../cells/BooleanCell.svelte"
 import FormulaCell from "../cells/FormulaCell.svelte"
 import JSONCell from "../cells/JSONCell.svelte"
 import AttachmentCell from "../cells/AttachmentCell.svelte"
+import AttachmentSingleCell from "../cells/AttachmentSingleCell.svelte"
 import BBReferenceCell from "../cells/BBReferenceCell.svelte"
 
 const TypeComponentMap = {
@@ -22,7 +23,8 @@ const TypeComponentMap = {
   [FieldType.ARRAY]: MultiSelectCell,
   [FieldType.NUMBER]: NumberCell,
   [FieldType.BOOLEAN]: BooleanCell,
-  [FieldType.ATTACHMENT]: AttachmentCell,
+  [FieldType.ATTACHMENTS]: AttachmentCell,
+  [FieldType.ATTACHMENT_SINGLE]: AttachmentSingleCell,
   [FieldType.LINK]: RelationshipCell,
   [FieldType.FORMULA]: FormulaCell,
   [FieldType.JSON]: JSONCell,

packages/frontend-core/src/components/grid/lib/utils.js

@@ -1,4 +1,4 @@
-import { FieldType, FieldTypeSubtypes } from "@budibase/types"
+import { TypeIconMap } from "../../../constants"
 
 export const getColor = (idx, opacity = 0.3) => {
   if (idx == null || idx === -1) {
@@ -7,26 +7,6 @@ export const getColor = (idx, opacity = 0.3) => {
   return `hsla(${((idx + 1) * 222) % 360}, 90%, 75%, ${opacity})`
 }
 
-const TypeIconMap = {
-  [FieldType.STRING]: "Text",
-  [FieldType.OPTIONS]: "Dropdown",
-  [FieldType.DATETIME]: "Date",
-  [FieldType.BARCODEQR]: "Camera",
-  [FieldType.LONGFORM]: "TextAlignLeft",
-  [FieldType.ARRAY]: "Dropdown",
-  [FieldType.NUMBER]: "123",
-  [FieldType.BOOLEAN]: "Boolean",
-  [FieldType.ATTACHMENT]: "AppleFiles",
-  [FieldType.LINK]: "DataCorrelated",
-  [FieldType.FORMULA]: "Calculator",
-  [FieldType.JSON]: "Brackets",
-  [FieldType.BIGINT]: "TagBold",
-  [FieldType.BB_REFERENCE]: {
-    [FieldTypeSubtypes.BB_REFERENCE.USER]: "User",
-    [FieldTypeSubtypes.BB_REFERENCE.USERS]: "UserGroup",
-  },
-}
-
 export const getColumnIcon = column => {
   if (column.schema.autocolumn) {
     return "MagicWand"

packages/frontend-core/src/constants.js

@@ -4,6 +4,7 @@
 export { OperatorOptions, SqlNumberTypeRangeMap } from "@budibase/shared-core"
 export { Feature as Features } from "@budibase/types"
 import { BpmCorrelationKey } from "@budibase/shared-core"
+import { FieldType, FieldTypeSubtypes } from "@budibase/types"
 
 // Cookie names
 export const Cookies = {
@@ -113,3 +114,27 @@ export const ContextScopes = {
   Local: "local",
   Global: "global",
 }
+
+export const TypeIconMap = {
+  [FieldType.STRING]: "Text",
+  [FieldType.OPTIONS]: "Dropdown",
+  [FieldType.DATETIME]: "Calendar",
+  [FieldType.BARCODEQR]: "Camera",
+  [FieldType.LONGFORM]: "TextAlignLeft",
+  [FieldType.ARRAY]: "Duplicate",
+  [FieldType.NUMBER]: "123",
+  [FieldType.BOOLEAN]: "Boolean",
+  [FieldType.ATTACHMENTS]: "Attach",
+  [FieldType.ATTACHMENT_SINGLE]: "Attach",
+  [FieldType.LINK]: "DataCorrelated",
+  [FieldType.FORMULA]: "Calculator",
+  [FieldType.JSON]: "Brackets",
+  [FieldType.BIGINT]: "TagBold",
+  [FieldType.AUTO]: "MagicWand",
+  [FieldType.USER]: "User",
+  [FieldType.USERS]: "UserGroup",
+  [FieldType.BB_REFERENCE]: {
+    [FieldTypeSubtypes.BB_REFERENCE.USER]: "User",
+    [FieldTypeSubtypes.BB_REFERENCE.USERS]: "UserGroup",
+  },
+}

packages/pro

@@ -1 +1 @@
-Subproject commit f8e8f87bd52081e1303a5ae92c432ea5b38f3bb4
+Subproject commit ef186d00241f96037f9fd34d7a3826041977ab3a

packages/server/package.json

@@ -83,7 +83,7 @@
     "joi": "17.6.0",
     "js-yaml": "4.1.0",
     "jsonschema": "1.4.0",
-    "knex": "2.4.0",
+    "knex": "2.4.2",
     "koa": "2.13.4",
     "koa-body": "4.2.0",
     "koa-compress": "4.0.1",
@@ -109,6 +109,8 @@
     "server-destroy": "1.0.1",
     "snowflake-promise": "^4.5.0",
     "socket.io": "4.6.1",
+    "sqlite3": "5.1.6",
+    "swagger-parser": "10.0.3",
     "tar": "6.1.15",
     "to-json-schema": "0.2.5",
     "undici": "^6.0.1",

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -6,12 +6,10 @@ import {
   FieldType,
   FilterType,
   IncludeRelationship,
-  ManyToManyRelationshipFieldMetadata,
   OneToManyRelationshipFieldMetadata,
   Operation,
   PaginationJson,
   RelationshipFieldMetadata,
-  RelationshipsJson,
   Row,
   SearchFilters,
   SortJson,
@@ -23,14 +21,20 @@ import {
   breakExternalTableId,
   breakRowIdField,
   convertRowId,
-  generateRowIdField,
   isRowId,
   isSQL,
+  generateRowIdField,
 } from "../../../integrations/utils"
+import {
+  buildExternalRelationships,
+  buildSqlFieldList,
+  generateIdForRow,
+  sqlOutputProcessing,
+  isManyToMany,
+} from "./utils"
 import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
-import { processDates, processFormulas } from "../../../utilities/rowProcessor"
 import { db as dbCore } from "@budibase/backend-core"
 import AliasTables from "./alias"
 import sdk from "../../../sdk"
@@ -154,7 +158,8 @@ function cleanupConfig(config: RunConfig, table: Table): RunConfig {
   // filter out fields which cannot be keys
   const fieldNames = Object.entries(table.schema)
     .filter(schema => primaryOptions.find(val => val === schema[1].type))
-    .map(([fieldName]) => fieldName)
+    // map to fieldName
+    .map(entry => entry[0])
   const iterateObject = (obj: { [key: string]: any }) => {
     for (let [field, value] of Object.entries(obj)) {
       if (fieldNames.find(name => name === field) && isRowId(value)) {
@@ -183,34 +188,6 @@ function cleanupConfig(config: RunConfig, table: Table): RunConfig {
   return config
 }
 
-function generateIdForRow(
-  row: Row | undefined,
-  table: Table,
-  isLinked: boolean = false
-): string {
-  const primary = table.primary
-  if (!row || !primary) {
-    return ""
-  }
-  // build id array
-  let idParts = []
-  for (let field of primary) {
-    let fieldValue = extractFieldValue({
-      row,
-      tableName: table.name,
-      fieldName: field,
-      isLinked,
-    })
-    if (fieldValue != null) {
-      idParts.push(fieldValue)
-    }
-  }
-  if (idParts.length === 0) {
-    return ""
-  }
-  return generateRowIdField(idParts)
-}
-
 function getEndpoint(tableId: string | undefined, operation: string) {
   if (!tableId) {
     throw new Error("Cannot get endpoint information - no table ID specified")
@@ -223,71 +200,6 @@ function getEndpoint(tableId: string | undefined, operation: string) {
   }
 }
 
-// need to handle table name + field or just field, depending on if relationships used
-function extractFieldValue({
-  row,
-  tableName,
-  fieldName,
-  isLinked,
-}: {
-  row: Row
-  tableName: string
-  fieldName: string
-  isLinked: boolean
-}) {
-  let value = row[`${tableName}.${fieldName}`]
-  if (value == null && !isLinked) {
-    value = row[fieldName]
-  }
-  return value
-}
-
-function basicProcessing({
-  row,
-  table,
-  isLinked,
-}: {
-  row: Row
-  table: Table
-  isLinked: boolean
-}): Row {
-  const thisRow: Row = {}
-  // filter the row down to what is actually the row (not joined)
-  for (let field of Object.values(table.schema)) {
-    const fieldName = field.name
-
-    const value = extractFieldValue({
-      row,
-      tableName: table.name,
-      fieldName,
-      isLinked,
-    })
-
-    // all responses include "select col as table.col" so that overlaps are handled
-    if (value != null) {
-      thisRow[fieldName] = value
-    }
-  }
-  thisRow._id = generateIdForRow(row, table, isLinked)
-  thisRow.tableId = table._id
-  thisRow._rev = "rev"
-  return thisRow
-}
-
-function fixArrayTypes(row: Row, table: Table) {
-  for (let [fieldName, schema] of Object.entries(table.schema)) {
-    if (schema.type === FieldType.ARRAY && typeof row[fieldName] === "string") {
-      try {
-        row[fieldName] = JSON.parse(row[fieldName])
-      } catch (err) {
-        // couldn't convert back to array, ignore
-        delete row[fieldName]
-      }
-    }
-  }
-  return row
-}
-
 function isOneSide(
   field: RelationshipFieldMetadata
 ): field is OneToManyRelationshipFieldMetadata {
@@ -296,12 +208,6 @@ function isOneSide(
   )
 }
 
-function isManyToMany(
-  field: RelationshipFieldMetadata
-): field is ManyToManyRelationshipFieldMetadata {
-  return !!(field as ManyToManyRelationshipFieldMetadata).through
-}
-
 function isEditableColumn(column: FieldSchema) {
   const isExternalAutoColumn =
     column.autocolumn &&
@@ -435,187 +341,6 @@ export class ExternalRequest<T extends Operation> {
     return { row: newRow, manyRelationships }
   }
 
-  async processRelationshipFields(
-    table: Table,
-    row: Row,
-    relationships: RelationshipsJson[]
-  ): Promise<Row> {
-    for (let relationship of relationships) {
-      const linkedTable = this.tables[relationship.tableName]
-      if (!linkedTable || !row[relationship.column]) {
-        continue
-      }
-      for (let key of Object.keys(row[relationship.column])) {
-        let relatedRow: Row = row[relationship.column][key]
-        // add this row as context for the relationship
-        for (let col of Object.values(linkedTable.schema)) {
-          if (col.type === FieldType.LINK && col.tableId === table._id) {
-            relatedRow[col.name] = [row]
-          }
-        }
-        // process additional types
-        relatedRow = processDates(table, relatedRow)
-        relatedRow = await processFormulas(linkedTable, relatedRow)
-        row[relationship.column][key] = relatedRow
-      }
-    }
-    return row
-  }
-
-  /**
-   * This iterates through the returned rows and works out what elements of the rows
-   * actually match up to another row (based on primary keys) - this is pretty specific
-   * to SQL and the way that SQL relationships are returned based on joins.
-   * This is complicated, but the idea is that when a SQL query returns all the relations
-   * will be separate rows, with all of the data in each row. We have to decipher what comes
-   * from where (which tables) and how to convert that into budibase columns.
-   */
-  updateRelationshipColumns(
-    table: Table,
-    row: Row,
-    rows: { [key: string]: Row },
-    relationships: RelationshipsJson[]
-  ) {
-    const columns: { [key: string]: any } = {}
-    for (let relationship of relationships) {
-      const linkedTable = this.tables[relationship.tableName]
-      if (!linkedTable) {
-        continue
-      }
-      const fromColumn = `${table.name}.${relationship.from}`
-      const toColumn = `${linkedTable.name}.${relationship.to}`
-      // this is important when working with multiple relationships
-      // between the same tables, don't want to overlap/multiply the relations
-      if (
-        !relationship.through &&
-        row[fromColumn]?.toString() !== row[toColumn]?.toString()
-      ) {
-        continue
-      }
-
-      let linked = basicProcessing({ row, table: linkedTable, isLinked: true })
-      if (!linked._id) {
-        continue
-      }
-      columns[relationship.column] = linked
-    }
-    for (let [column, related] of Object.entries(columns)) {
-      if (!row._id) {
-        continue
-      }
-      const rowId: string = row._id
-      if (!Array.isArray(rows[rowId][column])) {
-        rows[rowId][column] = []
-      }
-      // make sure relationship hasn't been found already
-      if (
-        !rows[rowId][column].find(
-          (relation: Row) => relation._id === related._id
-        )
-      ) {
-        rows[rowId][column].push(related)
-      }
-    }
-    return rows
-  }
-
-  async outputProcessing(
-    rows: Row[] = [],
-    table: Table,
-    relationships: RelationshipsJson[]
-  ) {
-    if (!rows || rows.length === 0 || rows[0].read === true) {
-      return []
-    }
-    let finalRows: { [key: string]: Row } = {}
-    for (let row of rows) {
-      const rowId = generateIdForRow(row, table)
-      row._id = rowId
-      // this is a relationship of some sort
-      if (finalRows[rowId]) {
-        finalRows = this.updateRelationshipColumns(
-          table,
-          row,
-          finalRows,
-          relationships
-        )
-        continue
-      }
-      const thisRow = fixArrayTypes(
-        basicProcessing({ row, table, isLinked: false }),
-        table
-      )
-      if (thisRow._id == null) {
-        throw "Unable to generate row ID for SQL rows"
-      }
-      finalRows[thisRow._id] = thisRow
-      // do this at end once its been added to the final rows
-      finalRows = this.updateRelationshipColumns(
-        table,
-        row,
-        finalRows,
-        relationships
-      )
-    }
-
-    // make sure all related rows are correct
-    let finalRowArray = []
-    for (let row of Object.values(finalRows)) {
-      finalRowArray.push(
-        await this.processRelationshipFields(table, row, relationships)
-      )
-    }
-
-    // process some additional types
-    finalRowArray = processDates(table, finalRowArray)
-    return finalRowArray
-  }
-
-  /**
-   * Gets the list of relationship JSON structures based on the columns in the table,
-   * this will be used by the underlying library to build whatever relationship mechanism
-   * it has (e.g. SQL joins).
-   */
-  buildRelationships(table: Table): RelationshipsJson[] {
-    const relationships = []
-    for (let [fieldName, field] of Object.entries(table.schema)) {
-      if (field.type !== FieldType.LINK) {
-        continue
-      }
-      const { tableName: linkTableName } = breakExternalTableId(field.tableId)
-      // no table to link to, this is not a valid relationships
-      if (!linkTableName || !this.tables[linkTableName]) {
-        continue
-      }
-      const linkTable = this.tables[linkTableName]
-      if (!table.primary || !linkTable.primary) {
-        continue
-      }
-      const definition: RelationshipsJson = {
-        tableName: linkTableName,
-        // need to specify where to put this back into
-        column: fieldName,
-      }
-      if (isManyToMany(field)) {
-        const { tableName: throughTableName } = breakExternalTableId(
-          field.through
-        )
-        definition.through = throughTableName
-        // don't support composite keys for relationships
-        definition.from = field.throughTo || table.primary[0]
-        definition.to = field.throughFrom || linkTable.primary[0]
-        definition.fromPrimary = table.primary[0]
-        definition.toPrimary = linkTable.primary[0]
-      } else {
-        // if no foreign key specified then use the name of the field in other table
-        definition.from = field.foreignKey || table.primary[0]
-        definition.to = field.fieldName
-      }
-      relationships.push(definition)
-    }
-    return relationships
-  }
-
   /**
    * This is a cached lookup, of relationship records, this is mainly for creating/deleting junction
    * information.
@@ -801,41 +526,6 @@ export class ExternalRequest<T extends Operation> {
     }
   }
 
-  /**
-   * This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which
-   * you have column overlap in relationships, e.g. we join a few different tables and they all have the
-   * concept of an ID, but for some of them it will be null (if they say don't have a relationship).
-   * Creating the specific list of fields that we desire, and excluding the ones that are no use to us
-   * is more performant and has the added benefit of protecting against this scenario.
-   */
-  buildFields(table: Table, includeRelations: boolean) {
-    function extractRealFields(table: Table, existing: string[] = []) {
-      return Object.entries(table.schema)
-        .filter(
-          column =>
-            column[1].type !== FieldType.LINK &&
-            column[1].type !== FieldType.FORMULA &&
-            !existing.find((field: string) => field === column[0])
-        )
-        .map(column => `${table.name}.${column[0]}`)
-    }
-    let fields = extractRealFields(table)
-    for (let field of Object.values(table.schema)) {
-      if (field.type !== FieldType.LINK || !includeRelations) {
-        continue
-      }
-      const { tableName: linkTableName } = breakExternalTableId(field.tableId)
-      if (linkTableName) {
-        const linkTable = this.tables[linkTableName]
-        if (linkTable) {
-          const linkedFields = extractRealFields(linkTable, fields)
-          fields = fields.concat(linkedFields)
-        }
-      }
-    }
-    return fields
-  }
-
   async run(config: RunConfig): Promise<ExternalRequestReturnType<T>> {
     const { operation, tableId } = this
     let { datasourceId, tableName } = breakExternalTableId(tableId)
@@ -869,14 +559,16 @@ export class ExternalRequest<T extends Operation> {
           delete sort?.[sortColumn]
           break
         case FieldType.NUMBER:
-          sort[sortColumn].type = SortType.number
+          if (sort && sort[sortColumn]) {
+            sort[sortColumn].type = SortType.NUMBER
+          }
           break
       }
     }
     filters = buildFilters(id, filters || {}, table)
-    const relationships = this.buildRelationships(table)
+    const relationships = buildExternalRelationships(table, this.tables)
 
-    const includeSqlRelationships =
+    const incRelationships =
       config.includeSqlRelationships === IncludeRelationship.INCLUDE
 
     // clean up row on ingress using schema
@@ -896,7 +588,11 @@ export class ExternalRequest<T extends Operation> {
       },
       resource: {
         // have to specify the fields to avoid column overlap (for SQL)
-        fields: isSql ? this.buildFields(table, includeSqlRelationships) : [],
+        fields: isSql
+          ? buildSqlFieldList(table, this.tables, {
+              relationships: incRelationships,
+            })
+          : [],
       },
       filters,
       sort,
@@ -935,9 +631,10 @@ export class ExternalRequest<T extends Operation> {
         processed.manyRelationships
       )
     }
-    const output = await this.outputProcessing(
+    const output = await sqlOutputProcessing(
-      responseRows,
+      response,
       table,
+      this.tables,
       relationships
     )
     // if reading it'll just be an array of rows, return whole thing

packages/server/src/api/controllers/row/alias.ts

@@ -155,7 +155,9 @@ export default class AliasTables {
     return map
   }
 
-  async queryWithAliasing(json: QueryJson): DatasourcePlusQueryResponse {
+  async queryWithAliasing(
+    json: QueryJson
+  ): Promise<DatasourcePlusQueryResponse> {
     const datasourceId = json.endpoint.datasourceId
     const datasource = await sdk.datasources.get(datasourceId)
 

packages/server/src/api/controllers/row/index.ts

@@ -13,7 +13,7 @@ import {
   PatchRowRequest,
   PatchRowResponse,
   Row,
-  SearchParams,
+  RowSearchParams,
   SearchRowRequest,
   SearchRowResponse,
   UserCtx,
@@ -192,7 +192,7 @@ export async function destroy(ctx: UserCtx<DeleteRowRequest>) {
 export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
   const tableId = utils.getTableId(ctx)
 
-  const searchParams: SearchParams = {
+  const searchParams: RowSearchParams = {
     ...ctx.request.body,
     tableId,
   }

packages/server/src/api/controllers/row/utils.ts

@@ -1,59 +0,0 @@
-import { InternalTables } from "../../../db/utils"
-import * as userController from "../user"
-import { context } from "@budibase/backend-core"
-import { Ctx, Row, UserCtx } from "@budibase/types"
-
-import validateJs from "validate.js"
-
-validateJs.extend(validateJs.validators.datetime, {
-  parse: function (value: string) {
-    return new Date(value).getTime()
-  },
-  // Input is a unix timestamp
-  format: function (value: string) {
-    return new Date(value).toISOString()
-  },
-})
-
-export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
-  const db = context.getAppDB()
-  let row: Row
-  // TODO remove special user case in future
-  if (tableId === InternalTables.USER_METADATA) {
-    ctx.params = {
-      id: rowId,
-    }
-    await userController.findMetadata(ctx)
-    row = ctx.body
-  } else {
-    row = await db.get(rowId)
-  }
-  if (row.tableId !== tableId) {
-    throw "Supplied tableId does not match the rows tableId"
-  }
-  return row
-}
-
-export function getTableId(ctx: Ctx): string {
-  // top priority, use the URL first
-  if (ctx.params?.sourceId) {
-    return ctx.params.sourceId
-  }
-  // now check for old way of specifying table ID
-  if (ctx.params?.tableId) {
-    return ctx.params.tableId
-  }
-  // check body for a table ID
-  if (ctx.request.body?.tableId) {
-    return ctx.request.body.tableId
-  }
-  // now check if a specific view name
-  if (ctx.params?.viewName) {
-    return ctx.params.viewName
-  }
-  throw new Error("Unable to find table ID in request")
-}
-
-export function isUserMetadataTable(tableId: string) {
-  return tableId === InternalTables.USER_METADATA
-}

packages/server/src/api/controllers/row/utils/basic.ts

@@ -0,0 +1,116 @@
+// need to handle table name + field or just field, depending on if relationships used
+import { FieldType, Row, Table } from "@budibase/types"
+import { generateRowIdField } from "../../../../integrations/utils"
+import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
+
+function extractFieldValue({
+  row,
+  tableName,
+  fieldName,
+  isLinked,
+}: {
+  row: Row
+  tableName: string
+  fieldName: string
+  isLinked: boolean
+}) {
+  let value = row[`${tableName}.${fieldName}`]
+  if (value == null && !isLinked) {
+    value = row[fieldName]
+  }
+  return value
+}
+
+export function getInternalRowId(row: Row, table: Table): string {
+  return extractFieldValue({
+    row,
+    tableName: table._id!,
+    fieldName: "_id",
+    isLinked: false,
+  })
+}
+
+export function generateIdForRow(
+  row: Row | undefined,
+  table: Table,
+  isLinked: boolean = false
+): string {
+  const primary = table.primary
+  if (!row || !primary) {
+    return ""
+  }
+  // build id array
+  let idParts = []
+  for (let field of primary) {
+    let fieldValue = extractFieldValue({
+      row,
+      tableName: table.name,
+      fieldName: field,
+      isLinked,
+    })
+    if (fieldValue != null) {
+      idParts.push(fieldValue)
+    }
+  }
+  if (idParts.length === 0) {
+    return ""
+  }
+  return generateRowIdField(idParts)
+}
+
+export function basicProcessing({
+  row,
+  table,
+  isLinked,
+  internal,
+}: {
+  row: Row
+  table: Table
+  isLinked: boolean
+  internal?: boolean
+}): Row {
+  const thisRow: Row = {}
+  // filter the row down to what is actually the row (not joined)
+  for (let field of Object.values(table.schema)) {
+    const fieldName = field.name
+    const value = extractFieldValue({
+      row,
+      tableName: table.name,
+      fieldName,
+      isLinked,
+    })
+    // all responses include "select col as table.col" so that overlaps are handled
+    if (value != null) {
+      thisRow[fieldName] = value
+    }
+  }
+  if (!internal) {
+    thisRow._id = generateIdForRow(row, table, isLinked)
+    thisRow.tableId = table._id
+    thisRow._rev = "rev"
+  } else {
+    for (let internalColumn of CONSTANT_INTERNAL_ROW_COLS) {
+      thisRow[internalColumn] = extractFieldValue({
+        row,
+        tableName: table._id!,
+        fieldName: internalColumn,
+        isLinked: false,
+      })
+    }
+  }
+  return thisRow
+}
+
+export function fixArrayTypes(row: Row, table: Table) {
+  for (let [fieldName, schema] of Object.entries(table.schema)) {
+    if (schema.type === FieldType.ARRAY && typeof row[fieldName] === "string") {
+      try {
+        row[fieldName] = JSON.parse(row[fieldName])
+      } catch (err) {
+        // couldn't convert back to array, ignore
+        delete row[fieldName]
+      }
+    }
+  }
+  return row
+}

packages/server/src/api/controllers/row/utils/index.ts

@@ -0,0 +1,3 @@
+export * from "./basic"
+export * from "./sqlUtils"
+export * from "./utils"

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -0,0 +1,194 @@
+import {
+  FieldType,
+  ManyToManyRelationshipFieldMetadata,
+  RelationshipFieldMetadata,
+  RelationshipsJson,
+  Row,
+  Table,
+} from "@budibase/types"
+import { breakExternalTableId } from "../../../../integrations/utils"
+import { basicProcessing } from "./basic"
+import { generateJunctionTableID } from "../../../../db/utils"
+
+type TableMap = Record<string, Table>
+
+export function isManyToMany(
+  field: RelationshipFieldMetadata
+): field is ManyToManyRelationshipFieldMetadata {
+  return !!(field as ManyToManyRelationshipFieldMetadata).through
+}
+
+/**
+ * This iterates through the returned rows and works out what elements of the rows
+ * actually match up to another row (based on primary keys) - this is pretty specific
+ * to SQL and the way that SQL relationships are returned based on joins.
+ * This is complicated, but the idea is that when a SQL query returns all the relations
+ * will be separate rows, with all of the data in each row. We have to decipher what comes
+ * from where (which tables) and how to convert that into budibase columns.
+ */
+export async function updateRelationshipColumns(
+  table: Table,
+  tables: TableMap,
+  row: Row,
+  rows: { [key: string]: Row },
+  relationships: RelationshipsJson[],
+  opts?: { sqs?: boolean }
+) {
+  const columns: { [key: string]: any } = {}
+  for (let relationship of relationships) {
+    const linkedTable = tables[relationship.tableName]
+    if (!linkedTable) {
+      continue
+    }
+    const fromColumn = `${table.name}.${relationship.from}`
+    const toColumn = `${linkedTable.name}.${relationship.to}`
+    // this is important when working with multiple relationships
+    // between the same tables, don't want to overlap/multiply the relations
+    if (
+      !relationship.through &&
+      row[fromColumn]?.toString() !== row[toColumn]?.toString()
+    ) {
+      continue
+    }
+
+    let linked = await basicProcessing({
+      row,
+      table: linkedTable,
+      isLinked: true,
+      internal: opts?.sqs,
+    })
+    if (!linked._id) {
+      continue
+    }
+    columns[relationship.column] = linked
+  }
+  for (let [column, related] of Object.entries(columns)) {
+    if (!row._id) {
+      continue
+    }
+    const rowId: string = row._id
+    if (!Array.isArray(rows[rowId][column])) {
+      rows[rowId][column] = []
+    }
+    // make sure relationship hasn't been found already
+    if (
+      !rows[rowId][column].find((relation: Row) => relation._id === related._id)
+    ) {
+      rows[rowId][column].push(related)
+    }
+  }
+  return rows
+}
+
+/**
+ * Gets the list of relationship JSON structures based on the columns in the table,
+ * this will be used by the underlying library to build whatever relationship mechanism
+ * it has (e.g. SQL joins).
+ */
+export function buildExternalRelationships(
+  table: Table,
+  tables: TableMap
+): RelationshipsJson[] {
+  const relationships = []
+  for (let [fieldName, field] of Object.entries(table.schema)) {
+    if (field.type !== FieldType.LINK) {
+      continue
+    }
+    const { tableName: linkTableName } = breakExternalTableId(field.tableId)
+    // no table to link to, this is not a valid relationships
+    if (!linkTableName || !tables[linkTableName]) {
+      continue
+    }
+    const linkTable = tables[linkTableName]
+    if (!table.primary || !linkTable.primary) {
+      continue
+    }
+    const definition: RelationshipsJson = {
+      tableName: linkTableName,
+      // need to specify where to put this back into
+      column: fieldName,
+    }
+    if (isManyToMany(field)) {
+      const { tableName: throughTableName } = breakExternalTableId(
+        field.through
+      )
+      definition.through = throughTableName
+      // don't support composite keys for relationships
+      definition.from = field.throughTo || table.primary[0]
+      definition.to = field.throughFrom || linkTable.primary[0]
+      definition.fromPrimary = table.primary[0]
+      definition.toPrimary = linkTable.primary[0]
+    } else {
+      // if no foreign key specified then use the name of the field in other table
+      definition.from = field.foreignKey || table.primary[0]
+      definition.to = field.fieldName
+    }
+    relationships.push(definition)
+  }
+  return relationships
+}
+
+export function buildInternalRelationships(table: Table): RelationshipsJson[] {
+  const relationships: RelationshipsJson[] = []
+  const links = Object.values(table.schema).filter(
+    column => column.type === FieldType.LINK
+  )
+  const tableId = table._id!
+  for (let link of links) {
+    if (link.type !== FieldType.LINK) {
+      continue
+    }
+    const linkTableId = link.tableId!
+    const junctionTableId = generateJunctionTableID(tableId, linkTableId)
+    const isFirstTable = tableId > linkTableId
+    relationships.push({
+      through: junctionTableId,
+      column: link.name,
+      tableName: linkTableId,
+      fromPrimary: "_id",
+      to: isFirstTable ? "doc2.rowId" : "doc1.rowId",
+      from: isFirstTable ? "doc1.rowId" : "doc2.rowId",
+      toPrimary: "_id",
+    })
+  }
+  return relationships
+}
+
+/**
+ * This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which
+ * you have column overlap in relationships, e.g. we join a few different tables and they all have the
+ * concept of an ID, but for some of them it will be null (if they say don't have a relationship).
+ * Creating the specific list of fields that we desire, and excluding the ones that are no use to us
+ * is more performant and has the added benefit of protecting against this scenario.
+ */
+export function buildSqlFieldList(
+  table: Table,
+  tables: TableMap,
+  opts?: { relationships: boolean }
+) {
+  function extractRealFields(table: Table, existing: string[] = []) {
+    return Object.entries(table.schema)
+      .filter(
+        column =>
+          column[1].type !== FieldType.LINK &&
+          column[1].type !== FieldType.FORMULA &&
+          !existing.find((field: string) => field === column[0])
+      )
+      .map(column => `${table.name}.${column[0]}`)
+  }
+  let fields = extractRealFields(table)
+  for (let field of Object.values(table.schema)) {
+    if (field.type !== FieldType.LINK || !opts?.relationships) {
+      continue
+    }
+    const { tableName: linkTableName } = breakExternalTableId(field.tableId)
+    if (linkTableName) {
+      const linkTable = tables[linkTableName]
+      if (linkTable) {
+        const linkedFields = extractRealFields(linkTable, fields)
+        fields = fields.concat(linkedFields)
+      }
+    }
+  }
+  return fields
+}

packages/server/src/api/controllers/row/utils/utils.ts

@@ -0,0 +1,189 @@
+import { InternalTables } from "../../../../db/utils"
+import * as userController from "../../user"
+import { context } from "@budibase/backend-core"
+import {
+  Ctx,
+  DatasourcePlusQueryResponse,
+  FieldType,
+  RelationshipsJson,
+  Row,
+  Table,
+  UserCtx,
+} from "@budibase/types"
+import {
+  processDates,
+  processFormulas,
+} from "../../../../utilities/rowProcessor"
+import { updateRelationshipColumns } from "./sqlUtils"
+import {
+  basicProcessing,
+  generateIdForRow,
+  fixArrayTypes,
+  getInternalRowId,
+} from "./basic"
+import sdk from "../../../../sdk"
+
+import validateJs from "validate.js"
+
+validateJs.extend(validateJs.validators.datetime, {
+  parse: function (value: string) {
+    return new Date(value).getTime()
+  },
+  // Input is a unix timestamp
+  format: function (value: string) {
+    return new Date(value).toISOString()
+  },
+})
+
+export async function processRelationshipFields(
+  table: Table,
+  tables: Record<string, Table>,
+  row: Row,
+  relationships: RelationshipsJson[]
+): Promise<Row> {
+  for (let relationship of relationships) {
+    const linkedTable = tables[relationship.tableName]
+    if (!linkedTable || !row[relationship.column]) {
+      continue
+    }
+    for (let key of Object.keys(row[relationship.column])) {
+      let relatedRow: Row = row[relationship.column][key]
+      // add this row as context for the relationship
+      for (let col of Object.values(linkedTable.schema)) {
+        if (col.type === FieldType.LINK && col.tableId === table._id) {
+          relatedRow[col.name] = [row]
+        }
+      }
+      // process additional types
+      relatedRow = processDates(table, relatedRow)
+      relatedRow = await processFormulas(linkedTable, relatedRow)
+      row[relationship.column][key] = relatedRow
+    }
+  }
+  return row
+}
+
+export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
+  const db = context.getAppDB()
+  let row: Row
+  // TODO remove special user case in future
+  if (tableId === InternalTables.USER_METADATA) {
+    ctx.params = {
+      id: rowId,
+    }
+    await userController.findMetadata(ctx)
+    row = ctx.body
+  } else {
+    row = await db.get(rowId)
+  }
+  if (row.tableId !== tableId) {
+    throw "Supplied tableId does not match the rows tableId"
+  }
+  return row
+}
+
+export function getTableId(ctx: Ctx): string {
+  // top priority, use the URL first
+  if (ctx.params?.sourceId) {
+    return ctx.params.sourceId
+  }
+  // now check for old way of specifying table ID
+  if (ctx.params?.tableId) {
+    return ctx.params.tableId
+  }
+  // check body for a table ID
+  if (ctx.request.body?.tableId) {
+    return ctx.request.body.tableId
+  }
+  // now check if a specific view name
+  if (ctx.params?.viewName) {
+    return ctx.params.viewName
+  }
+  throw new Error("Unable to find table ID in request")
+}
+
+export async function validate(
+  opts: { row: Row } & ({ tableId: string } | { table: Table })
+) {
+  let fetchedTable: Table
+  if ("tableId" in opts) {
+    fetchedTable = await sdk.tables.getTable(opts.tableId)
+  } else {
+    fetchedTable = opts.table
+  }
+  return sdk.rows.utils.validate({
+    ...opts,
+    table: fetchedTable,
+  })
+}
+
+export async function sqlOutputProcessing(
+  rows: DatasourcePlusQueryResponse,
+  table: Table,
+  tables: Record<string, Table>,
+  relationships: RelationshipsJson[],
+  opts?: { sqs?: boolean }
+): Promise<Row[]> {
+  if (!Array.isArray(rows) || rows.length === 0 || rows[0].read === true) {
+    return []
+  }
+  let finalRows: { [key: string]: Row } = {}
+  for (let row of rows as Row[]) {
+    let rowId = row._id
+    if (opts?.sqs) {
+      rowId = getInternalRowId(row, table)
+    } else if (!rowId) {
+      rowId = generateIdForRow(row, table)
+      row._id = rowId
+    }
+    // this is a relationship of some sort
+    if (finalRows[rowId]) {
+      finalRows = await updateRelationshipColumns(
+        table,
+        tables,
+        row,
+        finalRows,
+        relationships,
+        opts
+      )
+      continue
+    }
+    const thisRow = fixArrayTypes(
+      basicProcessing({
+        row,
+        table,
+        isLinked: false,
+        internal: opts?.sqs,
+      }),
+      table
+    )
+    if (thisRow._id == null) {
+      throw new Error("Unable to generate row ID for SQL rows")
+    }
+    finalRows[thisRow._id] = thisRow
+    // do this at end once its been added to the final rows
+    finalRows = await updateRelationshipColumns(
+      table,
+      tables,
+      row,
+      finalRows,
+      relationships
+    )
+  }
+
+  // make sure all related rows are correct
+  let finalRowArray = []
+  for (let row of Object.values(finalRows)) {
+    finalRowArray.push(
+      await processRelationshipFields(table, tables, row, relationships)
+    )
+  }
+
+  // process some additional types
+  finalRowArray = processDates(table, finalRowArray)
+  return finalRowArray
+}
+
+export function isUserMetadataTable(tableId: string) {
+  return tableId === InternalTables.USER_METADATA
+}

packages/server/src/api/controllers/row/views.ts

@@ -4,8 +4,8 @@ import {
   SearchRowResponse,
   SearchViewRowRequest,
   RequiredKeys,
-  SearchParams,
   SearchFilters,
+  RowSearchParams,
 } from "@budibase/types"
 import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../../sdk"
@@ -57,7 +57,7 @@ export async function searchView(
   }
 
   const searchOptions: RequiredKeys<SearchViewRowRequest> &
-    RequiredKeys<Pick<SearchParams, "tableId" | "query" | "fields">> = {
+    RequiredKeys<Pick<RowSearchParams, "tableId" | "query" | "fields">> = {
     tableId: view.tableId,
     query,
     fields: viewFields,

packages/server/src/api/controllers/static/index.ts

@@ -2,7 +2,7 @@ import { InvalidFileExtensions } from "@budibase/shared-core"
 import AppComponent from "./templates/BudibaseApp.svelte"
 import { join } from "../../../utilities/centralPath"
 import * as uuid from "uuid"
-import { ObjectStoreBuckets, devClientVersion } from "../../../constants"
+import { devClientVersion, ObjectStoreBuckets } from "../../../constants"
 import { processString } from "@budibase/string-templates"
 import {
   loadHandlebarsFile,
@@ -10,24 +10,24 @@ import {
   TOP_LEVEL_PATH,
 } from "../../../utilities/fileSystem"
 import env from "../../../environment"
-import { DocumentType } from "../../../db/utils"
 import {
+  BadRequestError,
+  configs,
   context,
   objectStore,
   utils,
-  configs,
-  BadRequestError,
 } from "@budibase/backend-core"
 import AWS from "aws-sdk"
 import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
 import {
-  UserCtx,
   App,
   Ctx,
-  ProcessAttachmentResponse,
+  DocumentType,
   Feature,
+  ProcessAttachmentResponse,
+  UserCtx,
 } from "@budibase/types"
 import {
   getAppMigrationVersion,
@@ -147,8 +147,7 @@ const requiresMigration = async (ctx: Ctx) => {
 
   const latestMigrationApplied = await getAppMigrationVersion(appId)
 
-  const requiresMigrations = latestMigrationApplied !== latestMigration
+  return latestMigrationApplied !== latestMigration
-  return requiresMigrations
 }
 
 export const serveApp = async function (ctx: UserCtx) {

packages/server/src/api/controllers/table/utils.ts

@@ -30,9 +30,9 @@ import {
   View,
   RelationshipFieldMetadata,
   FieldType,
-  FieldTypeSubtypes,
-  AttachmentFieldMetadata,
 } from "@budibase/types"
+import sdk from "../../../sdk"
+import env from "../../../environment"
 
 export async function clearColumns(table: Table, columnNames: string[]) {
   const db = context.getAppDB()
@@ -91,26 +91,6 @@ export async function checkForColumnUpdates(
     await checkForViewUpdates(updatedTable, deletedColumns, columnRename)
   }
 
-  const changedAttachmentSubtypeColumns = Object.values(
-    updatedTable.schema
-  ).filter(
-    (column): column is AttachmentFieldMetadata =>
-      column.type === FieldType.ATTACHMENT &&
-      column.subtype !== oldTable?.schema[column.name]?.subtype
-  )
-  for (const attachmentColumn of changedAttachmentSubtypeColumns) {
-    if (attachmentColumn.subtype === FieldTypeSubtypes.ATTACHMENT.SINGLE) {
-      attachmentColumn.constraints ??= { length: {} }
-      attachmentColumn.constraints.length ??= {}
-      attachmentColumn.constraints.length.maximum = 1
-      attachmentColumn.constraints.length.message =
-        "cannot contain multiple files"
-    } else {
-      delete attachmentColumn.constraints?.length?.maximum
-      delete attachmentColumn.constraints?.length?.message
-    }
-  }
-
   return { rows: updatedRows, table: updatedTable }
 }
 
@@ -342,6 +322,9 @@ class TableSaveFunctions {
       importRows: this.importRows,
       user: this.user,
     })
+    if (env.SQS_SEARCH_ENABLE) {
+      await sdk.tables.sqs.addTableToSqlite(table)
+    }
     return table
   }
 

packages/server/src/api/controllers/view/utils.ts

@@ -2,13 +2,18 @@ import {
   ViewName,
   generateMemoryViewID,
   getMemoryViewParams,
-  DocumentType,
   SEPARATOR,
 } from "../../../db/utils"
 import env from "../../../environment"
 import { context } from "@budibase/backend-core"
 import viewBuilder from "./viewBuilder"
-import { Database, DBView, DesignDocument, InMemoryView } from "@budibase/types"
+import {
+  Database,
+  DBView,
+  DocumentType,
+  DesignDocument,
+  InMemoryView,
+} from "@budibase/types"
 
 export async function getView(viewName: string) {
   const db = context.getAppDB()

packages/server/src/api/controllers/view/views.ts

@@ -4,7 +4,6 @@ import { csv, json, jsonWithSchema, Format, isFormat } from "./exporters"
 import { deleteView, getView, getViews, saveView } from "./utils"
 import { fetchView } from "../row"
 import { context, events } from "@budibase/backend-core"
-import { DocumentType } from "../../../db/utils"
 import sdk from "../../../sdk"
 import {
   FieldType,
@@ -14,6 +13,7 @@ import {
   TableExportFormat,
   TableSchema,
   View,
+  DocumentType,
 } from "@budibase/types"
 import { builderSocket } from "../../../websockets"
 

packages/server/src/api/routes/tests/row.spec.ts

@@ -6,14 +6,17 @@ import * as setup from "./utilities"
 import { context, InternalTable, tenancy } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
+  AttachmentFieldMetadata,
   AutoFieldSubType,
   Datasource,
+  DateFieldMetadata,
   DeleteRow,
   FieldSchema,
   FieldType,
   FieldTypeSubtypes,
   FormulaType,
   INTERNAL_TABLE_SOURCE_ID,
+  NumberFieldMetadata,
   QuotaUsageType,
   RelationshipType,
   Row,
@@ -232,9 +235,14 @@ describe.each([
           name: "str",
           constraints: { type: "string", presence: false },
         }
-        const attachment: FieldSchema = {
+        const singleAttachment: FieldSchema = {
-          type: FieldType.ATTACHMENT,
+          type: FieldType.ATTACHMENT_SINGLE,
-          name: "attachment",
+          name: "single attachment",
+          constraints: { presence: false },
+        }
+        const attachmentList: AttachmentFieldMetadata = {
+          type: FieldType.ATTACHMENTS,
+          name: "attachments",
           constraints: { type: "array", presence: false },
         }
         const bool: FieldSchema = {
@@ -242,12 +250,12 @@ describe.each([
           name: "boolean",
           constraints: { type: "boolean", presence: false },
         }
-        const number: FieldSchema = {
+        const number: NumberFieldMetadata = {
           type: FieldType.NUMBER,
           name: "str",
           constraints: { type: "number", presence: false },
         }
-        const datetime: FieldSchema = {
+        const datetime: DateFieldMetadata = {
           type: FieldType.DATETIME,
           name: "datetime",
           constraints: {
@@ -297,10 +305,12 @@ describe.each([
               boolUndefined: bool,
               boolString: bool,
               boolBool: bool,
-              attachmentNull: attachment,
+              singleAttachmentNull: singleAttachment,
-              attachmentUndefined: attachment,
+              singleAttachmentUndefined: singleAttachment,
-              attachmentEmpty: attachment,
+              attachmentListNull: attachmentList,
-              attachmentEmptyArrayStr: attachment,
+              attachmentListUndefined: attachmentList,
+              attachmentListEmpty: attachmentList,
+              attachmentListEmptyArrayStr: attachmentList,
               arrayFieldEmptyArrayStr: arrayField,
               arrayFieldArrayStrKnown: arrayField,
               arrayFieldNull: arrayField,
@@ -336,10 +346,12 @@ describe.each([
           boolString: "true",
           boolBool: true,
           tableId: table._id,
-          attachmentNull: null,
+          singleAttachmentNull: null,
-          attachmentUndefined: undefined,
+          singleAttachmentUndefined: undefined,
-          attachmentEmpty: "",
+          attachmentListNull: null,
-          attachmentEmptyArrayStr: "[]",
+          attachmentListUndefined: undefined,
+          attachmentListEmpty: "",
+          attachmentListEmptyArrayStr: "[]",
           arrayFieldEmptyArrayStr: "[]",
           arrayFieldUndefined: undefined,
           arrayFieldNull: null,
@@ -368,10 +380,12 @@ describe.each([
         expect(row.boolUndefined).toBe(undefined)
         expect(row.boolString).toBe(true)
         expect(row.boolBool).toBe(true)
-        expect(row.attachmentNull).toEqual([])
+        expect(row.singleAttachmentNull).toEqual(null)
-        expect(row.attachmentUndefined).toBe(undefined)
+        expect(row.singleAttachmentUndefined).toBe(undefined)
-        expect(row.attachmentEmpty).toEqual([])
+        expect(row.attachmentListNull).toEqual([])
-        expect(row.attachmentEmptyArrayStr).toEqual([])
+        expect(row.attachmentListUndefined).toBe(undefined)
+        expect(row.attachmentListEmpty).toEqual([])
+        expect(row.attachmentListEmptyArrayStr).toEqual([])
         expect(row.arrayFieldEmptyArrayStr).toEqual([])
         expect(row.arrayFieldNull).toEqual([])
         expect(row.arrayFieldUndefined).toEqual(undefined)
@@ -817,12 +831,44 @@ describe.each([
 
   isInternal &&
     describe("attachments", () => {
-      it("should allow enriching attachment rows", async () => {
+      it("should allow enriching single attachment rows", async () => {
         const table = await config.api.table.save(
           defaultTable({
             schema: {
               attachment: {
-                type: FieldType.ATTACHMENT,
+                type: FieldType.ATTACHMENT_SINGLE,
+                name: "attachment",
+                constraints: { presence: false },
+              },
+            },
+          })
+        )
+        const attachmentId = `${uuid.v4()}.csv`
+        const row = await config.api.row.save(table._id!, {
+          name: "test",
+          description: "test",
+          attachment: {
+            key: `${config.getAppId()}/attachments/${attachmentId}`,
+          },
+
+          tableId: table._id,
+        })
+        await config.withEnv({ SELF_HOSTED: "true" }, async () => {
+          return context.doInAppContext(config.getAppId(), async () => {
+            const enriched = await outputProcessing(table, [row])
+            expect((enriched as Row[])[0].attachment.url).toBe(
+              `/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
+            )
+          })
+        })
+      })
+
+      it("should allow enriching attachment list rows", async () => {
+        const table = await config.api.table.save(
+          defaultTable({
+            schema: {
+              attachment: {
+                type: FieldType.ATTACHMENTS,
                 name: "attachment",
                 constraints: { type: "array", presence: false },
               },
@@ -1272,7 +1318,6 @@ describe.each([
           ? {}
           : {
               hasNextPage: false,
-              bookmark: null,
             }),
       })
     })

packages/server/src/api/routes/tests/search.spec.ts

@@ -0,0 +1,282 @@
+import { tableForDatasource } from "../../../tests/utilities/structures"
+import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
+
+import * as setup from "./utilities"
+import {
+  Datasource,
+  EmptyFilterOption,
+  FieldType,
+  SearchFilters,
+  Table,
+} from "@budibase/types"
+
+jest.unmock("mssql")
+
+describe.each([
+  ["internal", undefined],
+  ["internal-sqs", undefined],
+  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+])("/api/:sourceId/search (%s)", (name, dsProvider) => {
+  const isSqs = name === "internal-sqs"
+  const config = setup.getConfig()
+
+  let envCleanup: (() => void) | undefined
+  let table: Table
+  let datasource: Datasource | undefined
+
+  beforeAll(async () => {
+    if (isSqs) {
+      envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" })
+    }
+    await config.init()
+    if (dsProvider) {
+      datasource = await config.createDatasource({
+        datasource: await dsProvider,
+      })
+    }
+  })
+
+  afterAll(async () => {
+    setup.afterAll()
+    if (envCleanup) {
+      envCleanup()
+    }
+  })
+
+  describe("strings", () => {
+    beforeEach(async () => {
+      table = await config.api.table.save(
+        tableForDatasource(datasource, {
+          schema: {
+            name: {
+              name: "name",
+              type: FieldType.STRING,
+            },
+          },
+        })
+      )
+    })
+
+    const rows = [{ name: "foo" }, { name: "bar" }]
+
+    interface StringSearchTest {
+      query: SearchFilters
+      expected: (typeof rows)[number][]
+    }
+
+    const stringSearchTests: StringSearchTest[] = [
+      { query: {}, expected: rows },
+      {
+        query: { onEmptyFilter: EmptyFilterOption.RETURN_ALL },
+        expected: rows,
+      },
+      {
+        query: { onEmptyFilter: EmptyFilterOption.RETURN_NONE },
+        expected: [],
+      },
+      { query: { string: { name: "foo" } }, expected: [rows[0]] },
+      { query: { string: { name: "none" } }, expected: [] },
+      { query: { fuzzy: { name: "oo" } }, expected: [rows[0]] },
+      { query: { equal: { name: "foo" } }, expected: [rows[0]] },
+      { query: { notEqual: { name: "foo" } }, expected: [rows[1]] },
+      { query: { oneOf: { name: ["foo"] } }, expected: [rows[0]] },
+      // { query: { contains: { name: "f" } }, expected: [0] },
+      // { query: { notContains: { name: ["f"] } }, expected: [1] },
+      // { query: { containsAny: { name: ["f"] } }, expected: [0] },
+    ]
+
+    it.each(stringSearchTests)(
+      `should be able to run query: $query`,
+      async ({ query, expected }) => {
+        const savedRows = await Promise.all(
+          rows.map(r => config.api.row.save(table._id!, r))
+        )
+        const { rows: foundRows } = await config.api.row.search(table._id!, {
+          tableId: table._id!,
+          query,
+        })
+        expect(foundRows).toEqual(
+          expect.arrayContaining(
+            expected.map(r =>
+              expect.objectContaining(savedRows.find(sr => sr.name === r.name)!)
+            )
+          )
+        )
+      }
+    )
+  })
+
+  describe("number", () => {
+    beforeEach(async () => {
+      table = await config.api.table.save(
+        tableForDatasource(datasource, {
+          schema: {
+            age: {
+              name: "age",
+              type: FieldType.NUMBER,
+            },
+          },
+        })
+      )
+    })
+
+    const rows = [{ age: 1 }, { age: 10 }]
+
+    interface NumberSearchTest {
+      query: SearchFilters
+      expected: (typeof rows)[number][]
+    }
+
+    const numberSearchTests: NumberSearchTest[] = [
+      { query: {}, expected: rows },
+      {
+        query: { onEmptyFilter: EmptyFilterOption.RETURN_ALL },
+        expected: rows,
+      },
+      {
+        query: { onEmptyFilter: EmptyFilterOption.RETURN_NONE },
+        expected: [],
+      },
+      { query: { equal: { age: 1 } }, expected: [rows[0]] },
+      { query: { equal: { age: 2 } }, expected: [] },
+      { query: { notEqual: { age: 1 } }, expected: [rows[1]] },
+      { query: { oneOf: { age: [1] } }, expected: [rows[0]] },
+      { query: { range: { age: { low: 1, high: 5 } } }, expected: [rows[0]] },
+      { query: { range: { age: { low: 0, high: 1 } } }, expected: [rows[0]] },
+      { query: { range: { age: { low: 3, high: 4 } } }, expected: [] },
+      { query: { range: { age: { low: 0, high: 11 } } }, expected: rows },
+    ]
+
+    it.each(numberSearchTests)(
+      `should be able to run query: $query`,
+      async ({ query, expected }) => {
+        const savedRows = await Promise.all(
+          rows.map(r => config.api.row.save(table._id!, r))
+        )
+        const { rows: foundRows } = await config.api.row.search(table._id!, {
+          tableId: table._id!,
+          query,
+        })
+        expect(foundRows).toEqual(
+          expect.arrayContaining(
+            expected.map(r =>
+              expect.objectContaining(savedRows.find(sr => sr.age === r.age)!)
+            )
+          )
+        )
+      }
+    )
+  })
+
+  describe("dates", () => {
+    beforeEach(async () => {
+      table = await config.api.table.save(
+        tableForDatasource(datasource, {
+          schema: {
+            dob: {
+              name: "dob",
+              type: FieldType.DATETIME,
+            },
+          },
+        })
+      )
+    })
+
+    const rows = [
+      { dob: new Date("2020-01-01") },
+      { dob: new Date("2020-01-10") },
+    ]
+
+    interface DateSearchTest {
+      query: SearchFilters
+      expected: (typeof rows)[number][]
+    }
+
+    const dateSearchTests: DateSearchTest[] = [
+      { query: {}, expected: rows },
+      {
+        query: { onEmptyFilter: EmptyFilterOption.RETURN_ALL },
+        expected: rows,
+      },
+      {
+        query: { onEmptyFilter: EmptyFilterOption.RETURN_NONE },
+        expected: [],
+      },
+      {
+        query: { equal: { dob: new Date("2020-01-01") } },
+        expected: [rows[0]],
+      },
+      { query: { equal: { dob: new Date("2020-01-02") } }, expected: [] },
+      {
+        query: { notEqual: { dob: new Date("2020-01-01") } },
+        expected: [rows[1]],
+      },
+      {
+        query: { oneOf: { dob: [new Date("2020-01-01")] } },
+        expected: [rows[0]],
+      },
+      {
+        query: {
+          range: {
+            dob: {
+              low: new Date("2020-01-01").toISOString(),
+              high: new Date("2020-01-05").toISOString(),
+            },
+          },
+        },
+        expected: [rows[0]],
+      },
+      {
+        query: {
+          range: {
+            dob: {
+              low: new Date("2020-01-01").toISOString(),
+              high: new Date("2020-01-10").toISOString(),
+            },
+          },
+        },
+        expected: rows,
+      },
+      {
+        query: {
+          range: {
+            dob: {
+              low: new Date("2020-01-05").toISOString(),
+              high: new Date("2020-01-10").toISOString(),
+            },
+          },
+        },
+        expected: [rows[1]],
+      },
+    ]
+
+    it.each(dateSearchTests)(
+      `should be able to run query: $query`,
+      async ({ query, expected }) => {
+        // TODO(samwho): most of these work for SQS, but not all. Fix 'em.
+        if (isSqs) {
+          return
+        }
+        const savedRows = await Promise.all(
+          rows.map(r => config.api.row.save(table._id!, r))
+        )
+        const { rows: foundRows } = await config.api.row.search(table._id!, {
+          tableId: table._id!,
+          query,
+        })
+        expect(foundRows).toEqual(
+          expect.arrayContaining(
+            expected.map(r =>
+              expect.objectContaining(
+                savedRows.find(sr => sr.dob === r.dob.toISOString())!
+              )
+            )
+          )
+        )
+      }
+    )
+  })
+})

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -652,7 +652,6 @@ describe.each([
             ? {}
             : {
                 hasNextPage: false,
-                bookmark: null,
               }),
         })
       })
@@ -705,7 +704,6 @@ describe.each([
             ? {}
             : {
                 hasNextPage: false,
-                bookmark: null,
               }),
         })
       })
@@ -813,7 +811,7 @@ describe.each([
           {
             field: "age",
             order: SortOrder.ASCENDING,
-            type: SortType.number,
+            type: SortType.NUMBER,
           },
           ["Danny", "Alice", "Charly", "Bob"],
         ],
@@ -835,7 +833,7 @@ describe.each([
           {
             field: "age",
             order: SortOrder.DESCENDING,
-            type: SortType.number,
+            type: SortType.NUMBER,
           },
           ["Bob", "Charly", "Alice", "Danny"],
         ],

packages/server/src/db/defaultData/datasource_bb_default.ts

@@ -299,7 +299,7 @@ export const DEFAULT_EMPLOYEE_TABLE_SCHEMA: Table = {
       sortable: false,
     },
     "Badge Photo": {
-      type: FieldType.ATTACHMENT,
+      type: FieldType.ATTACHMENTS,
       constraints: {
         type: FieldType.ARRAY,
         presence: false,
@@ -607,7 +607,7 @@ export const DEFAULT_EXPENSES_TABLE_SCHEMA: Table = {
       ignoreTimezones: true,
     },
     Attachment: {
-      type: FieldType.ATTACHMENT,
+      type: FieldType.ATTACHMENTS,
       constraints: {
         type: FieldType.ARRAY,
         presence: false,

packages/server/src/db/linkedRows/LinkDocument.ts

@@ -1,4 +1,4 @@
-import { generateLinkID } from "../utils"
+import { generateLinkID, generateJunctionTableID } from "../utils"
 import { FieldType, LinkDocument } from "@budibase/types"
 
 /**
@@ -16,6 +16,7 @@ import { FieldType, LinkDocument } from "@budibase/types"
 class LinkDocumentImpl implements LinkDocument {
   _id: string
   type: string
+  tableId: string
   doc1: {
     rowId: string
     fieldName: string
@@ -43,16 +44,20 @@ class LinkDocumentImpl implements LinkDocument {
       fieldName2
     )
     this.type = FieldType.LINK
-    this.doc1 = {
+    this.tableId = generateJunctionTableID(tableId1, tableId2)
+    const docA = {
       tableId: tableId1,
       fieldName: fieldName1,
       rowId: rowId1,
     }
-    this.doc2 = {
+    const docB = {
       tableId: tableId2,
       fieldName: fieldName2,
       rowId: rowId2,
     }
+    // have to determine which one will be doc1 - very important for SQL linking
+    this.doc1 = docA.tableId > docB.tableId ? docA : docB
+    this.doc2 = docA.tableId > docB.tableId ? docB : docA
   }
 }
 

packages/server/src/db/utils.ts

@@ -55,6 +55,14 @@ export const getUserMetadataParams = dbCore.getUserMetadataParams
 export const generateUserMetadataID = dbCore.generateUserMetadataID
 export const getGlobalIDFromUserMetadataID =
   dbCore.getGlobalIDFromUserMetadataID
+export const CONSTANT_INTERNAL_ROW_COLS = [
+  "_id",
+  "_rev",
+  "type",
+  "createdAt",
+  "updatedAt",
+  "tableId",
+]
 
 /**
  * Gets parameters for retrieving tables, this is a utility function for the getDocParams function.
@@ -286,6 +294,12 @@ export function generatePluginID(name: string) {
   return `${DocumentType.PLUGIN}${SEPARATOR}${name}`
 }
 
+export function generateJunctionTableID(tableId1: string, tableId2: string) {
+  const first = tableId1 > tableId2 ? tableId1 : tableId2
+  const second = tableId1 > tableId2 ? tableId2 : tableId1
+  return `${first}${SEPARATOR}${second}`
+}
+
 /**
  * Generates a new view ID.
  * @returns The new view ID which the view doc can be stored under.

packages/server/src/db/views/staticViews.ts

@@ -1,6 +1,6 @@
 import { context } from "@budibase/backend-core"
-import { DocumentType, SEPARATOR, ViewName } from "../utils"
+import { SEPARATOR, ViewName } from "../utils"
-import { LinkDocument, Row, SearchIndex } from "@budibase/types"
+import { DocumentType, LinkDocument, Row, SearchIndex } from "@budibase/types"
 
 const SCREEN_PREFIX = DocumentType.SCREEN + SEPARATOR
 

packages/server/src/definitions/datasource.ts

@@ -6,4 +6,5 @@
 
 export interface QueryOptions {
   disableReturning?: boolean
+  disableBindings?: boolean
 }

packages/server/src/environment.ts

@@ -86,6 +86,7 @@ const environment = {
   SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
   SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE,
   SQL_ALIASING_DISABLE: process.env.SQL_ALIASING_DISABLE,
+  SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE,
   // flags
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,

packages/server/src/integration-test/postgres.spec.ts

@@ -685,7 +685,6 @@ describe("postgres integrations", () => {
 
           expect(res.body).toEqual({
             rows: [],
-            bookmark: null,
             hasNextPage: false,
           })
         })
@@ -710,7 +709,6 @@ describe("postgres integrations", () => {
             rows: expect.arrayContaining(
               rows.map(r => expect.objectContaining(r.rowData))
             ),
-            bookmark: null,
             hasNextPage: false,
           })
           expect(res.body.rows).toHaveLength(rowsCount)
@@ -772,7 +770,6 @@ describe("postgres integrations", () => {
 
       expect(res.body).toEqual({
         rows: expect.arrayContaining(rowsToFilter.map(expect.objectContaining)),
-        bookmark: null,
         hasNextPage: false,
       })
       expect(res.body.rows).toHaveLength(4)

packages/server/src/integrations/base/query.ts

@@ -9,7 +9,7 @@ import sdk from "../../sdk"
 export async function makeExternalQuery(
   datasource: Datasource,
   json: QueryJson
-): DatasourcePlusQueryResponse {
+): Promise<DatasourcePlusQueryResponse> {
   datasource = await sdk.datasources.enrich(datasource)
   const Integration = await getIntegration(datasource.source)
   // query is the opinionated function

packages/server/src/integrations/base/sql.ts

@@ -1,7 +1,12 @@
 import { Knex, knex } from "knex"
 import { db as dbCore } from "@budibase/backend-core"
 import { QueryOptions } from "../../definitions/datasource"
-import { isIsoDateString, SqlClient, isValidFilter } from "../utils"
+import {
+  isIsoDateString,
+  SqlClient,
+  isValidFilter,
+  getNativeSql,
+} from "../utils"
 import SqlTableQueryBuilder from "./sqlTable"
 import {
   BBReferenceFieldMetadata,
@@ -11,14 +16,16 @@ import {
   JsonFieldMetadata,
   Operation,
   QueryJson,
+  SqlQuery,
   RelationshipsJson,
   SearchFilters,
   SortDirection,
+  SqlQueryBinding,
   Table,
 } from "@budibase/types"
 import environment from "../../environment"
 
-type QueryFunction = (query: Knex.SqlNative, operation: Operation) => any
+type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any
 
 const envLimit = environment.SQL_MAX_ROWS
   ? parseInt(environment.SQL_MAX_ROWS)
@@ -43,8 +50,11 @@ function likeKey(client: string, key: string): string {
       start = "["
       end = "]"
       break
+    case SqlClient.SQL_LITE:
+      start = end = "'"
+      break
     default:
-      throw "Unknown client"
+      throw new Error("Unknown client generating like key")
   }
   const parts = key.split(".")
   key = parts.map(part => `${start}${part}${end}`).join(".")
@@ -587,9 +597,15 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
    * which for the sake of mySQL stops adding the returning statement to inserts, updates and deletes.
    * @return the query ready to be passed to the driver.
    */
-  _query(json: QueryJson, opts: QueryOptions = {}): Knex.SqlNative | Knex.Sql {
+  _query(json: QueryJson, opts: QueryOptions = {}): SqlQuery | SqlQuery[] {
     const sqlClient = this.getSqlClient()
-    const client = knex({ client: sqlClient })
+    const config: { client: string; useNullAsDefault?: boolean } = {
+      client: sqlClient,
+    }
+    if (sqlClient === SqlClient.SQL_LITE) {
+      config.useNullAsDefault = true
+    }
+    const client = knex(config)
     let query: Knex.QueryBuilder
     const builder = new InternalBuilder(sqlClient)
     switch (this._operation(json)) {
@@ -615,7 +631,12 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
       default:
         throw `Operation type is not supported by SQL query builder`
     }
-    return query.toSQL().toNative()
+
+    if (opts?.disableBindings) {
+      return { sql: query.toString() }
+    } else {
+      return getNativeSql(query)
+    }
   }
 
   async getReturningRow(queryFn: QueryFunction, json: QueryJson) {
@@ -730,7 +751,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     )
   }
 
-  log(query: string, values?: any[]) {
+  log(query: string, values?: SqlQueryBinding) {
     if (!environment.SQL_LOGGING_ENABLE) {
       return
     }

packages/server/src/integrations/base/sqlTable.ts

@@ -8,8 +8,9 @@ import {
   RenameColumn,
   Table,
   FieldType,
+  SqlQuery,
 } from "@budibase/types"
-import { breakExternalTableId, SqlClient } from "../utils"
+import { breakExternalTableId, getNativeSql, SqlClient } from "../utils"
 import SchemaBuilder = Knex.SchemaBuilder
 import CreateTableBuilder = Knex.CreateTableBuilder
 import { utils } from "@budibase/shared-core"
@@ -199,7 +200,7 @@ class SqlTableQueryBuilder {
     return json.endpoint.operation
   }
 
-  _tableQuery(json: QueryJson): Knex.Sql | Knex.SqlNative {
+  _tableQuery(json: QueryJson): SqlQuery | SqlQuery[] {
     let client = knex({ client: this.sqlClient }).schema
     let schemaName = json?.endpoint?.schema
     if (schemaName) {
@@ -246,7 +247,7 @@ class SqlTableQueryBuilder {
           const tableName = schemaName
             ? `${schemaName}.${json.table.name}`
             : `${json.table.name}`
-          const sql = query.toSQL()
+          const sql = getNativeSql(query)
           if (Array.isArray(sql)) {
             for (const query of sql) {
               if (query.sql.startsWith("exec sp_rename")) {
@@ -265,7 +266,7 @@ class SqlTableQueryBuilder {
       default:
         throw "Table operation is of unknown type"
     }
-    return query.toSQL()
+    return getNativeSql(query)
   }
 }
 

packages/server/src/integrations/googlesheets.ts

@@ -336,7 +336,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     return { tables: externalTables, errors }
   }
 
-  async query(json: QueryJson): DatasourcePlusQueryResponse {
+  async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
     const sheet = json.endpoint.entityId
     switch (json.endpoint.operation) {
       case Operation.CREATE:

packages/server/src/integrations/microsoftSqlServer.ts

@@ -496,7 +496,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
     return response.recordset || [{ deleted: true }]
   }
 
-  async query(json: QueryJson): DatasourcePlusQueryResponse {
+  async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
     const schema = this.config.schema
     await this.connect()
     if (schema && schema !== DEFAULT_SCHEMA && json?.endpoint) {

packages/server/src/integrations/mysql.ts

@@ -13,6 +13,7 @@ import {
   Schema,
   TableSourceType,
   DatasourcePlusQueryResponse,
+  SqlQueryBinding,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -113,7 +114,7 @@ const defaultTypeCasting = function (field: any, next: any) {
   return next()
 }
 
-export function bindingTypeCoerce(bindings: any[]) {
+export function bindingTypeCoerce(bindings: SqlQueryBinding) {
   for (let i = 0; i < bindings.length; i++) {
     const binding = bindings[i]
     if (typeof binding !== "string") {
@@ -143,7 +144,7 @@ export function bindingTypeCoerce(bindings: any[]) {
 }
 
 class MySQLIntegration extends Sql implements DatasourcePlus {
-  private config: MySQLConfig
+  private readonly config: MySQLConfig
   private client?: mysql.Connection
 
   constructor(config: MySQLConfig) {
@@ -382,7 +383,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     return results.length ? results : [{ deleted: true }]
   }
 
-  async query(json: QueryJson): DatasourcePlusQueryResponse {
+  async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
     await this.connect()
     try {
       const queryFn = (query: any) =>

packages/server/src/integrations/oracle.ts

@@ -423,7 +423,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
       : [{ deleted: true }]
   }
 
-  async query(json: QueryJson): DatasourcePlusQueryResponse {
+  async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
     const operation = this._operation(json)
     const input = this._query(json, { disableReturning: true }) as SqlQuery
     if (Array.isArray(input)) {

packages/server/src/integrations/postgres.ts

@@ -421,7 +421,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
     return response.rows.length ? response.rows : [{ deleted: true }]
   }
 
-  async query(json: QueryJson): DatasourcePlusQueryResponse {
+  async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
     const operation = this._operation(json).toLowerCase()
     const input = this._query(json) as SqlQuery
     if (Array.isArray(input)) {

packages/server/src/integrations/tests/sqlAlias.spec.ts

@@ -1,10 +1,15 @@
-import { Datasource, Operation, QueryJson, SourceName } from "@budibase/types"
+import {
+  Datasource,
+  Operation,
+  QueryJson,
+  SourceName,
+  SqlQuery,
+} from "@budibase/types"
 import { join } from "path"
 import Sql from "../base/sql"
 import { SqlClient } from "../utils"
 import AliasTables from "../../api/controllers/row/alias"
 import { generator } from "@budibase/backend-core/tests"
-import { Knex } from "knex"
 
 function multiline(sql: string) {
   return sql.replace(/\n/g, "").replace(/ +/g, " ")
@@ -172,8 +177,8 @@ describe("Captures of real examples", () => {
       })
 
       // now check returning
-      let returningQuery: Knex.SqlNative = { sql: "", bindings: [] }
+      let returningQuery: SqlQuery | SqlQuery[] = { sql: "", bindings: [] }
-      SQL.getReturningRow((input: Knex.SqlNative) => {
+      SQL.getReturningRow((input: SqlQuery | SqlQuery[]) => {
         returningQuery = input
       }, queryJson)
       expect(returningQuery).toEqual({

packages/server/src/integrations/utils.ts

@@ -1,19 +1,15 @@
 import {
   SqlQuery,
   Table,
-  SearchFilters,
   Datasource,
   FieldType,
   TableSourceType,
 } from "@budibase/types"
 import { DocumentType, SEPARATOR } from "../db/utils"
-import {
+import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../constants"
-  InvalidColumns,
-  NoEmptyFilterStrings,
-  DEFAULT_BB_DATASOURCE_ID,
-} from "../constants"
 import { helpers } from "@budibase/shared-core"
 import env from "../environment"
+import { Knex } from "knex"
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
@@ -91,6 +87,7 @@ export enum SqlClient {
   POSTGRES = "pg",
   MY_SQL = "mysql2",
   ORACLE = "oracledb",
+  SQL_LITE = "sqlite3",
 }
 
 const isCloud = env.isProd() && !env.SELF_HOSTED
@@ -109,6 +106,23 @@ export function isInternalTableID(tableId: string) {
   return !isExternalTableID(tableId)
 }
 
+export function getNativeSql(
+  query: Knex.SchemaBuilder | Knex.QueryBuilder
+): SqlQuery | SqlQuery[] {
+  let sql = query.toSQL()
+  if (Array.isArray(sql)) {
+    return sql as SqlQuery[]
+  }
+  let native: Knex.SqlNative | undefined
+  if (sql.toNative) {
+    native = sql.toNative()
+  }
+  return {
+    sql: native?.sql || sql.sql,
+    bindings: native?.bindings || sql.bindings,
+  } as SqlQuery
+}
+
 export function isExternalTable(table: Table) {
   if (
     table?.sourceId &&
@@ -420,32 +434,3 @@ export function getPrimaryDisplay(testValue: unknown): string | undefined {
 export function isValidFilter(value: any) {
   return value != null && value !== ""
 }
-
-// don't do a pure falsy check, as 0 is included
-// https://github.com/Budibase/budibase/issues/10118
-export function removeEmptyFilters(filters: SearchFilters) {
-  for (let filterField of NoEmptyFilterStrings) {
-    if (!filters[filterField]) {
-      continue
-    }
-
-    for (let filterType of Object.keys(filters)) {
-      if (filterType !== filterField) {
-        continue
-      }
-      // don't know which one we're checking, type could be anything
-      const value = filters[filterType] as unknown
-      if (typeof value === "object") {
-        for (let [key, value] of Object.entries(
-          filters[filterType] as object
-        )) {
-          if (value == null || value === "") {
-            // @ts-ignore
-            delete filters[filterField][key]
-          }
-        }
-      }
-    }
-  }
-  return filters
-}

packages/server/src/middleware/builder.ts

@@ -1,8 +1,4 @@
-import {
+import { APP_DEV_PREFIX, getGlobalIDFromUserMetadataID } from "../db/utils"
-  APP_DEV_PREFIX,
-  DocumentType,
-  getGlobalIDFromUserMetadataID,
-} from "../db/utils"
 import {
   doesUserHaveLock,
   updateLock,
@@ -10,7 +6,7 @@ import {
   setDebounce,
 } from "../utilities/redis"
 import { db as dbCore, cache } from "@budibase/backend-core"
-import { UserCtx, Database } from "@budibase/types"
+import { DocumentType, UserCtx, Database } from "@budibase/types"
 
 const DEBOUNCE_TIME_SEC = 30
 

packages/server/src/sdk/app/backups/imports.ts

@@ -5,6 +5,7 @@ import {
   Automation,
   AutomationTriggerStepId,
   RowAttachment,
+  FieldType,
 } from "@budibase/types"
 import { getAutomationParams } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
@@ -58,10 +59,19 @@ export async function updateAttachmentColumns(prodAppId: string, db: Database) {
     updatedRows = updatedRows.concat(
       rows.map(row => {
         for (let column of columns) {
-          if (Array.isArray(row[column])) {
+          const columnType = table.schema[column].type
+          if (
+            columnType === FieldType.ATTACHMENTS &&
+            Array.isArray(row[column])
+          ) {
             row[column] = row[column].map((attachment: RowAttachment) =>
               rewriteAttachmentUrl(prodAppId, attachment)
             )
+          } else if (
+            columnType === FieldType.ATTACHMENT_SINGLE &&
+            row[column]
+          ) {
+            row[column] = rewriteAttachmentUrl(prodAppId, row[column])
           }
         }
         return row

packages/server/src/sdk/app/rows/attachments.ts

@@ -30,7 +30,10 @@ export async function getRowsWithAttachments(appId: string, table: Table) {
   const db = dbCore.getDB(appId)
   const attachmentCols: string[] = []
   for (let [key, column] of Object.entries(table.schema)) {
-    if (column.type === FieldType.ATTACHMENT) {
+    if (
+      column.type === FieldType.ATTACHMENTS ||
+      column.type === FieldType.ATTACHMENT_SINGLE
+    ) {
       attachmentCols.push(key)
     }
   }

packages/server/src/sdk/app/rows/search.ts

@@ -1,10 +1,18 @@
-import { Row, SearchFilters, SearchParams, SortOrder } from "@budibase/types"
+import {
+  Row,
+  RowSearchParams,
+  SearchFilters,
+  SearchResponse,
+} from "@budibase/types"
 import { isExternalTableID } from "../../../integrations/utils"
 import * as internal from "./search/internal"
 import * as external from "./search/external"
-import { Format } from "../../../api/controllers/view/exporters"
+import { NoEmptyFilterStrings } from "../../../constants"
+import * as sqs from "./search/sqs"
+import env from "../../../environment"
+import { ExportRowsParams, ExportRowsResult } from "./search/types"
 
-export { isValidFilter, removeEmptyFilters } from "../../../integrations/utils"
+export { isValidFilter } from "../../../integrations/utils"
 
 export interface ViewParams {
   calculation: string
@@ -19,29 +27,46 @@ function pickApi(tableId: any) {
   return internal
 }
 
-export async function search(options: SearchParams): Promise<{
+// don't do a pure falsy check, as 0 is included
-  rows: any[]
+// https://github.com/Budibase/budibase/issues/10118
-  hasNextPage?: boolean
+export function removeEmptyFilters(filters: SearchFilters) {
-  bookmark?: number | null
+  for (let filterField of NoEmptyFilterStrings) {
-}> {
+    if (!filters[filterField]) {
-  return pickApi(options.tableId).search(options)
+      continue
+    }
+
+    for (let filterType of Object.keys(filters)) {
+      if (filterType !== filterField) {
+        continue
+      }
+      // don't know which one we're checking, type could be anything
+      const value = filters[filterType] as unknown
+      if (typeof value === "object") {
+        for (let [key, value] of Object.entries(
+          filters[filterType] as object
+        )) {
+          if (value == null || value === "") {
+            // @ts-ignore
+            delete filters[filterField][key]
+          }
+        }
+      }
+    }
+  }
+  return filters
 }
 
-export interface ExportRowsParams {
+export async function search(
-  tableId: string
+  options: RowSearchParams
-  format: Format
+): Promise<SearchResponse<Row>> {
-  delimiter?: string
+  const isExternalTable = isExternalTableID(options.tableId)
-  rowIds?: string[]
+  if (isExternalTable) {
-  columns?: string[]
+    return external.search(options)
-  query?: SearchFilters
+  } else if (env.SQS_SEARCH_ENABLE) {
-  sort?: string
+    return sqs.search(options)
-  sortOrder?: SortOrder
+  } else {
-  customHeaders?: { [key: string]: string }
+    return internal.search(options)
-}
+  }
-
-export interface ExportRowsResult {
-  fileName: string
-  content: string
 }
 
 export async function exportRows(

packages/server/src/sdk/app/rows/search/external.ts

@@ -6,28 +6,31 @@ import {
   IncludeRelationship,
   Row,
   SearchFilters,
-  SearchParams,
+  RowSearchParams,
+  SearchResponse,
 } from "@budibase/types"
 import * as exporters from "../../../../api/controllers/view/exporters"
-import sdk from "../../../../sdk"
 import { handleRequest } from "../../../../api/controllers/row/external"
 import {
   breakExternalTableId,
   breakRowIdField,
 } from "../../../../integrations/utils"
-import { cleanExportRows } from "../utils"
 import { utils } from "@budibase/shared-core"
-import { ExportRowsParams, ExportRowsResult } from "../search"
+import { ExportRowsParams, ExportRowsResult } from "./types"
 import { HTTPError, db } from "@budibase/backend-core"
 import { searchInputMapping } from "./utils"
 import pick from "lodash/pick"
 import { outputProcessing } from "../../../../utilities/rowProcessor"
+import sdk from "../../../"
 
-export async function search(options: SearchParams) {
+export async function search(
+  options: RowSearchParams
+): Promise<SearchResponse<Row>> {
   const { tableId } = options
   const { paginate, query, ...params } = options
   const { limit } = params
-  let bookmark = (params.bookmark && parseInt(params.bookmark)) || null
+  let bookmark =
+    (params.bookmark && parseInt(params.bookmark as string)) || undefined
   if (paginate && !bookmark) {
     bookmark = 1
   }
@@ -92,7 +95,7 @@ export async function search(options: SearchParams) {
       rows = rows.map((r: any) => pick(r, fields))
     }
 
-    rows = await outputProcessing(table, rows, {
+    rows = await outputProcessing<Row[]>(table, rows, {
       preserveLinks: true,
       squash: true,
     })
@@ -158,7 +161,6 @@ export async function exportRows(
   if (!tableName) {
     throw new HTTPError("Could not find table name.", 400)
   }
-  const schema = datasource.entities[tableName].schema
 
   // Filter data to only specified columns if required
   if (columns && columns.length) {
@@ -173,7 +175,14 @@ export async function exportRows(
     rows = result.rows
   }
 
-  let exportRows = cleanExportRows(rows, schema, format, columns, customHeaders)
+  const schema = datasource.entities[tableName].schema
+  let exportRows = sdk.rows.utils.cleanExportRows(
+    rows,
+    schema,
+    format,
+    columns,
+    customHeaders
+  )
 
   let content: string
   switch (format) {

packages/server/src/sdk/app/rows/search/internal.ts

@@ -1,20 +1,19 @@
-import {
+import { context, db, HTTPError } from "@budibase/backend-core"
-  context,
-  db,
-  HTTPError,
-  SearchParams as InternalSearchParams,
-} from "@budibase/backend-core"
 import env from "../../../../environment"
-import { fullSearch, paginatedSearch } from "./internalSearch"
+import { fullSearch, paginatedSearch, searchInputMapping } from "./utils"
+import { getRowParams, InternalTables } from "../../../../db/utils"
 import {
+  Database,
   DocumentType,
-  getRowParams,
+  Row,
-  InternalTables,
+  RowSearchParams,
-} from "../../../../db/utils"
+  SearchResponse,
+  SortType,
+  Table,
+  User,
+} from "@budibase/types"
 import { getGlobalUsersFromMetadata } from "../../../../utilities/global"
 import { outputProcessing } from "../../../../utilities/rowProcessor"
-import { Database, Row, SearchParams, Table } from "@budibase/types"
-import { cleanExportRows } from "../utils"
 import {
   csv,
   Format,
@@ -29,17 +28,18 @@ import {
   migrateToInMemoryView,
 } from "../../../../api/controllers/view/utils"
 import sdk from "../../../../sdk"
-import { ExportRowsParams, ExportRowsResult } from "../search"
+import { ExportRowsParams, ExportRowsResult } from "./types"
-import { searchInputMapping } from "./utils"
 import pick from "lodash/pick"
 import { breakRowIdField } from "../../../../integrations/utils"
 
-export async function search(options: SearchParams) {
+export async function search(
+  options: RowSearchParams
+): Promise<SearchResponse<Row>> {
   const { tableId } = options
 
   const { paginate, query } = options
 
-  const params: InternalSearchParams<any> = {
+  const params: RowSearchParams = {
     tableId: options.tableId,
     sort: options.sort,
     sortOrder: options.sortOrder,
@@ -48,6 +48,7 @@ export async function search(options: SearchParams) {
     bookmark: options.bookmark,
     version: options.version,
     disableEscaping: options.disableEscaping,
+    query: {},
   }
 
   let table = await sdk.tables.getTable(tableId)
@@ -55,7 +56,8 @@ export async function search(options: SearchParams) {
   if (params.sort && !params.sortType) {
     const schema = table.schema
     const sortField = schema[params.sort]
-    params.sortType = sortField.type === "number" ? "number" : "string"
+    params.sortType =
+      sortField.type === "number" ? SortType.NUMBER : SortType.STRING
   }
 
   let response
@@ -69,7 +71,7 @@ export async function search(options: SearchParams) {
   if (response.rows && response.rows.length) {
     // enrich with global users if from users table
     if (tableId === InternalTables.USER_METADATA) {
-      response.rows = await getGlobalUsersFromMetadata(response.rows)
+      response.rows = await getGlobalUsersFromMetadata(response.rows as User[])
     }
 
     if (options.fields) {
@@ -100,10 +102,10 @@ export async function exportRows(
   const db = context.getAppDB()
   const table = await sdk.tables.getTable(tableId)
 
-  let result
+  let result: Row[] = []
   if (rowIds) {
     let response = (
-      await db.allDocs({
+      await db.allDocs<Row>({
         include_docs: true,
         keys: rowIds.map((row: string) => {
           const ids = breakRowIdField(row)
@@ -116,9 +118,9 @@ export async function exportRows(
           return ids[0]
         }),
       })
-    ).rows.map(row => row.doc)
+    ).rows.map(row => row.doc!)
 
-    result = await outputProcessing(table, response)
+    result = await outputProcessing<Row[]>(table, response)
   } else if (query) {
     let searchResponse = await search({
       tableId,
@@ -145,7 +147,13 @@ export async function exportRows(
     rows = result
   }
 
-  let exportRows = cleanExportRows(rows, schema, format, columns, customHeaders)
+  let exportRows = sdk.rows.utils.cleanExportRows(
+    rows,
+    schema,
+    format,
+    columns,
+    customHeaders
+  )
   if (format === Format.CSV) {
     return {
       fileName: "export.csv",

packages/server/src/sdk/app/rows/search/internalSearch.ts

@@ -1,18 +0,0 @@
-import { db as dbCore, context, SearchParams } from "@budibase/backend-core"
-import { SearchFilters, Row, SearchIndex } from "@budibase/types"
-
-export async function paginatedSearch(
-  query: SearchFilters,
-  params: SearchParams<Row>
-) {
-  const appId = context.getAppId()
-  return dbCore.paginatedSearch(appId!, SearchIndex.ROWS, query, params)
-}
-
-export async function fullSearch(
-  query: SearchFilters,
-  params: SearchParams<Row>
-) {
-  const appId = context.getAppId()
-  return dbCore.fullSearch(appId!, SearchIndex.ROWS, query, params)
-}

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -0,0 +1,190 @@
+import {
+  FieldType,
+  Operation,
+  QueryJson,
+  RelationshipFieldMetadata,
+  Row,
+  SearchFilters,
+  RowSearchParams,
+  SearchResponse,
+  SortDirection,
+  SortOrder,
+  SortType,
+  Table,
+} from "@budibase/types"
+import SqlQueryBuilder from "../../../../integrations/base/sql"
+import { SqlClient } from "../../../../integrations/utils"
+import {
+  buildInternalRelationships,
+  sqlOutputProcessing,
+} from "../../../../api/controllers/row/utils"
+import sdk from "../../../index"
+import { context } from "@budibase/backend-core"
+import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
+
+function buildInternalFieldList(
+  table: Table,
+  tables: Table[],
+  opts: { relationships: boolean } = { relationships: true }
+) {
+  let fieldList: string[] = []
+  fieldList = fieldList.concat(
+    CONSTANT_INTERNAL_ROW_COLS.map(col => `${table._id}.${col}`)
+  )
+  if (opts.relationships) {
+    for (let col of Object.values(table.schema)) {
+      if (col.type === FieldType.LINK) {
+        const linkCol = col as RelationshipFieldMetadata
+        const relatedTable = tables.find(
+          table => table._id === linkCol.tableId
+        )!
+        fieldList = fieldList.concat(
+          buildInternalFieldList(relatedTable, tables, { relationships: false })
+        )
+      } else {
+        fieldList.push(`${table._id}.${col.name}`)
+      }
+    }
+  }
+  return fieldList
+}
+
+function tableInFilter(name: string) {
+  return `:${name}.`
+}
+
+function cleanupFilters(filters: SearchFilters, tables: Table[]) {
+  for (let filter of Object.values(filters)) {
+    if (typeof filter !== "object") {
+      continue
+    }
+    for (let [key, keyFilter] of Object.entries(filter)) {
+      if (keyFilter === "") {
+        delete filter[key]
+      }
+
+      // relationship, switch to table ID
+      const tableRelated = tables.find(
+        table =>
+          table.originalName && key.includes(tableInFilter(table.originalName))
+      )
+      if (tableRelated && tableRelated.originalName) {
+        filter[
+          key.replace(
+            tableInFilter(tableRelated.originalName),
+            tableInFilter(tableRelated._id!)
+          )
+        ] = filter[key]
+        delete filter[key]
+      }
+    }
+  }
+  return filters
+}
+
+function buildTableMap(tables: Table[]) {
+  const tableMap: Record<string, Table> = {}
+  for (let table of tables) {
+    // update the table name, should never query by name for SQLite
+    table.originalName = table.name
+    table.name = table._id!
+    tableMap[table._id!] = table
+  }
+  return tableMap
+}
+
+export async function search(
+  options: RowSearchParams
+): Promise<SearchResponse<Row>> {
+  const { tableId, paginate, query, ...params } = options
+
+  const builder = new SqlQueryBuilder(SqlClient.SQL_LITE)
+  const allTables = await sdk.tables.getAllInternalTables()
+  const allTablesMap = buildTableMap(allTables)
+  const table = allTables.find(table => table._id === tableId)
+  if (!table) {
+    throw new Error("Unable to find table")
+  }
+
+  const relationships = buildInternalRelationships(table)
+
+  const request: QueryJson = {
+    endpoint: {
+      // not important, we query ourselves
+      datasourceId: "internal",
+      entityId: table._id!,
+      operation: Operation.READ,
+    },
+    filters: cleanupFilters(query, allTables),
+    table,
+    meta: {
+      table,
+      tables: allTablesMap,
+    },
+    resource: {
+      fields: buildInternalFieldList(table, allTables),
+    },
+    relationships,
+  }
+  // make sure only rows returned
+  request.filters!.equal = {
+    ...request.filters?.equal,
+    type: "row",
+  }
+
+  if (params.sort && !params.sortType) {
+    const sortField = table.schema[params.sort]
+    const sortType =
+      sortField.type === FieldType.NUMBER ? SortType.NUMBER : SortType.STRING
+    const sortDirection =
+      params.sortOrder === SortOrder.ASCENDING
+        ? SortDirection.ASCENDING
+        : SortDirection.DESCENDING
+    request.sort = {
+      [sortField.name]: {
+        direction: sortDirection,
+        type: sortType as SortType,
+      },
+    }
+  }
+  if (paginate && params.limit) {
+    request.paginate = {
+      limit: params.limit,
+      page: params.bookmark,
+    }
+  }
+  try {
+    const query = builder._query(request, {
+      disableReturning: true,
+      disableBindings: true,
+    })
+
+    if (Array.isArray(query)) {
+      throw new Error("SQS cannot currently handle multiple queries")
+    }
+
+    let sql = query.sql
+
+    // quick hack for docIds
+    sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
+    sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
+
+    const db = context.getAppDB()
+    const rows = await db.sql<Row>(sql)
+
+    return {
+      rows: await sqlOutputProcessing(
+        rows,
+        table!,
+        allTablesMap,
+        relationships,
+        {
+          sqs: true,
+        }
+      ),
+    }
+  } catch (err: any) {
+    const msg = typeof err === "string" ? err : err.message
+    throw new Error(`Unable to search by SQL - ${msg}`)
+  }
+}

packages/server/src/sdk/app/rows/search/tests/external.spec.ts

@@ -6,7 +6,7 @@ import {
   Row,
   SourceName,
   Table,
-  SearchParams,
+  RowSearchParams,
   TableSourceType,
 } from "@budibase/types"
 
@@ -108,7 +108,7 @@ describe("external search", () => {
     await config.doInContext(config.appId, async () => {
       const tableId = config.table!._id!
 
-      const searchParams: SearchParams = {
+      const searchParams: RowSearchParams = {
         tableId,
         query: {},
       }
@@ -125,7 +125,7 @@ describe("external search", () => {
     await config.doInContext(config.appId, async () => {
       const tableId = config.table!._id!
 
-      const searchParams: SearchParams = {
+      const searchParams: RowSearchParams = {
         tableId,
         query: {},
         fields: ["name", "age"],
@@ -149,7 +149,7 @@ describe("external search", () => {
     await config.doInContext(config.appId, async () => {
       const tableId = config.table!._id!
 
-      const searchParams: SearchParams = {
+      const searchParams: RowSearchParams = {
         tableId,
         query: {
           oneOf: {