Merge branch 'master' into prefill-cards

.eslintrc.json

@@ -1,4 +1,5 @@
 {
+  "root": true,
   "env": {
     "browser": true,
     "es6": true,

.github/workflows/budibase_ci.yml

@@ -170,7 +170,8 @@ jobs:
           docker pull mongo:7.0-jammy &
           docker pull mariadb:lts &
           docker pull testcontainers/ryuk:0.5.1 &
-          docker pull budibase/couchdb:v3.2.1-sql &
+          docker pull budibase/couchdb:v3.2.1-sqs &
+          docker pull minio/minio &
           docker pull redis &
 
           wait $(jobs -p)

.prettierignore

@@ -12,4 +12,5 @@ packages/pro/coverage
 packages/account-portal/packages/ui/build
 packages/account-portal/packages/ui/.routify
 packages/account-portal/packages/server/build
+packages/account-portal/packages/server/coverage
 **/*.ivm.bundle.js

charts/budibase/templates/automation-worker-service-hpa.yaml

@@ -2,7 +2,7 @@
 apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }}
 kind: HorizontalPodAutoscaler
 metadata:
-  name: {{ include "budibase.fullname" . }}-apps
+  name: {{ include "budibase.fullname" . }}-automation-worker
   labels:
     {{- include "budibase.labels" . | nindent 4 }}
 spec:

globalSetup.ts

@@ -46,7 +46,7 @@ export default async function setup() {
   await killContainers(containers)
 
   try {
-    let couchdb = new GenericContainer("budibase/couchdb:v3.2.1-sqs")
+    const couchdb = new GenericContainer("budibase/couchdb:v3.2.1-sqs")
       .withExposedPorts(5984, 4984)
       .withEnvironment({
         COUCHDB_PASSWORD: "budibase",
@@ -69,7 +69,20 @@ export default async function setup() {
         ).withStartupTimeout(20000)
       )
 
-    await couchdb.start()
+    const minio = new GenericContainer("minio/minio")
+      .withExposedPorts(9000)
+      .withCommand(["server", "/data"])
+      .withEnvironment({
+        MINIO_ACCESS_KEY: "budibase",
+        MINIO_SECRET_KEY: "budibase",
+      })
+      .withLabels({ "com.budibase": "true" })
+      .withReuse()
+      .withWaitStrategy(
+        Wait.forHttp("/minio/health/ready", 9000).withStartupTimeout(10000)
+      )
+
+    await Promise.all([couchdb.start(), minio.start()])
   } finally {
     lockfile.unlockSync(lockPath)
   }

hosting/couchdb/runner.v2.sh

@@ -70,10 +70,10 @@ sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/clouseau/clouse
 /opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
 
 # Start CouchDB.
-/docker-entrypoint.sh /opt/couchdb/bin/couchdb &
+/docker-entrypoint.sh /opt/couchdb/bin/couchdb > /dev/stdout 2>&1 &
 
-# Start SQS.
+# Start SQS. Use 127.0.0.1 instead of localhost to avoid IPv6 issues.
-/opt/sqs/sqs --server "http://localhost:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 &
+/opt/sqs/sqs --server "http://127.0.0.1:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 > /dev/stdout 2>&1 &
 
 # Wait for CouchDB to start up.
 while [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; do

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.23.12",
+  "version": "2.25.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/context/mainContext.ts

@@ -281,7 +281,7 @@ export function doInScimContext(task: any) {
   return newContext(updates, task)
 }
 
-export async function ensureSnippetContext() {
+export async function ensureSnippetContext(enabled = !env.isTest()) {
   const ctx = getCurrentContext()
 
   // If we've already added snippets to context, continue
@@ -292,7 +292,7 @@ export async function ensureSnippetContext() {
   // Otherwise get snippets for this app and update context
   let snippets: Snippet[] | undefined
   const db = getAppDB()
-  if (db && !env.isTest()) {
+  if (db && enabled) {
     const app = await db.get<App>(DocumentType.APP_METADATA)
     snippets = app.snippets
   }

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -3,11 +3,11 @@ import {
   AllDocsResponse,
   AnyDocument,
   Database,
-  DatabaseOpts,
-  DatabaseQueryOpts,
-  DatabasePutOpts,
   DatabaseCreateIndexOpts,
   DatabaseDeleteIndexOpts,
+  DatabaseOpts,
+  DatabasePutOpts,
+  DatabaseQueryOpts,
   Document,
   isDocument,
   RowResponse,
@@ -17,7 +17,7 @@ import {
 import { getCouchInfo } from "./connections"
 import { directCouchUrlCall } from "./utils"
 import { getPouchDB } from "./pouchDB"
-import { WriteStream, ReadStream } from "fs"
+import { ReadStream, WriteStream } from "fs"
 import { newid } from "../../docIds/newid"
 import { SQLITE_DESIGN_DOC_ID } from "../../constants"
 import { DDInstrumentedDatabase } from "../instrumentation"
@@ -38,6 +38,39 @@ function buildNano(couchInfo: { url: string; cookie: string }) {
 
 type DBCall<T> = () => Promise<T>
 
+class CouchDBError extends Error {
+  status: number
+  statusCode: number
+  reason: string
+  name: string
+  errid: string
+  error: string
+  description: string
+
+  constructor(
+    message: string,
+    info: {
+      status: number | undefined
+      statusCode: number | undefined
+      name: string
+      errid: string
+      description: string
+      reason: string
+      error: string
+    }
+  ) {
+    super(message)
+    const statusCode = info.status || info.statusCode || 500
+    this.status = statusCode
+    this.statusCode = statusCode
+    this.reason = info.reason
+    this.name = info.name
+    this.errid = info.errid
+    this.description = info.description
+    this.error = info.error
+  }
+}
+
 export function DatabaseWithConnection(
   dbName: string,
   connection: string,
@@ -119,7 +152,7 @@ export class DatabaseImpl implements Database {
       } catch (err: any) {
         // Handling race conditions
         if (err.statusCode !== 412) {
-          throw err
+          throw new CouchDBError(err.message, err)
         }
       }
     }
@@ -138,10 +171,9 @@ export class DatabaseImpl implements Database {
       if (err.statusCode === 404 && err.reason === DATABASE_NOT_FOUND) {
         await this.checkAndCreateDb()
         return await this.performCall(call)
-      } else if (err.statusCode) {
-        err.status = err.statusCode
       }
-      throw err
+      // stripping the error down the props which are safe/useful, drop everything else
+      throw new CouchDBError(`CouchDB error: ${err.message}`, err)
     }
   }
 
@@ -288,7 +320,7 @@ export class DatabaseImpl implements Database {
       if (err.statusCode === 404) {
         return
       } else {
-        throw { ...err, status: err.statusCode }
+        throw new CouchDBError(err.message, err)
       }
     }
   }

packages/backend-core/src/db/lucene.ts

@@ -12,6 +12,10 @@ import { dataFilters } from "@budibase/shared-core"
 
 export const removeKeyNumbering = dataFilters.removeKeyNumbering
 
+function isEmpty(value: any) {
+  return value == null || value === ""
+}
+
 /**
  * Class to build lucene query URLs.
  * Optionally takes a base lucene query object.
@@ -282,15 +286,14 @@ export class QueryBuilder<T> {
     }
 
     const equal = (key: string, value: any) => {
-      // 0 evaluates to false, which means we would return all rows if we don't check it
+      if (isEmpty(value)) {
-      if (!value && value !== 0) {
         return null
       }
       return `${key}:${builder.preprocess(value, allPreProcessingOpts)}`
     }
 
     const contains = (key: string, value: any, mode = "AND") => {
-      if (!value || (Array.isArray(value) && value.length === 0)) {
+      if (isEmpty(value)) {
         return null
       }
       if (!Array.isArray(value)) {
@@ -306,7 +309,7 @@ export class QueryBuilder<T> {
     }
 
     const fuzzy = (key: string, value: any) => {
-      if (!value) {
+      if (isEmpty(value)) {
         return null
       }
       value = builder.preprocess(value, {
@@ -328,7 +331,7 @@ export class QueryBuilder<T> {
     }
 
     const oneOf = (key: string, value: any) => {
-      if (!value) {
+      if (isEmpty(value)) {
         return `*:*`
       }
       if (!Array.isArray(value)) {
@@ -386,7 +389,7 @@ export class QueryBuilder<T> {
     // Construct the actual lucene search query string from JSON structure
     if (this.#query.string) {
       build(this.#query.string, (key: string, value: any) => {
-        if (!value) {
+        if (isEmpty(value)) {
           return null
         }
         value = builder.preprocess(value, {
@@ -399,7 +402,7 @@ export class QueryBuilder<T> {
     }
     if (this.#query.range) {
       build(this.#query.range, (key: string, value: any) => {
-        if (!value) {
+        if (isEmpty(value)) {
           return null
         }
         if (value.low == null || value.low === "") {
@@ -421,7 +424,7 @@ export class QueryBuilder<T> {
     }
     if (this.#query.notEqual) {
       build(this.#query.notEqual, (key: string, value: any) => {
-        if (!value) {
+        if (isEmpty(value)) {
           return null
         }
         if (typeof value === "boolean") {
@@ -431,10 +434,28 @@ export class QueryBuilder<T> {
       })
     }
     if (this.#query.empty) {
-      build(this.#query.empty, (key: string) => `(*:* -${key}:["" TO *])`)
+      build(this.#query.empty, (key: string) => {
+        // Because the structure of an empty filter looks like this:
+        //   { empty: { someKey: null } }
+        //
+        // The check inside of `build` does not set `allFiltersEmpty`, which results
+        // in weird behaviour when the empty filter is the only filter. We get around
+        // this by setting `allFiltersEmpty` to false here.
+        allFiltersEmpty = false
+        return `(*:* -${key}:["" TO *])`
+      })
     }
     if (this.#query.notEmpty) {
-      build(this.#query.notEmpty, (key: string) => `${key}:["" TO *]`)
+      build(this.#query.notEmpty, (key: string) => {
+        // Because the structure of a notEmpty filter looks like this:
+        //   { notEmpty: { someKey: null } }
+        //
+        // The check inside of `build` does not set `allFiltersEmpty`, which results
+        // in weird behaviour when the empty filter is the only filter. We get around
+        // this by setting `allFiltersEmpty` to false here.
+        allFiltersEmpty = false
+        return `${key}:["" TO *]`
+      })
     }
     if (this.#query.oneOf) {
       build(this.#query.oneOf, oneOf)

packages/backend-core/src/objectStore/objectStore.ts

@@ -13,13 +13,14 @@ import { bucketTTLConfig, budibaseTempDir } from "./utils"
 import { v4 } from "uuid"
 import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
 import fsp from "fs/promises"
+import { HeadObjectOutput } from "aws-sdk/clients/s3"
 
 const streamPipeline = promisify(stream.pipeline)
 // use this as a temporary store of buckets that are being created
 const STATE = {
   bucketCreationPromises: {},
 }
-const signedFilePrefix = "/files/signed"
+export const SIGNED_FILE_PREFIX = "/files/signed"
 
 type ListParams = {
   ContinuationToken?: string
@@ -40,8 +41,13 @@ type UploadParams = BaseUploadParams & {
   path?: string | PathLike
 }
 
-type StreamUploadParams = BaseUploadParams & {
+export type StreamTypes =
-  stream: ReadStream
+  | ReadStream
+  | NodeJS.ReadableStream
+  | ReadableStream<Uint8Array>
+
+export type StreamUploadParams = BaseUploadParams & {
+  stream?: StreamTypes
 }
 
 const CONTENT_TYPE_MAP: any = {
@@ -83,7 +89,7 @@ export function ObjectStore(
   bucket: string,
   opts: { presigning: boolean } = { presigning: false }
 ) {
-  const config: any = {
+  const config: AWS.S3.ClientConfiguration = {
     s3ForcePathStyle: true,
     signatureVersion: "v4",
     apiVersion: "2006-03-01",
@@ -174,11 +180,9 @@ export async function upload({
   const objectStore = ObjectStore(bucketName)
   const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
 
-  if (ttl && (bucketCreated.created || bucketCreated.exists)) {
+  if (ttl && bucketCreated.created) {
     let ttlConfig = bucketTTLConfig(bucketName, ttl)
-    if (objectStore.putBucketLifecycleConfiguration) {
+    await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
-      await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
-    }
   }
 
   let contentType = type
@@ -222,11 +226,9 @@ export async function streamUpload({
   const objectStore = ObjectStore(bucketName)
   const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
 
-  if (ttl && (bucketCreated.created || bucketCreated.exists)) {
+  if (ttl && bucketCreated.created) {
     let ttlConfig = bucketTTLConfig(bucketName, ttl)
-    if (objectStore.putBucketLifecycleConfiguration) {
+    await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
-      await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
-    }
   }
 
   // Set content type for certain known extensions
@@ -333,7 +335,7 @@ export function getPresignedUrl(
     const signedUrl = new URL(url)
     const path = signedUrl.pathname
     const query = signedUrl.search
-    return `${signedFilePrefix}${path}${query}`
+    return `${SIGNED_FILE_PREFIX}${path}${query}`
   }
 }
 
@@ -521,6 +523,26 @@ export async function getReadStream(
   return client.getObject(params).createReadStream()
 }
 
+export async function getObjectMetadata(
+  bucket: string,
+  path: string
+): Promise<HeadObjectOutput> {
+  bucket = sanitizeBucket(bucket)
+  path = sanitizeKey(path)
+
+  const client = ObjectStore(bucket)
+  const params = {
+    Bucket: bucket,
+    Key: path,
+  }
+
+  try {
+    return await client.headObject(params).promise()
+  } catch (err: any) {
+    throw new Error("Unable to retrieve metadata from object")
+  }
+}
+
 /*
 Given a signed url like '/files/signed/tmp-files-attachments/app_123456/myfile.txt' extract
 the bucket and the path from it
@@ -530,7 +552,9 @@ export function extractBucketAndPath(
 ): { bucket: string; path: string } | null {
   const baseUrl = url.split("?")[0]
 
-  const regex = new RegExp(`^${signedFilePrefix}/(?<bucket>[^/]+)/(?<path>.+)$`)
+  const regex = new RegExp(
+    `^${SIGNED_FILE_PREFIX}/(?<bucket>[^/]+)/(?<path>.+)$`
+  )
   const match = baseUrl.match(regex)
 
   if (match && match.groups) {

packages/backend-core/src/objectStore/utils.ts

@@ -1,9 +1,14 @@
-import { join } from "path"
+import path, { join } from "path"
 import { tmpdir } from "os"
 import fs from "fs"
 import env from "../environment"
 import { PutBucketLifecycleConfigurationRequest } from "aws-sdk/clients/s3"
-
+import * as objectStore from "./objectStore"
+import {
+  AutomationAttachment,
+  AutomationAttachmentContent,
+  BucketedContent,
+} from "@budibase/types"
 /****************************************************
  *      NOTE: When adding a new bucket - name       *
  *     sure that S3 usages (like budibase-infra)    *
@@ -55,3 +60,50 @@ export const bucketTTLConfig = (
 
   return params
 }
+
+async function processUrlAttachment(
+  attachment: AutomationAttachment
+): Promise<AutomationAttachmentContent> {
+  const response = await fetch(attachment.url)
+  if (!response.ok || !response.body) {
+    throw new Error(`Unexpected response ${response.statusText}`)
+  }
+  const fallbackFilename = path.basename(new URL(attachment.url).pathname)
+  return {
+    filename: attachment.filename || fallbackFilename,
+    content: response.body,
+  }
+}
+
+export async function processObjectStoreAttachment(
+  attachment: AutomationAttachment
+): Promise<BucketedContent> {
+  const result = objectStore.extractBucketAndPath(attachment.url)
+
+  if (result === null) {
+    throw new Error("Invalid signed URL")
+  }
+
+  const { bucket, path: objectPath } = result
+  const readStream = await objectStore.getReadStream(bucket, objectPath)
+  const fallbackFilename = path.basename(objectPath)
+  return {
+    bucket,
+    path: objectPath,
+    filename: attachment.filename || fallbackFilename,
+    content: readStream,
+  }
+}
+
+export async function processAutomationAttachment(
+  attachment: AutomationAttachment
+): Promise<AutomationAttachmentContent | BucketedContent> {
+  const isFullyFormedUrl =
+    attachment.url?.startsWith("http://") ||
+    attachment.url?.startsWith("https://")
+  if (isFullyFormedUrl) {
+    return await processUrlAttachment(attachment)
+  } else {
+    return await processObjectStoreAttachment(attachment)
+  }
+}

packages/backend-core/tests/core/utilities/index.ts

@@ -4,6 +4,3 @@ export { generator } from "./structures"
 export * as testContainerUtils from "./testContainerUtils"
 export * as utils from "./utils"
 export * from "./jestUtils"
-import * as minio from "./minio"
-
-export const objectStoreTestProviders = { minio }

packages/backend-core/tests/core/utilities/minio.ts

@@ -1,34 +0,0 @@
-import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
-import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
-import env from "../../../src/environment"
-
-let container: StartedTestContainer | undefined
-
-class ObjectStoreWaitStrategy extends AbstractWaitStrategy {
-  async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
-    const logs = Wait.forListeningPorts()
-    await logs.waitUntilReady(container, boundPorts, startTime)
-  }
-}
-
-export async function start(): Promise<void> {
-  container = await new GenericContainer("minio/minio")
-    .withExposedPorts(9000)
-    .withCommand(["server", "/data"])
-    .withEnvironment({
-      MINIO_ACCESS_KEY: "budibase",
-      MINIO_SECRET_KEY: "budibase",
-    })
-    .withWaitStrategy(new ObjectStoreWaitStrategy().withStartupTimeout(30000))
-    .start()
-
-  const port = container.getMappedPort(9000)
-  env._set("MINIO_URL", `http://0.0.0.0:${port}`)
-}
-
-export async function stop() {
-  if (container) {
-    await container.stop()
-    container = undefined
-  }
-}

packages/backend-core/tests/core/utilities/testContainerUtils.ts

@@ -86,10 +86,18 @@ export function setupEnv(...envs: any[]) {
     throw new Error("CouchDB SQL port not found")
   }
 
+  const minio = getContainerByImage("minio/minio")
+
+  const minioPort = getExposedV4Port(minio, 9000)
+  if (!minioPort) {
+    throw new Error("Minio port not found")
+  }
+
   const configs = [
     { key: "COUCH_DB_PORT", value: `${couchPort}` },
     { key: "COUCH_DB_URL", value: `http://127.0.0.1:${couchPort}` },
     { key: "COUCH_DB_SQL_URL", value: `http://127.0.0.1:${couchSqlPort}` },
+    { key: "MINIO_URL", value: `http://127.0.0.1:${minioPort}` },
   ]
 
   for (const config of configs.filter(x => !!x.value)) {

packages/bbui/package.json

@@ -83,7 +83,6 @@
     "dayjs": "^1.10.8",
     "easymde": "^2.16.1",
     "svelte-dnd-action": "^0.9.8",
-    "svelte-flatpickr": "3.2.3",
     "svelte-portal": "^1.0.0"
   },
   "resolutions": {

packages/bbui/src/Actions/click_outside.js

@@ -1,23 +1,25 @@
+// These class names will never trigger a callback if clicked, no matter what
 const ignoredClasses = [
   ".download-js-link",
-  ".flatpickr-calendar",
   ".spectrum-Menu",
   ".date-time-popover",
 ]
+
+// These class names will only trigger a callback when clicked if the registered
+// component is not nested inside them. For example, clicking inside a modal
+// will not close the modal, or clicking inside a popover will not close the
+// popover.
 const conditionallyIgnoredClasses = [
   ".spectrum-Underlay",
   ".drawer-wrapper",
   ".spectrum-Popover",
 ]
 let clickHandlers = []
+let candidateTarget
 
-/**
+// Processes a "click outside" event and invokes callbacks if our source element
- * Handle a body click event
+// is valid
- */
 const handleClick = event => {
-  // Treat right clicks (context menu events) as normal clicks
-  const eventType = event.type === "contextmenu" ? "click" : event.type
-
   // Ignore click if this is an ignored class
   if (event.target.closest('[data-ignore-click-outside="true"]')) {
     return
@@ -30,11 +32,6 @@ const handleClick = event => {
 
   // Process handlers
   clickHandlers.forEach(handler => {
-    // Check that we're the right kind of click event
-    if (handler.allowedType && eventType !== handler.allowedType) {
-      return
-    }
-
     // Check that the click isn't inside the target
     if (handler.element.contains(event.target)) {
       return
@@ -52,17 +49,43 @@ const handleClick = event => {
     handler.callback?.(event)
   })
 }
-document.documentElement.addEventListener("click", handleClick, true)
+
-document.documentElement.addEventListener("mousedown", handleClick, true)
+// On mouse up we only trigger a "click outside" callback if we targetted the
-document.documentElement.addEventListener("contextmenu", handleClick, true)
+// same element that we did on mouse down. This fixes all sorts of issues where
+// we get annoying callbacks firing when we drag to select text.
+const handleMouseUp = e => {
+  if (candidateTarget === e.target) {
+    handleClick(e)
+  }
+  candidateTarget = null
+}
+
+// On mouse down we store which element was targetted for comparison later
+const handleMouseDown = e => {
+  // Only handle the primary mouse button here.
+  // We handle context menu (right click) events in another handler.
+  if (e.button !== 0) {
+    return
+  }
+  candidateTarget = e.target
+
+  // Clear any previous listeners in case of multiple down events, and register
+  // a single mouse up listener
+  document.removeEventListener("mouseup", handleMouseUp)
+  document.addEventListener("mouseup", handleMouseUp, true)
+}
+
+// Global singleton listeners for our events
+document.addEventListener("mousedown", handleMouseDown)
+document.addEventListener("contextmenu", handleClick)
 
 /**
  * Adds or updates a click handler
  */
-const updateHandler = (id, element, anchor, callback, allowedType) => {
+const updateHandler = (id, element, anchor, callback) => {
   let existingHandler = clickHandlers.find(x => x.id === id)
   if (!existingHandler) {
-    clickHandlers.push({ id, element, anchor, callback, allowedType })
+    clickHandlers.push({ id, element, anchor, callback })
   } else {
     existingHandler.callback = callback
   }
@@ -89,8 +112,7 @@ export default (element, opts) => {
     const callback =
       newOpts?.callback || (typeof newOpts === "function" ? newOpts : null)
     const anchor = newOpts?.anchor || element
-    const allowedType = newOpts?.allowedType || "click"
+    updateHandler(id, element, anchor, callback)
-    updateHandler(id, element, anchor, callback, allowedType)
   }
   update(opts)
   return {

packages/bbui/src/Banner/Banner.svelte

@@ -8,6 +8,8 @@
   export let size = "S"
   export let extraButtonText
   export let extraButtonAction
+  export let extraLinkText
+  export let extraLinkAction
   export let showCloseButton = true
 
   let show = true
@@ -28,8 +30,13 @@
       <use xlink:href="#spectrum-icon-18-{icon}" />
     </svg>
     <div class="spectrum-Toast-body">
-      <div class="spectrum-Toast-content">
+      <div class="spectrum-Toast-content row-content">
         <slot />
+        {#if extraLinkText}
+          <button class="link" on:click={extraLinkAction}>
+            <u>{extraLinkText}</u>
+          </button>
+        {/if}
       </div>
       {#if extraButtonText && extraButtonAction}
         <button
@@ -73,4 +80,23 @@
   .spectrum-Button {
     border: 1px solid rgba(255, 255, 255, 0.2);
   }
+
+  .row-content {
+    display: flex;
+  }
+
+  .link {
+    background: none;
+    border: none;
+    margin: 0;
+    margin-left: 0.5em;
+    padding: 0;
+    cursor: pointer;
+    color: white;
+    font-weight: 600;
+  }
+
+  u {
+    font-weight: 600;
+  }
 </style>

packages/bbui/src/FancyForm/FancyInput.svelte

@@ -11,6 +11,7 @@
   export let error = null
   export let validate = null
   export let suffix = null
+  export let validateOn = "change"
 
   const dispatch = createEventDispatcher()
 
@@ -24,7 +25,16 @@
     const newValue = e.target.value
     dispatch("change", newValue)
     value = newValue
-    if (validate) {
+    if (validate && (error || validateOn === "change")) {
+      error = validate(newValue)
+    }
+  }
+
+  const onBlur = e => {
+    focused = false
+    const newValue = e.target.value
+    dispatch("blur", newValue)
+    if (validate && validateOn === "blur") {
       error = validate(newValue)
     }
   }
@@ -61,7 +71,7 @@
     type={type || "text"}
     on:input={onChange}
     on:focus={() => (focused = true)}
-    on:blur={() => (focused = false)}
+    on:blur={onBlur}
     class:placeholder
     bind:this={ref}
   />

packages/bbui/src/Form/Core/DatePicker/Calendar.svelte

@@ -4,6 +4,9 @@
   import dayjs from "dayjs"
   import NumberInput from "./NumberInput.svelte"
   import { createEventDispatcher } from "svelte"
+  import isoWeek from "dayjs/plugin/isoWeek"
+
+  dayjs.extend(isoWeek)
 
   export let value
 
@@ -43,7 +46,7 @@
       return []
     }
     let monthEnd = monthStart.endOf("month")
-    let calendarStart = monthStart.startOf("week")
+    let calendarStart = monthStart.startOf("isoWeek")
     const numWeeks = Math.ceil((monthEnd.diff(calendarStart, "day") + 1) / 7)
 
     let mondays = []

packages/bbui/src/Layout/Page.svelte

@@ -7,11 +7,11 @@
   export let narrower = false
   export let noPadding = false
 
-  let sidePanelVisble = false
+  let sidePanelVisible = false
 
   setContext("side-panel", {
-    open: () => (sidePanelVisble = true),
+    open: () => (sidePanelVisible = true),
-    close: () => (sidePanelVisble = false),
+    close: () => (sidePanelVisible = false),
   })
 </script>
 
@@ -24,9 +24,9 @@
   </div>
   <div
     id="side-panel"
-    class:visible={sidePanelVisble}
+    class:visible={sidePanelVisible}
     use:clickOutside={() => {
-      sidePanelVisble = false
+      sidePanelVisible = false
     }}
   >
     <slot name="side-panel" />

packages/bbui/src/helpers.js

@@ -154,7 +154,7 @@ export const parseDate = (value, { enableTime = true }) => {
 // schema flags
 export const stringifyDate = (
   value,
-  { enableTime = true, timeOnly = false, ignoreTimezones = false }
+  { enableTime = true, timeOnly = false, ignoreTimezones = false } = {}
 ) => {
   if (!value) {
     return null
@@ -210,7 +210,7 @@ const localeDateFormat = new Intl.DateTimeFormat()
 // Formats a dayjs date according to schema flags
 export const getDateDisplayValue = (
   value,
-  { enableTime = true, timeOnly = false }
+  { enableTime = true, timeOnly = false } = {}
 ) => {
   if (!value?.isValid()) {
     return ""

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -358,7 +358,8 @@
       value.customType !== "cron" &&
       value.customType !== "triggerSchema" &&
       value.customType !== "automationFields" &&
-      value.type !== "attachment"
+      value.type !== "attachment" &&
+      value.type !== "attachment_single"
     )
   }
 

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -2,6 +2,8 @@
   import { tables } from "stores/builder"
   import { Select, Checkbox, Label } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
+  import { FieldType } from "@budibase/types"
+
   import RowSelectorTypes from "./RowSelectorTypes.svelte"
   import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
@@ -14,7 +16,6 @@
   export let bindings
   export let isTestModal
   export let isUpdateRow
-
   $: parsedBindings = bindings.map(binding => {
     let clone = Object.assign({}, binding)
     clone.icon = "ShareAndroid"
@@ -26,15 +27,19 @@
 
   $: {
     table = $tables.list.find(table => table._id === value?.tableId)
-    schemaFields = Object.entries(table?.schema ?? {})
+
-    // surface the schema so the user can see it in the json
+    // Just sorting attachment types to the bottom here for a cleaner UX
-    schemaFields.map(([, schema]) => {
+    schemaFields = Object.entries(table?.schema ?? {}).sort(
+      ([, schemaA], [, schemaB]) =>
+        (schemaA.type === "attachment") - (schemaB.type === "attachment")
+    )
+
+    schemaFields.forEach(([, schema]) => {
       if (!schema.autocolumn && !value[schema.name]) {
         value[schema.name] = ""
       }
     })
   }
-
   const onChangeTable = e => {
     value["tableId"] = e.detail
     dispatch("change", value)
@@ -114,10 +119,16 @@
 </div>
 {#if schemaFields.length}
   {#each schemaFields as [field, schema]}
-    {#if !schema.autocolumn && schema.type !== "attachment"}
+    {#if !schema.autocolumn}
-      <div class="schema-fields">
+      <div
+        class:schema-fields={schema.type !== FieldType.ATTACHMENTS &&
+          schema.type !== FieldType.ATTACHMENT_SINGLE}
+      >
         <Label>{field}</Label>
-        <div class="field-width">
+        <div
+          class:field-width={schema.type !== FieldType.ATTACHMENTS &&
+            schema.type !== FieldType.ATTACHMENT_SINGLE}
+        >
           {#if isTestModal}
             <RowSelectorTypes
               {isTestModal}

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -1,10 +1,12 @@
 <script>
   import { Select, DatePicker, Multiselect, TextArea } from "@budibase/bbui"
+  import { FieldType } from "@budibase/types"
   import LinkedRowSelector from "components/common/LinkedRowSelector.svelte"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
   import ModalBindableInput from "../../common/bindings/ModalBindableInput.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import Editor from "components/integration/QueryEditor.svelte"
+  import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
 
   export let onChange
   export let field
@@ -22,6 +24,27 @@
   function schemaHasOptions(schema) {
     return !!schema.constraints?.inclusion?.length
   }
+
+  const handleAttachmentParams = keyValuObj => {
+    let params = {}
+
+    if (
+      schema.type === FieldType.ATTACHMENT_SINGLE &&
+      Object.keys(keyValuObj).length === 0
+    ) {
+      return []
+    }
+    if (!Array.isArray(keyValuObj)) {
+      keyValuObj = [keyValuObj]
+    }
+
+    if (keyValuObj.length) {
+      for (let param of keyValuObj) {
+        params[param.url] = param.filename
+      }
+    }
+    return params
+  }
 </script>
 
 {#if schemaHasOptions(schema) && schema.type !== "array"}
@@ -77,6 +100,35 @@
     on:change={e => onChange(e, field)}
     useLabel={false}
   />
+{:else if schema.type === FieldType.ATTACHMENTS || schema.type === FieldType.ATTACHMENT_SINGLE}
+  <div class="attachment-field-spacinng">
+    <KeyValueBuilder
+      on:change={e =>
+        onChange(
+          {
+            detail:
+              schema.type === FieldType.ATTACHMENT_SINGLE
+                ? e.detail.length > 0
+                  ? { url: e.detail[0].name, filename: e.detail[0].value }
+                  : {}
+                : e.detail.map(({ name, value }) => ({
+                    url: name,
+                    filename: value,
+                  })),
+          },
+          field
+        )}
+      object={handleAttachmentParams(value[field])}
+      allowJS
+      {bindings}
+      keyBindings
+      customButtonText={"Add attachment"}
+      keyPlaceholder={"URL"}
+      valuePlaceholder={"Filename"}
+      actionButtonDisabled={schema.type === FieldType.ATTACHMENT_SINGLE &&
+        Object.keys(value[field]).length >= 1}
+    />
+  </div>
 {:else if ["string", "number", "bigint", "barcodeqr", "array"].includes(schema.type)}
   <svelte:component
     this={isTestModal ? ModalBindableInput : DrawerBindableInput}
@@ -90,3 +142,10 @@
     title={schema.name}
   />
 {/if}
+
+<style>
+  .attachment-field-spacinng {
+    margin-top: var(--spacing-s);
+    margin-bottom: var(--spacing-l);
+  }
+</style>

packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte

@@ -1,7 +1,9 @@
 <script>
   import { createEventDispatcher } from "svelte"
-  import { ActionButton, Modal, ModalContent } from "@budibase/bbui"
+  import { ActionButton, Drawer, DrawerContent, Button } from "@budibase/bbui"
   import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte"
+  import { getUserBindings } from "dataBinding"
+  import { makePropSafe } from "@budibase/string-templates"
 
   export let schema
   export let filters
@@ -10,7 +12,7 @@
 
   const dispatch = createEventDispatcher()
 
-  let modal
+  let drawer
 
   $: tempValue = filters || []
   $: schemaFields = Object.entries(schema || {}).map(
@@ -22,37 +24,53 @@
 
   $: text = getText(filters)
   $: selected = tempValue.filter(x => !x.onEmptyFilter)?.length > 0
-
+  $: bindings = [
+    {
+      type: "context",
+      runtimeBinding: `${makePropSafe("now")}`,
+      readableBinding: `Date`,
+      category: "Date",
+      icon: "Date",
+      display: {
+        name: "Server date",
+      },
+    },
+    ...getUserBindings(),
+  ]
   const getText = filters => {
     const count = filters?.filter(filter => filter.field)?.length
     return count ? `Filter (${count})` : "Filter"
   }
 </script>
 
-<ActionButton icon="Filter" quiet {disabled} on:click={modal.show} {selected}>
+<ActionButton icon="Filter" quiet {disabled} on:click={drawer.show} {selected}>
   {text}
 </ActionButton>
-<Modal bind:this={modal}>
-  <ModalContent
-    title="Filter"
-    confirmText="Save"
-    size="XL"
-    onConfirm={() => dispatch("change", tempValue)}
-  >
-    <div class="wrapper">
-      <FilterBuilder
-        allowBindings={false}
-        {filters}
-        {schemaFields}
-        datasource={{ type: "table", tableId }}
-        on:change={e => (tempValue = e.detail)}
-      />
-    </div>
-  </ModalContent>
-</Modal>
 
-<style>
+<Drawer
-  .wrapper :global(.main) {
+  bind:this={drawer}
-    padding: 0;
+  title="Filtering"
-  }
+  on:drawerHide
-</style>
+  on:drawerShow
+  forceModal
+>
+  <Button
+    cta
+    slot="buttons"
+    on:click={() => {
+      dispatch("change", tempValue)
+      drawer.hide()
+    }}
+  >
+    Save
+  </Button>
+  <DrawerContent slot="body">
+    <FilterBuilder
+      {filters}
+      {schemaFields}
+      datasource={{ type: "table", tableId }}
+      on:change={e => (tempValue = e.detail)}
+      {bindings}
+    />
+  </DrawerContent>
+</Drawer>

packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte

@@ -4,6 +4,7 @@
     readableToRuntimeBinding,
     runtimeToReadableBinding,
   } from "dataBinding"
+  import { FieldType } from "@budibase/types"
 
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
   import { createEventDispatcher, setContext } from "svelte"
@@ -102,6 +103,8 @@
     longform: value => !isJSBinding(value),
     json: value => !isJSBinding(value),
     boolean: isValidBoolean,
+    attachment: false,
+    attachment_single: false,
   }
 
   const isValid = value => {
@@ -116,7 +119,16 @@
     if (type === "json" && !isJSBinding(value)) {
       return "json-slot-icon"
     }
-    if (!["string", "number", "bigint", "barcodeqr"].includes(type)) {
+    if (
+      ![
+        "string",
+        "number",
+        "bigint",
+        "barcodeqr",
+        "attachment",
+        "attachment_single",
+      ].includes(type)
+    ) {
       return "slot-icon"
     }
     return ""
@@ -157,7 +169,7 @@
       {updateOnChange}
     />
   {/if}
-  {#if !disabled && type !== "formula"}
+  {#if !disabled && type !== "formula" && !disabled && type !== FieldType.ATTACHMENTS && !disabled && type !== FieldType.ATTACHMENT_SINGLE}
     <div
       class={`icon ${getIconClass(value, type)}`}
       on:click={() => {

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/PromptUser.svelte

@@ -1,8 +1,10 @@
 <script>
-  import { Body, Label, Input } from "@budibase/bbui"
+  import { Body, Label } from "@budibase/bbui"
   import { onMount } from "svelte"
+  import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
 
   export let parameters
+  export let bindings
 
   onMount(() => {
     if (!parameters.confirm) {
@@ -15,11 +17,18 @@
   <Body size="S">Enter the message you wish to display to the user.</Body>
   <div class="params">
     <Label small>Title</Label>
-    <Input placeholder="Prompt User" bind:value={parameters.customTitleText} />
+    <DrawerBindableInput
+      placeholder="Title"
+      value={parameters.customTitleText}
+      on:change={e => (parameters.customTitleText = e.detail)}
+      {bindings}
+    />
     <Label small>Message</Label>
-    <Input
+    <DrawerBindableInput
       placeholder="Are you sure you want to continue?"
-      bind:value={parameters.confirmText}
+      value={parameters.confirmText}
+      on:change={e => (parameters.confirmText = e.detail)}
+      {bindings}
     />
   </div>
 </div>

packages/builder/src/components/design/settings/controls/FormStepConfiguration.svelte

@@ -21,26 +21,24 @@
   const currentStep = derived(multiStepStore, state => state.currentStep)
   const componentType = "@budibase/standard-components/multistepformblockstep"
 
+  setContext("multi-step-form-block", multiStepStore)
+
   let cachedValue
   let cachedInstance = {}
 
   $: if (!isEqual(cachedValue, value)) {
     cachedValue = value
   }
-
   $: if (!isEqual(componentInstance, cachedInstance)) {
     cachedInstance = componentInstance
   }
-
-  setContext("multi-step-form-block", multiStepStore)
-
   $: stepCount = cachedValue?.length || 0
   $: updateStore(stepCount)
   $: dataSource = getDatasourceForProvider($selectedScreen, cachedInstance)
   $: emitCurrentStep($currentStep)
   $: stepLabel = getStepLabel($multiStepStore)
   $: stepDef = getDefinition(stepLabel)
-  $: stepSettings = cachedValue?.[$currentStep] || {}
+  $: savedInstance = cachedValue?.[$currentStep] || {}
   $: defaults = Utils.buildMultiStepFormBlockDefaultProps({
     _id: cachedInstance._id,
     stepCount: $multiStepStore.stepCount,
@@ -48,14 +46,16 @@
     actionType: cachedInstance.actionType,
     dataSource: cachedInstance.dataSource,
   })
+  // For backwards compatibility we need to sometimes manually set base
+  // properties like _id and _component as we didn't used to save these
   $: stepInstance = {
-    _id: Helpers.uuid(),
+    _id: savedInstance._id || Helpers.uuid(),
-    _component: componentType,
+    _component: savedInstance._component || componentType,
     _instanceName: `Step ${currentStep + 1}`,
-    title: stepSettings.title ?? defaults?.title,
+    title: savedInstance.title ?? defaults?.title,
-    buttons: stepSettings.buttons || defaults?.buttons,
+    buttons: savedInstance.buttons || defaults?.buttons,
-    fields: stepSettings.fields,
+    fields: savedInstance.fields,
-    desc: stepSettings.desc,
+    desc: savedInstance.desc,
 
     // Needed for field configuration
     dataSource,
@@ -92,7 +92,8 @@
   }
 
   const addStep = () => {
-    value = value.toSpliced($currentStep + 1, 0, {})
+    const newInstance = componentStore.createInstance(componentType)
+    value = value.toSpliced($currentStep + 1, 0, newInstance)
     dispatch("change", value)
     multiStepStore.update(state => ({
       ...state,

packages/builder/src/components/design/settings/controls/FormStepControls.svelte

@@ -1,6 +1,6 @@
 <script>
   import { createEventDispatcher, getContext } from "svelte"
-  import { ActionButton } from "@budibase/bbui"
+  import { ActionButton, AbsTooltip } from "@budibase/bbui"
 
   const multiStepStore = getContext("multi-step-form-block")
   const dispatch = createEventDispatcher()
@@ -28,45 +28,49 @@
   </div>
 {:else}
   <div class="step-actions">
-    <ActionButton
+    <AbsTooltip text="Previous step" noWrap>
-      size="S"
+      <ActionButton
-      secondary
+        size="S"
-      icon="ChevronLeft"
+        secondary
-      disabled={currentStep === 0}
+        icon="ChevronLeft"
-      on:click={() => {
+        disabled={currentStep === 0}
-        stepAction("previousStep")
+        on:click={() => {
-      }}
+          stepAction("previousStep")
-      tooltip={"Previous step"}
+        }}
-    />
+      />
-    <ActionButton
+    </AbsTooltip>
-      size="S"
+    <AbsTooltip text="Next step" noWrap>
-      secondary
+      <ActionButton
-      disabled={currentStep === stepCount - 1}
+        size="S"
-      icon="ChevronRight"
+        secondary
-      on:click={() => {
+        disabled={currentStep === stepCount - 1}
-        stepAction("nextStep")
+        icon="ChevronRight"
-      }}
+        on:click={() => {
-      tooltip={"Next step"}
+          stepAction("nextStep")
-    />
+        }}
-    <ActionButton
+      />
-      size="S"
+    </AbsTooltip>
-      secondary
+    <AbsTooltip text="Remove step" noWrap>
-      icon="Close"
+      <ActionButton
-      disabled={stepCount === 1}
+        size="S"
-      on:click={() => {
+        secondary
-        stepAction("removeStep")
+        icon="Close"
-      }}
+        disabled={stepCount === 1}
-      tooltip={"Remove step"}
+        on:click={() => {
-    />
+          stepAction("removeStep")
-    <ActionButton
+        }}
-      size="S"
+      />
-      secondary
+    </AbsTooltip>
-      icon="MultipleAdd"
+    <AbsTooltip text="Add step" noWrap>
-      on:click={() => {
+      <ActionButton
-        stepAction("addStep")
+        size="S"
-      }}
+        secondary
-      tooltip={"Add step"}
+        icon="MultipleAdd"
-    />
+        on:click={() => {
+          stepAction("addStep")
+        }}
+      />
+    </AbsTooltip>
   </div>
 {/if}
 

packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js

@@ -75,6 +75,7 @@ const toDraggableListFormat = (gridFormatColumns, createComponent, schema) => {
     return createComponent(
       "@budibase/standard-components/labelfield",
       {
+        _id: column.field,
         _instanceName: column.field,
         active: column.active,
         field: column.field,

packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.test.js

@@ -65,6 +65,7 @@ describe("getColumns", () => {
     it("returns the selected and unselected fields in the modern format, respecting the original order", ctx => {
       expect(ctx.columns.sortable).toEqual([
         {
+          _id: "three",
           _instanceName: "three",
           active: true,
           columnType: "foo",
@@ -73,6 +74,7 @@ describe("getColumns", () => {
           label: "three label",
         },
         {
+          _id: "two",
           _instanceName: "two",
           active: true,
           columnType: "foo",
@@ -81,6 +83,7 @@ describe("getColumns", () => {
           label: "two label",
         },
         {
+          _id: "one",
           _instanceName: "one",
           active: false,
           columnType: "foo",
@@ -91,6 +94,7 @@ describe("getColumns", () => {
       ])
 
       expect(ctx.columns.primary).toEqual({
+        _id: "four",
         _instanceName: "four",
         active: true,
         columnType: "foo",
@@ -115,6 +119,7 @@ describe("getColumns", () => {
     it("returns all columns, with non-hidden columns automatically selected", ctx => {
       expect(ctx.columns.sortable).toEqual([
         {
+          _id: "two",
           _instanceName: "two",
           active: true,
           columnType: "foo",
@@ -123,6 +128,7 @@ describe("getColumns", () => {
           label: "two",
         },
         {
+          _id: "three",
           _instanceName: "three",
           active: true,
           columnType: "foo",
@@ -131,6 +137,7 @@ describe("getColumns", () => {
           label: "three",
         },
         {
+          _id: "one",
           _instanceName: "one",
           active: false,
           columnType: "foo",
@@ -141,6 +148,7 @@ describe("getColumns", () => {
       ])
 
       expect(ctx.columns.primary).toEqual({
+        _id: "four",
         _instanceName: "four",
         active: true,
         columnType: "foo",
@@ -173,6 +181,7 @@ describe("getColumns", () => {
     it("returns all columns, including those missing from the initial data", ctx => {
       expect(ctx.columns.sortable).toEqual([
         {
+          _id: "three",
           _instanceName: "three",
           active: true,
           columnType: "foo",
@@ -181,6 +190,7 @@ describe("getColumns", () => {
           label: "three label",
         },
         {
+          _id: "two",
           _instanceName: "two",
           active: false,
           columnType: "foo",
@@ -189,6 +199,7 @@ describe("getColumns", () => {
           label: "two",
         },
         {
+          _id: "one",
           _instanceName: "one",
           active: false,
           columnType: "foo",
@@ -199,6 +210,7 @@ describe("getColumns", () => {
       ])
 
       expect(ctx.columns.primary).toEqual({
+        _id: "four",
         _instanceName: "four",
         active: true,
         columnType: "foo",
@@ -228,6 +240,7 @@ describe("getColumns", () => {
     it("returns all valid columns, excluding those that aren't valid for the schema", ctx => {
       expect(ctx.columns.sortable).toEqual([
         {
+          _id: "three",
           _instanceName: "three",
           active: true,
           columnType: "foo",
@@ -236,6 +249,7 @@ describe("getColumns", () => {
           label: "three label",
         },
         {
+          _id: "two",
           _instanceName: "two",
           active: false,
           columnType: "foo",
@@ -244,6 +258,7 @@ describe("getColumns", () => {
           label: "two",
         },
         {
+          _id: "one",
           _instanceName: "one",
           active: false,
           columnType: "foo",
@@ -254,6 +269,7 @@ describe("getColumns", () => {
       ])
 
       expect(ctx.columns.primary).toEqual({
+        _id: "four",
         _instanceName: "four",
         active: true,
         columnType: "foo",
@@ -318,6 +334,7 @@ describe("getColumns", () => {
       beforeEach(ctx => {
         ctx.updateSortable([
           {
+            _id: "three",
             _instanceName: "three",
             active: true,
             columnType: "foo",
@@ -326,6 +343,7 @@ describe("getColumns", () => {
             label: "three",
           },
           {
+            _id: "one",
             _instanceName: "one",
             active: true,
             columnType: "foo",
@@ -334,6 +352,7 @@ describe("getColumns", () => {
             label: "one",
           },
           {
+            _id: "two",
             _instanceName: "two",
             active: false,
             columnType: "foo",

packages/builder/src/components/integration/KeyValueBuilder.svelte

@@ -37,6 +37,7 @@
   export let customButtonText = null
   export let keyBindings = false
   export let allowJS = false
+  export let actionButtonDisabled = false
   export let compare = (option, value) => option === value
 
   let fields = Object.entries(object || {}).map(([name, value]) => ({
@@ -189,7 +190,14 @@
 {/if}
 {#if !readOnly && !noAddButton}
   <div>
-    <ActionButton icon="Add" secondary thin outline on:click={addEntry}>
+    <ActionButton
+      disabled={actionButtonDisabled}
+      icon="Add"
+      secondary
+      thin
+      outline
+      on:click={addEntry}
+    >
       {#if customButtonText}
         {customButtonText}
       {:else}

packages/builder/src/components/integration/QueryViewerSidePanel/SchemaPanel.svelte

@@ -25,6 +25,6 @@
     name="field"
     headings
     options={SchemaTypeOptionsExpanded}
-    compare={(option, value) => option.type === value.type}
+    compare={(option, value) => option.type === value?.type}
   />
 {/key}

packages/builder/src/components/integration/RestQueryViewer.svelte

@@ -695,7 +695,7 @@
                   menuItems={schemaMenuItems}
                   showMenu={!schemaReadOnly}
                   readOnly={schemaReadOnly}
-                  compare={(option, value) => option.type === value.type}
+                  compare={(option, value) => option.type === value?.type}
                 />
               </Tab>
             {/if}

packages/builder/src/components/portal/licensing/EnterpriseBasicTrialBanner.svelte

@@ -0,0 +1,43 @@
+<script>
+  import "@spectrum-css/toast/dist/index-vars.css"
+  import Portal from "svelte-portal"
+  import { fly } from "svelte/transition"
+  import { Banner, BANNER_TYPES } from "@budibase/bbui"
+  import { licensing } from "stores/portal"
+
+  export let show = true
+
+  const oneDayInSeconds = 86400
+
+  $: license = $licensing.license
+
+  function daysUntilCancel() {
+    const cancelAt = license?.billing?.subscription?.cancelAt
+    const diffTime = Math.abs(cancelAt - new Date().getTime()) / 1000
+    return Math.floor(diffTime / oneDayInSeconds)
+  }
+</script>
+
+<Portal target=".banner-container">
+  <div class="banner">
+    {#if show}
+      <div transition:fly={{ y: -30 }}>
+        <Banner
+          type={BANNER_TYPES.INFO}
+          extraLinkText={"Please select a plan."}
+          extraLinkAction={$licensing.goToUpgradePage}
+          showCloseButton={false}
+        >
+          Your free trial will end in {daysUntilCancel()} days.
+        </Banner>
+      </div>
+    {/if}
+  </div>
+</Portal>
+
+<style>
+  .banner {
+    pointer-events: none;
+    width: 100%;
+  }
+</style>

packages/builder/src/components/portal/licensing/licensingBanners.js

@@ -12,7 +12,7 @@ const defaultCacheFn = key => {
 const upgradeAction = key => {
   return defaultNavigateAction(
     key,
-    "Upgrade Plan",
+    "Upgrade",
     `${get(admin).accountPortalUrl}/portal/upgrade`
   )
 }

packages/builder/src/components/portal/onboarding/EnterpriseBasicTrialModal.svelte

@@ -0,0 +1,66 @@
+<script>
+  import { Modal, ModalContent } from "@budibase/bbui"
+  import FreeTrial from "../../../../assets/FreeTrial.svelte"
+  import { get } from "svelte/store"
+  import { auth, licensing } from "stores/portal"
+  import { API } from "api"
+  import { PlanType } from "@budibase/types"
+  import { sdk } from "@budibase/shared-core"
+
+  let freeTrialModal
+
+  $: planType = $licensing?.license?.plan?.type
+  $: showFreeTrialModal(planType, freeTrialModal)
+
+  const showFreeTrialModal = (planType, freeTrialModal) => {
+    if (
+      planType === PlanType.ENTERPRISE_BASIC_TRIAL &&
+      !$auth.user?.freeTrialConfirmedAt &&
+      sdk.users.isAdmin($auth.user)
+    ) {
+      freeTrialModal?.show()
+    }
+  }
+</script>
+
+<Modal bind:this={freeTrialModal} disableCancel={true}>
+  <ModalContent
+    confirmText="Get started"
+    size="M"
+    showCancelButton={false}
+    showCloseIcon={false}
+    onConfirm={async () => {
+      if (get(auth).user) {
+        try {
+          await API.updateSelf({
+            freeTrialConfirmedAt: new Date().toISOString(),
+          })
+          // Update the cached user
+          await auth.getSelf()
+        } finally {
+          freeTrialModal.hide()
+        }
+      }
+    }}
+  >
+    <h1>Experience all of Budibase with a free 14-day trial</h1>
+    <div class="free-trial-text">
+      We've upgraded you to a free 14-day trial that allows you to try all our
+      features before deciding which plan is right for you.
+      <p>
+        At the end of your trial, we'll automatically downgrade you to the Free
+        plan unless you choose to upgrade.
+      </p>
+    </div>
+    <FreeTrial />
+  </ModalContent>
+</Modal>
+
+<style>
+  h1 {
+    font-size: 26px;
+  }
+  .free-trial-text {
+    font-size: 16px;
+  }
+</style>

packages/builder/src/constants/backend/index.js

@@ -253,6 +253,7 @@ export const SchemaTypeOptions = [
   { label: "Number", value: FieldType.NUMBER },
   { label: "Boolean", value: FieldType.BOOLEAN },
   { label: "Datetime", value: FieldType.DATETIME },
+  { label: "JSON", value: FieldType.JSON },
 ]
 
 export const SchemaTypeOptionsExpanded = SchemaTypeOptions.map(el => ({

packages/builder/src/dataBinding.js

@@ -1106,50 +1106,51 @@ export const getAllStateVariables = () => {
   getAllAssets().forEach(asset => {
     findAllMatchingComponents(asset.props, component => {
       const settings = componentStore.getComponentSettings(component._component)
+      const nestedTypes = [
+        "buttonConfiguration",
+        "fieldConfiguration",
+        "stepConfiguration",
+      ]
 
+      // Extracts all event settings from a component instance.
+      // Recurses into nested types to find all event-like settings at any
+      // depth.
       const parseEventSettings = (settings, comp) => {
+        if (!settings?.length) {
+          return
+        }
+
+        // Extract top level event settings
         settings
           .filter(setting => setting.type === "event")
           .forEach(setting => {
             eventSettings.push(comp[setting.key])
           })
-      }
 
-      const parseComponentSettings = (settings, component) => {
+        // Recurse into any nested instance types
-        // Parse the nested button configurations
         settings
-          .filter(setting => setting.type === "buttonConfiguration")
+          .filter(setting => nestedTypes.includes(setting.type))
           .forEach(setting => {
-            const buttonConfig = component[setting.key]
+            const instances = comp[setting.key]
+            if (Array.isArray(instances) && instances.length) {
+              instances.forEach(instance => {
+                let type = instance?._component
 
-            if (Array.isArray(buttonConfig)) {
+                // Backwards compatibility for multi-step from blocks which
-              buttonConfig.forEach(button => {
+                // didn't set a proper component type previously.
-                const nestedSettings = componentStore.getComponentSettings(
+                if (setting.type === "stepConfiguration" && !type) {
-                  button._component
+                  type = "@budibase/standard-components/multistepformblockstep"
-                )
+                }
-                parseEventSettings(nestedSettings, button)
+
+                // Parsed nested component instances inside this setting
+                const nestedSettings = componentStore.getComponentSettings(type)
+                parseEventSettings(nestedSettings, instance)
               })
             }
           })
-
-        parseEventSettings(settings, component)
       }
 
-      // Parse the base component settings
+      parseEventSettings(settings, component)
-      parseComponentSettings(settings, component)
-
-      // Parse step configuration
-      const stepSetting = settings.find(
-        setting => setting.type === "stepConfiguration"
-      )
-      const steps = stepSetting ? component[stepSetting.key] : []
-      const stepDefinition = componentStore.getComponentSettings(
-        "@budibase/standard-components/multistepformblockstep"
-      )
-
-      steps?.forEach(step => {
-        parseComponentSettings(stepDefinition, step)
-      })
     })
   })
 

packages/builder/src/helpers/planTitle.js

@@ -20,6 +20,9 @@ export function getFormattedPlanName(userPlanType) {
     case PlanType.ENTERPRISE:
       planName = "Enterprise"
       break
+    case PlanType.ENTERPRISE_BASIC_TRIAL:
+      planName = "Trial"
+      break
     default:
       planName = "Free" // Default to "Free" if the type is not explicitly handled
   }

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -32,6 +32,7 @@
   import { UserAvatars } from "@budibase/frontend-core"
   import { TOUR_KEYS } from "components/portal/onboarding/tours.js"
   import PreviewOverlay from "./_components/PreviewOverlay.svelte"
+  import EnterpriseBasicTrialModal from "components/portal/onboarding/EnterpriseBasicTrialModal.svelte"
 
   export let application
 
@@ -192,6 +193,8 @@
   <CommandPalette />
 </Modal>
 
+<EnterpriseBasicTrialModal />
+
 <style>
   .back-to-apps {
     display: contents;

packages/builder/src/pages/builder/app/[application]/settings/backups/index.svelte

@@ -98,14 +98,22 @@
     })
   }
 
-  async function fetchBackups(filters, page, dateRange) {
+  async function fetchBackups(filters, page, dateRange = []) {
-    const response = await backups.searchBackups({
+    const body = {
       appId: $appStore.appId,
       ...filters,
       page,
-      startDate: dateRange[0],
+    }
-      endDate: dateRange[1],
+
-    })
+    const [startDate, endDate] = dateRange
+    if (startDate) {
+      body.startDate = startDate
+    }
+    if (endDate) {
+      body.endDate = endDate
+    }
+
+    const response = await backups.searchBackups(body)
     pageInfo.fetched(response.hasNextPage, response.nextPage)
 
     // flatten so we have an easier structure to use for the table schema
@@ -120,7 +128,7 @@
       })
       await fetchBackups(filterOpt, page)
       notifications.success(response.message)
-    } catch {
+    } catch (err) {
       notifications.error("Unable to create backup")
     }
   }

packages/builder/src/pages/builder/portal/_components/UpgradeButton.svelte

@@ -6,7 +6,7 @@
   import { sdk } from "@budibase/shared-core"
 </script>
 
-{#if isEnabled(TENANT_FEATURE_FLAGS.LICENSING) && !$licensing.isEnterprisePlan}
+{#if isEnabled(TENANT_FEATURE_FLAGS.LICENSING) && !$licensing.isEnterprisePlan && !$licensing.isEnterpriseTrial}
   {#if $admin.cloud && $auth?.user?.accountPortalAccess}
     <Button
       cta

packages/builder/src/pages/builder/portal/_layout.svelte

@@ -1,7 +1,7 @@
 <script>
   import { isActive, redirect, goto, url } from "@roxi/routify"
   import { Icon, notifications, Tabs, Tab } from "@budibase/bbui"
-  import { organisation, auth, menu, appsStore } from "stores/portal"
+  import { organisation, auth, menu, appsStore, licensing } from "stores/portal"
   import { onMount } from "svelte"
   import UpgradeButton from "./_components/UpgradeButton.svelte"
   import MobileMenu from "./_components/MobileMenu.svelte"
@@ -10,6 +10,8 @@
   import HelpMenu from "components/common/HelpMenu.svelte"
   import VerificationPromptBanner from "components/common/VerificationPromptBanner.svelte"
   import { sdk } from "@budibase/shared-core"
+  import EnterpriseBasicTrialBanner from "components/portal/licensing/EnterpriseBasicTrialBanner.svelte"
+  import { Constants } from "@budibase/frontend-core"
 
   let loaded = false
   let mobileMenuVisible = false
@@ -33,6 +35,14 @@
   const showMobileMenu = () => (mobileMenuVisible = true)
   const hideMobileMenu = () => (mobileMenuVisible = false)
 
+  const showFreeTrialBanner = () => {
+    return (
+      $licensing.license?.plan?.type ===
+        Constants.PlanType.ENTERPRISE_BASIC_TRIAL &&
+      sdk.users.isAdmin($auth.user)
+    )
+  }
+
   onMount(async () => {
     // Prevent non-builders from accessing the portal
     if ($auth.user) {
@@ -58,6 +68,7 @@
     <HelpMenu />
     <div class="container">
       <VerificationPromptBanner />
+      <EnterpriseBasicTrialBanner show={showFreeTrialBanner()} />
       <div class="nav">
         <div class="branding">
           <Logo />

packages/builder/src/pages/builder/portal/account/usage.svelte

@@ -29,6 +29,7 @@
   const manageUrl = `${$admin.accountPortalUrl}/portal/billing`
 
   const WARN_USAGE = ["Queries", "Automations", "Rows", "Day Passes", "Users"]
+  const oneDayInSeconds = 86400
 
   const EXCLUDE_QUOTAS = {
     Queries: () => true,
@@ -104,24 +105,17 @@
     if (!timestamp) {
       return
     }
-    const now = new Date()
+    const diffTime = Math.abs(timestamp - new Date().getTime()) / 1000
-    now.setHours(0)
+    return Math.floor(diffTime / oneDayInSeconds)
-    now.setMinutes(0)
-
-    const thenDate = new Date(timestamp)
-    thenDate.setHours(0)
-    thenDate.setMinutes(0)
-
-    const difference = thenDate.getTime() - now
-    // return the difference in days
-    return (difference / (1000 * 3600 * 24)).toFixed(0)
   }
 
   const setTextRows = () => {
     textRows = []
 
     if (cancelAt && !usesInvoicing) {
-      textRows.push({ message: "Subscription has been cancelled" })
+      if (plan?.type !== Constants.PlanType.ENTERPRISE_BASIC_TRIAL) {
+        textRows.push({ message: "Subscription has been cancelled" })
+      }
       textRows.push({
         message: `${getDaysRemaining(cancelAt)} days remaining`,
         tooltip: new Date(cancelAt),

packages/builder/src/stores/portal/licensing.js

@@ -103,6 +103,8 @@ export const createLicensingStore = () => {
       const isEnterprisePlan = planType === Constants.PlanType.ENTERPRISE
       const isFreePlan = planType === Constants.PlanType.FREE
       const isBusinessPlan = planType === Constants.PlanType.BUSINESS
+      const isEnterpriseTrial =
+        planType === Constants.PlanType.ENTERPRISE_BASIC_TRIAL
       const groupsEnabled = license.features.includes(
         Constants.Features.USER_GROUPS
       )
@@ -143,6 +145,7 @@ export const createLicensingStore = () => {
           isEnterprisePlan,
           isFreePlan,
           isBusinessPlan,
+          isEnterpriseTrial,
           groupsEnabled,
           backupsEnabled,
           brandingEnabled,

packages/client/manifest.json

@@ -4220,8 +4220,8 @@
     ]
   },
   "attachmentfield": {
-    "name": "Attachment list",
+    "name": "Attachment List",
-    "icon": "Attach",
+    "icon": "DocumentFragmentGroup",
     "styles": ["size"],
     "requiredAncestors": ["form"],
     "editable": true,
@@ -4318,7 +4318,7 @@
   },
   "attachmentsinglefield": {
     "name": "Single Attachment",
-    "icon": "Attach",
+    "icon": "DocumentFragment",
     "styles": ["size"],
     "requiredAncestors": ["form"],
     "editable": true,
@@ -6718,7 +6718,20 @@
     "illegalChildren": ["section", "sidepanel"],
     "showEmptyState": false,
     "draggable": false,
-    "info": "Side panels are hidden by default. They will only be revealed when triggered by the 'Open Side Panel' action."
+    "info": "Side panels are hidden by default. They will only be revealed when triggered by the 'Open Side Panel' action.",
+    "settings": [
+      {
+        "type": "boolean",
+        "key": "ignoreClicksOutside",
+        "label": "Ignore clicks outside",
+        "defaultValue": false
+      },
+      {
+        "type": "event",
+        "key": "onClose",
+        "label": "On close"
+      }
+    ]
   },
   "rowexplorer": {
     "block": true,

packages/client/package.json

@@ -34,7 +34,6 @@
     "screenfull": "^6.0.1",
     "shortid": "^2.2.15",
     "svelte-apexcharts": "^1.0.2",
-    "svelte-flatpickr": "^3.3.4",
     "svelte-spa-router": "^4.0.1"
   },
   "devDependencies": {

packages/client/src/components/ClientApp.svelte

@@ -206,13 +206,6 @@
                               />
                             {/key}
 
-                            <!--
-                        Flatpickr needs to be inside the theme wrapper.
-                        It also needs its own container because otherwise it hijacks
-                        key events on the whole page. It is painful to work with.
-                      -->
-                            <div id="flatpickr-root" />
-
                             <!-- Modal container to ensure they sit on top -->
                             <div class="modal-container" />
 

packages/client/src/components/CustomThemeWrapper.svelte

@@ -60,16 +60,6 @@
     --spectrum-link-primary-m-text-color-hover: var(--primaryColorHover);
   }
 
-  /* Theme flatpickr */
-  :global(.flatpickr-day.selected) {
-    background: var(--primaryColor);
-    border-color: var(--primaryColor);
-  }
-  :global(.flatpickr-day.selected:hover) {
-    background: var(--primaryColorHover);
-    border-color: var(--primaryColorHover);
-  }
-
   /* Custom scrollbars */
   :global(::-webkit-scrollbar) {
     width: 8px;

packages/client/src/components/app/Layout.svelte

@@ -73,7 +73,10 @@
     $context.device.width,
     $context.device.height
   )
-  $: autoCloseSidePanel = !$builderStore.inBuilder && $sidePanelStore.open
+  $: autoCloseSidePanel =
+    !$builderStore.inBuilder &&
+    $sidePanelStore.open &&
+    !$sidePanelStore.ignoreClicksOutside
   $: screenId = $builderStore.inBuilder
     ? `${$builderStore.screen?._id}-screen`
     : "screen"
@@ -191,6 +194,11 @@
     }
     return url
   }
+
+  const handleClickLink = () => {
+    mobileOpen = false
+    sidePanelStore.actions.close()
+  }
 </script>
 
 <!-- svelte-ignore a11y-no-static-element-interactions -->
@@ -281,7 +289,7 @@
                     url={navItem.url}
                     subLinks={navItem.subLinks}
                     internalLink={navItem.internalLink}
-                    on:clickLink={() => (mobileOpen = false)}
+                    on:clickLink={handleClickLink}
                     leftNav={navigation === "Left"}
                     {mobile}
                     {navStateStore}
@@ -316,10 +324,7 @@
   <div
     id="side-panel-container"
     class:open={$sidePanelStore.open}
-    use:clickOutside={{
+    use:clickOutside={autoCloseSidePanel ? sidePanelStore.actions.close : null}
-      callback: autoCloseSidePanel ? sidePanelStore.actions.close : null,
-      allowedType: "mousedown",
-    }}
     class:builder={$builderStore.inBuilder}
   >
     <div class="side-panel-header">

packages/client/src/components/app/SidePanel.svelte

@@ -5,6 +5,9 @@
   const { styleable, sidePanelStore, builderStore, dndIsDragging } =
     getContext("sdk")
 
+  export let onClose
+  export let ignoreClicksOutside
+
   // Automatically show and hide the side panel when inside the builder.
   // For some unknown reason, svelte reactivity breaks if we reference the
   // reactive variable "open" inside the following expression, or if we define
@@ -26,6 +29,10 @@
     }
   }
 
+  // $: {
+
+  // }
+
   // Derive visibility
   $: open = $sidePanelStore.contentId === $component.id
 
@@ -36,10 +43,17 @@
   let renderKey = null
   $: {
     if (open) {
+      sidePanelStore.actions.setIgnoreClicksOutside(ignoreClicksOutside)
       renderKey = Math.random()
     }
   }
 
+  const handleSidePanelClose = async () => {
+    if (onClose) {
+      await onClose()
+    }
+  }
+
   const showInSidePanel = (el, visible) => {
     const update = visible => {
       const target = document.getElementById("side-panel-container")
@@ -51,6 +65,7 @@
       } else {
         if (target.contains(node)) {
           target.removeChild(node)
+          handleSidePanelClose()
         }
       }
     }

packages/client/src/components/app/forms/DateTimeField.svelte

@@ -49,7 +49,6 @@
       readonly={fieldState.readonly}
       error={fieldState.error}
       id={fieldState.fieldId}
-      appendTo={document.getElementById("flatpickr-root")}
       {enableTime}
       {timeOnly}
       {time24hr}

packages/client/src/components/app/forms/validation.js

@@ -1,5 +1,6 @@
-import flatpickr from "flatpickr"
+import dayjs from "dayjs"
 import { FieldTypes } from "../../../constants"
+import { Helpers } from "@budibase/bbui"
 
 /**
  * Creates a validation function from a combination of schema-level constraints
@@ -81,7 +82,7 @@ export const createValidatorFromConstraints = (
     // Date constraint
     if (exists(schemaConstraints.datetime?.earliest)) {
       const limit = schemaConstraints.datetime.earliest
-      const limitString = flatpickr.formatDate(new Date(limit), "F j Y, H:i")
+      const limitString = Helpers.getDateDisplayValue(dayjs(limit))
       rules.push({
         type: "datetime",
         constraint: "minValue",
@@ -91,7 +92,7 @@ export const createValidatorFromConstraints = (
     }
     if (exists(schemaConstraints.datetime?.latest)) {
       const limit = schemaConstraints.datetime.latest
-      const limitString = flatpickr.formatDate(new Date(limit), "F j Y, H:i")
+      const limitString = Helpers.getDateDisplayValue(dayjs(limit))
       rules.push({
         type: "datetime",
         constraint: "maxValue",

packages/client/src/stores/sidePanel.js

@@ -3,6 +3,7 @@ import { writable, derived } from "svelte/store"
 export const createSidePanelStore = () => {
   const initialState = {
     contentId: null,
+    ignoreClicksOutside: true,
   }
   const store = writable(initialState)
   const derivedStore = derived(store, $store => {
@@ -32,11 +33,18 @@ export const createSidePanelStore = () => {
     }, 50)
   }
 
+  const setIgnoreClicksOutside = bool => {
+    store.update(state => {
+      state.ignoreClicksOutside = bool
+      return state
+    })
+  }
   return {
     subscribe: derivedStore.subscribe,
     actions: {
       open,
       close,
+      setIgnoreClicksOutside,
     },
   }
 }

packages/client/src/utils/buttonActions.js

@@ -240,6 +240,7 @@ const triggerAutomationHandler = async action => {
 const navigationHandler = action => {
   const { url, peek, externalNewTab } = action.parameters
   routeStore.actions.navigate(url, peek, externalNewTab)
+  closeSidePanelHandler()
 }
 
 const queryExecutionHandler = async action => {
@@ -541,16 +542,22 @@ export const enrichButtonActions = (actions, context) => {
                 // then execute the rest of the actions in the chain
                 const result = await callback()
                 if (result !== false) {
-                  // Generate a new total context to pass into the next enrichment
+                  // Generate a new total context for the next enrichment
                   buttonContext.push(result)
                   const newContext = { ...context, actions: buttonContext }
 
-                  // Enrich and call the next button action if there is more than one action remaining
+                  // Enrich and call the next button action if there is more
+                  // than one action remaining
                   const next = enrichButtonActions(
                     actions.slice(i + 1),
                     newContext
                   )
-                  resolve(typeof next === "function" ? await next() : true)
+                  if (typeof next === "function") {
+                    // Pass the event context back into the new action chain
+                    resolve(await next(eventContext))
+                  } else {
+                    resolve(true)
+                  }
                 } else {
                   resolve(false)
                 }

packages/frontend-core/src/components/FilterBuilder.svelte

@@ -124,6 +124,7 @@
     const fieldSchema = schemaFields.find(x => x.name === filter.field)
     filter.type = fieldSchema?.type
     filter.subtype = fieldSchema?.subtype
+    filter.formulaType = fieldSchema?.formulaType
 
     // Update external type based on field
     filter.externalType = getSchema(filter)?.externalType
@@ -288,6 +289,7 @@
                     OperatorOptions.ContainsAny.value,
                   ].includes(filter.operator)}
                   disabled={filter.noValue}
+                  type={filter.valueType}
                 />
               {:else}
                 <Input disabled />
@@ -324,8 +326,6 @@
 <style>
   .container {
     width: 100%;
-    max-width: 1000px;
-    margin: 0 auto;
   }
   .fields {
     display: grid;

packages/frontend-core/src/components/FilterUsers.svelte

@@ -4,6 +4,7 @@
   import { createAPIClient } from "../api"
 
   export let API = createAPIClient()
+
   export let value = null
   export let disabled
   export let multiselect = false
@@ -23,12 +24,14 @@
   $: component = multiselect ? Multiselect : Select
 </script>
 
-<svelte:component
+<div class="user-control">
-  this={component}
+  <svelte:component
-  bind:value
+    this={component}
-  autocomplete
+    bind:value
-  {options}
+    autocomplete
-  getOptionLabel={option => option.email}
+    {options}
-  getOptionValue={option => option._id}
+    getOptionLabel={option => option.email}
-  {disabled}
+    getOptionValue={option => option._id}
-/>
+    {disabled}
+  />
+</div>

packages/frontend-core/src/components/grid/cells/HeaderCell.svelte

@@ -121,8 +121,14 @@
 
   const onContextMenu = e => {
     e.preventDefault()
-    ui.actions.blur()
+
-    open = !open
+    // The timeout allows time for clickoutside to close other open popvers
+    // before we show this one. Without the timeout, this popover closes again
+    // before it's even visible as clickoutside closes it.
+    setTimeout(() => {
+      ui.actions.blur()
+      open = !open
+    }, 10)
   }
 
   const sortAscending = () => {

packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte

@@ -18,7 +18,7 @@
     focusedCellAPI,
     focusedRowId,
     notifications,
-    isDatasourcePlus,
+    hasBudibaseIdentifiers,
   } = getContext("grid")
 
   let anchor
@@ -82,7 +82,7 @@
         </MenuItem>
         <MenuItem
           icon="Copy"
-          disabled={isNewRow || !$focusedRow?._id || !$isDatasourcePlus}
+          disabled={isNewRow || !$focusedRow?._id || !$hasBudibaseIdentifiers}
           on:click={() => copyToClipboard($focusedRow?._id)}
           on:click={menu.actions.close}
         >
@@ -90,7 +90,7 @@
         </MenuItem>
         <MenuItem
           icon="Copy"
-          disabled={isNewRow || !$focusedRow?._rev}
+          disabled={isNewRow || !$focusedRow?._rev || !$hasBudibaseIdentifiers}
           on:click={() => copyToClipboard($focusedRow?._rev)}
           on:click={menu.actions.close}
         >

packages/frontend-core/src/components/grid/stores/datasource.js

@@ -75,14 +75,18 @@ export const deriveStores = context => {
     }
   )
 
-  const isDatasourcePlus = derived(datasource, $datasource => {
+  const hasBudibaseIdentifiers = derived(datasource, $datasource => {
-    return ["table", "viewV2"].includes($datasource?.type)
+    let type = $datasource?.type
+    if (type === "provider") {
+      type = $datasource.value?.datasource?.type
+    }
+    return ["table", "viewV2", "link"].includes(type)
   })
 
   return {
     schema,
     enrichedSchema,
-    isDatasourcePlus,
+    hasBudibaseIdentifiers,
   }
 }
 

packages/frontend-core/src/components/grid/stores/menu.js

@@ -17,6 +17,7 @@ export const createActions = context => {
 
   const open = (cellId, e) => {
     e.preventDefault()
+    e.stopPropagation()
 
     // Get DOM node for grid data wrapper to compute relative position to
     const gridNode = document.getElementById(gridID)

packages/frontend-core/src/components/grid/stores/rows.js

@@ -83,7 +83,7 @@ export const createActions = context => {
     error,
     notifications,
     fetch,
-    isDatasourcePlus,
+    hasBudibaseIdentifiers,
     refreshing,
   } = context
   const instanceLoaded = writable(false)
@@ -196,9 +196,16 @@ export const createActions = context => {
   // Handles validation errors from the rows API and updates local validation
   // state, storing error messages against relevant cells
   const handleValidationError = (rowId, error) => {
+    let errorString
+    if (typeof error === "string") {
+      errorString = error
+    } else if (typeof error?.message === "string") {
+      errorString = error.message
+    }
+
     // If the server doesn't reply with a valid error, assume that the source
     // of the error is the focused cell's column
-    if (!error?.json?.validationErrors && error?.message) {
+    if (!error?.json?.validationErrors && errorString) {
       const focusedColumn = get(focusedCellId)?.split("-")[1]
       if (focusedColumn) {
         error = {
@@ -261,7 +268,7 @@ export const createActions = context => {
         focusedCellId.set(`${rowId}-${erroredColumns[0]}`)
       }
     } else {
-      get(notifications).error(error?.message || "An unknown error occurred")
+      get(notifications).error(errorString || "An unknown error occurred")
     }
   }
 
@@ -458,14 +465,14 @@ export const createActions = context => {
     }
     let rowsToAppend = []
     let newRow
-    const $isDatasourcePlus = get(isDatasourcePlus)
+    const $hasBudibaseIdentifiers = get(hasBudibaseIdentifiers)
     for (let i = 0; i < newRows.length; i++) {
       newRow = newRows[i]
 
       // Ensure we have a unique _id.
       // This means generating one for non DS+, overwriting any that may already
       // exist as we cannot allow duplicates.
-      if (!$isDatasourcePlus) {
+      if (!$hasBudibaseIdentifiers) {
         newRow._id = Helpers.uuid()
       }
 
@@ -510,7 +517,7 @@ export const createActions = context => {
   const cleanRow = row => {
     let clone = { ...row }
     delete clone.__idx
-    if (!get(isDatasourcePlus)) {
+    if (!get(hasBudibaseIdentifiers)) {
       delete clone._id
     }
     return clone

packages/frontend-core/src/constants.js

@@ -57,6 +57,7 @@ export const PlanType = {
   PRO: "pro",
   BUSINESS: "business",
   ENTERPRISE: "enterprise",
+  ENTERPRISE_BASIC_TRIAL: "enterprise_basic_trial",
 }
 
 /**
@@ -124,8 +125,8 @@ export const TypeIconMap = {
   [FieldType.ARRAY]: "Duplicate",
   [FieldType.NUMBER]: "123",
   [FieldType.BOOLEAN]: "Boolean",
-  [FieldType.ATTACHMENTS]: "Attach",
+  [FieldType.ATTACHMENTS]: "DocumentFragmentGroup",
-  [FieldType.ATTACHMENT_SINGLE]: "Attach",
+  [FieldType.ATTACHMENT_SINGLE]: "DocumentFragment",
   [FieldType.LINK]: "DataCorrelated",
   [FieldType.FORMULA]: "Calculator",
   [FieldType.JSON]: "Brackets",

packages/pro

@@ -1 +1 @@
-Subproject commit 479879246aac5dd3073cc695945c62c41fae5b0e
+Subproject commit ff397e5454ad3361b25efdf14746c36dcbd3f409

packages/server/src/api/controllers/application.ts

@@ -52,6 +52,8 @@ import {
   FetchAppPackageResponse,
   DuplicateAppRequest,
   DuplicateAppResponse,
+  UpdateAppRequest,
+  UpdateAppResponse,
 } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import sdk from "../../sdk"
@@ -450,7 +452,7 @@ export async function create(ctx: UserCtx<CreateAppRequest, App>) {
 // This endpoint currently operates as a PATCH rather than a PUT
 // Thus name and url fields are handled only if present
 export async function update(
-  ctx: UserCtx<{ name?: string; url?: string }, App>
+  ctx: UserCtx<UpdateAppRequest, UpdateAppResponse>
 ) {
   const apps = (await dbCore.getAllApps({ dev: true })) as App[]
   // validation

packages/server/src/api/controllers/row/index.ts

@@ -2,7 +2,7 @@ import stream from "stream"
 import archiver from "archiver"
 
 import { quotas } from "@budibase/pro"
-import { objectStore } from "@budibase/backend-core"
+import { objectStore, context } from "@budibase/backend-core"
 import * as internal from "./internal"
 import * as external from "./external"
 import { isExternalTableID } from "../../../integrations/utils"
@@ -198,8 +198,18 @@ export async function destroy(ctx: UserCtx<DeleteRowRequest>) {
 export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
   const tableId = utils.getTableId(ctx)
 
+  await context.ensureSnippetContext(true)
+
+  const enrichedQuery = await utils.enrichSearchContext(
+    { ...ctx.request.body.query },
+    {
+      user: sdk.users.getUserContextBindings(ctx.user),
+    }
+  )
+
   const searchParams: RowSearchParams = {
     ...ctx.request.body,
+    query: enrichedQuery,
     tableId,
   }
 

packages/server/src/api/controllers/row/staticFormula.ts

@@ -1,20 +1,11 @@
 import { getRowParams } from "../../../db/utils"
 import {
   outputProcessing,
-  processAutoColumn,
   processFormulas,
 } from "../../../utilities/rowProcessor"
-import { context, locks } from "@budibase/backend-core"
+import { context } from "@budibase/backend-core"
-import {
+import { Table, Row, FormulaType, FieldType } from "@budibase/types"
-  Table,
-  Row,
-  LockType,
-  LockName,
-  FormulaType,
-  FieldType,
-} from "@budibase/types"
 import * as linkRows from "../../../db/linkedRows"
-import sdk from "../../../sdk"
 import isEqual from "lodash/isEqual"
 import { cloneDeep } from "lodash/fp"
 
@@ -151,30 +142,7 @@ export async function finaliseRow(
   // if another row has been written since processing this will
   // handle the auto ID clash
   if (oldTable && !isEqual(oldTable, table)) {
-    try {
+    await db.put(table)
-      await db.put(table)
-    } catch (err: any) {
-      if (err.status === 409) {
-        // Some conflicts with the autocolumns occurred, we need to refetch the table and recalculate
-        await locks.doWithLock(
-          {
-            type: LockType.AUTO_EXTEND,
-            name: LockName.PROCESS_AUTO_COLUMNS,
-            resource: table._id,
-          },
-          async () => {
-            const latestTable = await sdk.tables.getTable(table._id!)
-            let response = processAutoColumn(null, latestTable, row, {
-              reprocessing: true,
-            })
-            await db.put(response.table)
-            row = response.row
-          }
-        )
-      } else {
-        throw err
-      }
-    }
   }
   const response = await db.put(row)
   // for response, calculate the formulas for the enriched row

packages/server/src/api/controllers/row/utils/basic.ts

@@ -73,12 +73,15 @@ export function basicProcessing({
   // filter the row down to what is actually the row (not joined)
   for (let field of Object.values(table.schema)) {
     const fieldName = field.name
-    const value = extractFieldValue({
+    let value = extractFieldValue({
       row,
       tableName: table.name,
       fieldName,
       isLinked,
     })
+    if (value instanceof Buffer) {
+      value = value.toString()
+    }
     // all responses include "select col as table.col" so that overlaps are handled
     if (value != null) {
       thisRow[fieldName] = value

packages/server/src/api/controllers/row/utils/utils.ts

@@ -22,7 +22,7 @@ import {
   getInternalRowId,
 } from "./basic"
 import sdk from "../../../../sdk"
-
+import { processStringSync } from "@budibase/string-templates"
 import validateJs from "validate.js"
 
 validateJs.extend(validateJs.validators.datetime, {
@@ -117,6 +117,19 @@ export async function validate(
   })
 }
 
+function fixBooleanFields({ row, table }: { row: Row; table: Table }) {
+  for (let col of Object.values(table.schema)) {
+    if (col.type === FieldType.BOOLEAN) {
+      if (row[col.name] === 1) {
+        row[col.name] = true
+      } else if (row[col.name] === 0) {
+        row[col.name] = false
+      }
+    }
+  }
+  return row
+}
+
 export async function sqlOutputProcessing(
   rows: DatasourcePlusQueryResponse,
   table: Table,
@@ -161,7 +174,9 @@ export async function sqlOutputProcessing(
     if (thisRow._id == null) {
       throw new Error("Unable to generate row ID for SQL rows")
     }
-    finalRows[thisRow._id] = thisRow
+
+    finalRows[thisRow._id] = fixBooleanFields({ row: thisRow, table })
+
     // do this at end once its been added to the final rows
     finalRows = await updateRelationshipColumns(
       table,
@@ -189,3 +204,63 @@ export async function sqlOutputProcessing(
 export function isUserMetadataTable(tableId: string) {
   return tableId === InternalTables.USER_METADATA
 }
+
+export async function enrichArrayContext(
+  fields: any[],
+  inputs = {},
+  helpers = true
+): Promise<any[]> {
+  const map: Record<string, any> = {}
+  for (let index in fields) {
+    map[index] = fields[index]
+  }
+  const output = await enrichSearchContext(map, inputs, helpers)
+  const outputArray: any[] = []
+  for (let [key, value] of Object.entries(output)) {
+    outputArray[parseInt(key)] = value
+  }
+  return outputArray
+}
+
+export async function enrichSearchContext(
+  fields: Record<string, any>,
+  inputs = {},
+  helpers = true
+): Promise<Record<string, any>> {
+  const enrichedQuery: Record<string, any> = {}
+  if (!fields || !inputs) {
+    return enrichedQuery
+  }
+  const parameters = { ...inputs }
+
+  if (Array.isArray(fields)) {
+    return enrichArrayContext(fields, inputs, helpers)
+  }
+
+  // enrich the fields with dynamic parameters
+  for (let key of Object.keys(fields)) {
+    if (fields[key] == null) {
+      enrichedQuery[key] = null
+      continue
+    }
+    if (typeof fields[key] === "object") {
+      // enrich nested fields object
+      enrichedQuery[key] = await enrichSearchContext(
+        fields[key],
+        parameters,
+        helpers
+      )
+    } else if (typeof fields[key] === "string") {
+      // enrich string value as normal
+      enrichedQuery[key] = processStringSync(fields[key], parameters, {
+        noEscaping: true,
+        noHelpers: !helpers,
+        escapeNewlines: true,
+      })
+    } else {
+      enrichedQuery[key] = fields[key]
+    }
+  }
+
+  return enrichedQuery
+}

packages/server/src/api/controllers/row/views.ts

@@ -9,7 +9,8 @@ import {
 } from "@budibase/types"
 import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../../sdk"
-import { db } from "@budibase/backend-core"
+import { db, context } from "@budibase/backend-core"
+import { enrichSearchContext } from "./utils"
 
 export async function searchView(
   ctx: UserCtx<SearchViewRowRequest, SearchRowResponse>
@@ -56,10 +57,16 @@ export async function searchView(
     })
   }
 
+  await context.ensureSnippetContext(true)
+
+  const enrichedQuery = await enrichSearchContext(query, {
+    user: sdk.users.getUserContextBindings(ctx.user),
+  })
+
   const searchOptions: RequiredKeys<SearchViewRowRequest> &
     RequiredKeys<Pick<RowSearchParams, "tableId" | "query" | "fields">> = {
     tableId: view.tableId,
-    query,
+    query: enrichedQuery,
     fields: viewFields,
     ...getSortOptions(body, view),
     limit: body.limit,

packages/server/src/api/controllers/table/utils.ts

@@ -128,6 +128,7 @@ export async function importToRows(
   for (let i = 0; i < data.length; i++) {
     let row = data[i]
     row._id = generateRowID(table._id!)
+    row.type = "row"
     row.tableId = table._id
 
     // We use a reference to table here and update it after input processing,

packages/server/src/api/routes/tests/attachment.spec.ts

@@ -4,10 +4,12 @@ import { APIError } from "@budibase/types"
 describe("/api/applications/:appId/sync", () => {
   let config = setup.getConfig()
 
-  afterAll(setup.afterAll)
   beforeAll(async () => {
     await config.init()
   })
+  afterAll(async () => {
+    setup.afterAll()
+  })
 
   describe("/api/attachments/process", () => {
     it("should accept an image file upload", async () => {
@@ -18,7 +20,8 @@ describe("/api/applications/:appId/sync", () => {
       expect(resp.length).toBe(1)
 
       let upload = resp[0]
-      expect(upload.url.endsWith(".jpg")).toBe(true)
+
+      expect(upload.url.split("?")[0].endsWith(".jpg")).toBe(true)
       expect(upload.extension).toBe("jpg")
       expect(upload.size).toBe(1)
       expect(upload.name).toBe("1px.jpg")

packages/server/src/api/routes/tests/backup.spec.ts

@@ -1,16 +1,18 @@
+import { mocks } from "@budibase/backend-core/tests"
 import tk from "timekeeper"
 import * as setup from "./utilities"
 import { events } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
-import { mocks } from "@budibase/backend-core/tests"
 
 mocks.licenses.useBackups()
 
 describe("/backups", () => {
   let config = setup.getConfig()
 
-  afterAll(setup.afterAll)
+  afterAll(async () => {
+    setup.afterAll()
+  })
 
   beforeEach(async () => {
     tk.reset()

packages/server/src/api/routes/tests/row.spec.ts

@@ -32,8 +32,6 @@ import * as uuid from "uuid"
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
 
-jest.unmock("mssql")
-
 describe.each([
   ["internal", undefined],
   [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
@@ -131,7 +129,13 @@ describe.each([
 
   const assertRowUsage = async (expected: number) => {
     const usage = await getRowUsage()
-    expect(usage).toBe(expected)
+
+    // Because our quota tracking is not perfect, we allow a 10% margin of
+    // error.  This is to account for the fact that parallel writes can result
+    // in some quota updates getting lost. We don't have any need to solve this
+    // right now, so we just allow for some error.
+    expect(usage).toBeGreaterThan(expected * 0.9)
+    expect(usage).toBeLessThan(expected * 1.1)
   }
 
   const defaultRowFields = isInternal
@@ -194,39 +198,99 @@ describe.each([
       await assertRowUsage(rowUsage)
     })
 
-    it("increment row autoId per create row request", async () => {
+    isInternal &&
-      const rowUsage = await getRowUsage()
+      it("increment row autoId per create row request", async () => {
+        const rowUsage = await getRowUsage()
 
-      const newTable = await config.api.table.save(
+        const newTable = await config.api.table.save(
-        saveTableRequest({
+          saveTableRequest({
-          schema: {
+            schema: {
-            "Row ID": {
+              "Row ID": {
-              name: "Row ID",
+                name: "Row ID",
-              type: FieldType.NUMBER,
+                type: FieldType.NUMBER,
-              subtype: AutoFieldSubType.AUTO_ID,
+                subtype: AutoFieldSubType.AUTO_ID,
-              icon: "ri-magic-line",
+                icon: "ri-magic-line",
-              autocolumn: true,
+                autocolumn: true,
-              constraints: {
+                constraints: {
-                type: "number",
+                  type: "number",
-                presence: true,
+                  presence: true,
-                numericality: {
+                  numericality: {
-                  greaterThanOrEqualTo: "",
+                    greaterThanOrEqualTo: "",
-                  lessThanOrEqualTo: "",
+                    lessThanOrEqualTo: "",
+                  },
                 },
               },
             },
-          },
+          })
-        })
+        )
-      )
 
-      let previousId = 0
+        let previousId = 0
-      for (let i = 0; i < 10; i++) {
+        for (let i = 0; i < 10; i++) {
-        const row = await config.api.row.save(newTable._id!, {})
+          const row = await config.api.row.save(newTable._id!, {})
-        expect(row["Row ID"]).toBeGreaterThan(previousId)
+          expect(row["Row ID"]).toBeGreaterThan(previousId)
-        previousId = row["Row ID"]
+          previousId = row["Row ID"]
-      }
+        }
-      await assertRowUsage(rowUsage + 10)
+        await assertRowUsage(rowUsage + 10)
-    })
+      })
+
+    isInternal &&
+      it("should increment auto ID correctly when creating rows in parallel", async () => {
+        const table = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              "Row ID": {
+                name: "Row ID",
+                type: FieldType.NUMBER,
+                subtype: AutoFieldSubType.AUTO_ID,
+                icon: "ri-magic-line",
+                autocolumn: true,
+                constraints: {
+                  type: "number",
+                  presence: true,
+                  numericality: {
+                    greaterThanOrEqualTo: "",
+                    lessThanOrEqualTo: "",
+                  },
+                },
+              },
+            },
+          })
+        )
+
+        const sequence = Array(50)
+          .fill(0)
+          .map((_, i) => i + 1)
+
+        // This block of code is simulating users creating auto ID rows at the
+        // same time. It's expected that this operation will sometimes return
+        // a document conflict error (409), but the idea is to retry in those
+        // situations. The code below does this a large number of times with
+        // small, random delays between them to try and get through the list
+        // as quickly as possible.
+        await Promise.all(
+          sequence.map(async () => {
+            const attempts = 20
+            for (let attempt = 0; attempt < attempts; attempt++) {
+              try {
+                await config.api.row.save(table._id!, {})
+                return
+              } catch (e) {
+                await new Promise(r => setTimeout(r, Math.random() * 15))
+              }
+            }
+            throw new Error(`Failed to create row after ${attempts} attempts`)
+          })
+        )
+
+        const rows = await config.api.row.fetch(table._id!)
+        expect(rows).toHaveLength(50)
+
+        // The main purpose of this test is to ensure that even under pressure,
+        // we maintain data integrity. An auto ID column should hand out
+        // monotonically increasing unique integers no matter what.
+        const ids = rows.map(r => r["Row ID"])
+        expect(ids).toEqual(expect.arrayContaining(sequence))
+      })
 
     isInternal &&
       it("row values are coerced", async () => {
@@ -856,7 +920,7 @@ describe.each([
         await config.withEnv({ SELF_HOSTED: "true" }, async () => {
           return context.doInAppContext(config.getAppId(), async () => {
             const enriched = await outputProcessing(table, [row])
-            expect((enriched as Row[])[0].attachment.url).toBe(
+            expect((enriched as Row[])[0].attachment.url.split("?")[0]).toBe(
               `/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
             )
           })
@@ -889,7 +953,7 @@ describe.each([
         await config.withEnv({ SELF_HOSTED: "true" }, async () => {
           return context.doInAppContext(config.getAppId(), async () => {
             const enriched = await outputProcessing(table, [row])
-            expect((enriched as Row[])[0].attachment[0].url).toBe(
+            expect((enriched as Row[])[0].attachment[0].url.split("?")[0]).toBe(
               `/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
             )
           })

packages/server/src/api/routes/tests/search.spec.ts

@@ -1,10 +1,13 @@
 import { tableForDatasource } from "../../../tests/utilities/structures"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
+import { db as dbCore } from "@budibase/backend-core"
 
 import * as setup from "./utilities"
 import {
+  AutoFieldSubType,
   Datasource,
   EmptyFilterOption,
+  BBReferenceFieldSubType,
   FieldType,
   RowSearchParams,
   SearchFilters,
@@ -12,32 +15,51 @@ import {
   SortType,
   Table,
   TableSchema,
+  User,
 } from "@budibase/types"
 import _ from "lodash"
+import tk from "timekeeper"
+import { encodeJSBinding } from "@budibase/string-templates"
 
-jest.unmock("mssql")
+const serverTime = new Date("2024-05-06T00:00:00.000Z")
+tk.freeze(serverTime)
 
 describe.each([
-  ["internal", undefined],
+  ["lucene", undefined],
-  ["internal-sqs", undefined],
+  ["sqs", undefined],
   [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
   [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
   [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
   [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
 ])("/api/:sourceId/search (%s)", (name, dsProvider) => {
-  const isSqs = name === "internal-sqs"
+  const isSqs = name === "sqs"
-  const isInternal = name === "internal"
+  const isLucene = name === "lucene"
+  const isInternal = isSqs || isLucene
   const config = setup.getConfig()
 
   let envCleanup: (() => void) | undefined
   let datasource: Datasource | undefined
   let table: Table
 
+  const snippets = [
+    {
+      name: "WeeksAgo",
+      code: "return function (weeks) {\n  const currentTime = new Date();\n  currentTime.setDate(currentTime.getDate()-(7 * (weeks || 1)));\n  return currentTime.toISOString();\n}",
+    },
+  ]
+
   beforeAll(async () => {
     if (isSqs) {
       envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" })
     }
     await config.init()
+
+    if (config.app?.appId) {
+      config.app = await config.api.application.update(config.app?.appId, {
+        snippets,
+      })
+    }
+
     if (dsProvider) {
       datasource = await config.createDatasource({
         datasource: await dsProvider,
@@ -59,12 +81,28 @@ describe.each([
   }
 
   async function createRows(rows: Record<string, any>[]) {
-    await Promise.all(rows.map(r => config.api.row.save(table._id!, r)))
+    await config.api.row.bulkImport(table._id!, { rows })
   }
 
   class SearchAssertion {
     constructor(private readonly query: RowSearchParams) {}
 
+    private findRow(expectedRow: any, foundRows: any[]) {
+      const row = foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
+      if (!row) {
+        const fields = Object.keys(expectedRow)
+        // To make the error message more readable, we only include the fields
+        // that are present in the expected row.
+        const searchedObjects = foundRows.map(row => _.pick(row, fields))
+        throw new Error(
+          `Failed to find row: ${JSON.stringify(
+            expectedRow
+          )} in ${JSON.stringify(searchedObjects)}`
+        )
+      }
+      return row
+    }
+
     // Asserts that the query returns rows matching exactly the set of rows
     // passed in. The order of the rows matters. Rows returned in an order
     // different to the one passed in will cause the assertion to fail.  Extra
@@ -80,9 +118,7 @@ describe.each([
       // eslint-disable-next-line jest/no-standalone-expect
       expect(foundRows).toEqual(
         expectedRows.map((expectedRow: any) =>
-          expect.objectContaining(
+          expect.objectContaining(this.findRow(expectedRow, foundRows))
-            foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
-          )
         )
       )
     }
@@ -102,9 +138,7 @@ describe.each([
       expect(foundRows).toEqual(
         expect.arrayContaining(
           expectedRows.map((expectedRow: any) =>
-            expect.objectContaining(
+            expect.objectContaining(this.findRow(expectedRow, foundRows))
-              foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
-            )
           )
         )
       )
@@ -123,9 +157,7 @@ describe.each([
       expect(foundRows).toEqual(
         expect.arrayContaining(
           expectedRows.map((expectedRow: any) =>
-            expect.objectContaining(
+            expect.objectContaining(this.findRow(expectedRow, foundRows))
-              foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
-            )
           )
         )
       )
@@ -154,7 +186,294 @@ describe.each([
     return expectSearch({ query })
   }
 
-  describe("strings", () => {
+  describe("boolean", () => {
+    beforeAll(async () => {
+      await createTable({
+        isTrue: { name: "isTrue", type: FieldType.BOOLEAN },
+      })
+      await createRows([{ isTrue: true }, { isTrue: false }])
+    })
+
+    describe("equal", () => {
+      it("successfully finds true row", () =>
+        expectQuery({ equal: { isTrue: true } }).toMatchExactly([
+          { isTrue: true },
+        ]))
+
+      it("successfully finds false row", () =>
+        expectQuery({ equal: { isTrue: false } }).toMatchExactly([
+          { isTrue: false },
+        ]))
+    })
+
+    describe("notEqual", () => {
+      it("successfully finds false row", () =>
+        expectQuery({ notEqual: { isTrue: true } }).toContainExactly([
+          { isTrue: false },
+        ]))
+
+      it("successfully finds true row", () =>
+        expectQuery({ notEqual: { isTrue: false } }).toContainExactly([
+          { isTrue: true },
+        ]))
+    })
+
+    describe("oneOf", () => {
+      it("successfully finds true row", () =>
+        expectQuery({ oneOf: { isTrue: [true] } }).toContainExactly([
+          { isTrue: true },
+        ]))
+
+      it("successfully finds false row", () =>
+        expectQuery({ oneOf: { isTrue: [false] } }).toContainExactly([
+          { isTrue: false },
+        ]))
+    })
+
+    describe("sort", () => {
+      it("sorts ascending", () =>
+        expectSearch({
+          query: {},
+          sort: "isTrue",
+          sortOrder: SortOrder.ASCENDING,
+        }).toMatchExactly([{ isTrue: false }, { isTrue: true }]))
+
+      it("sorts descending", () =>
+        expectSearch({
+          query: {},
+          sort: "isTrue",
+          sortOrder: SortOrder.DESCENDING,
+        }).toMatchExactly([{ isTrue: true }, { isTrue: false }]))
+    })
+  })
+
+  // Ensure all bindings resolve and perform as expected
+  describe("bindings", () => {
+    let globalUsers: any = []
+
+    const future = new Date(serverTime.getTime())
+    future.setDate(future.getDate() + 30)
+
+    const rows = (currentUser: User) => {
+      return [
+        { name: "foo", appointment: "1982-01-05T00:00:00.000Z" },
+        { name: "bar", appointment: "1995-05-06T00:00:00.000Z" },
+        { name: currentUser.firstName, appointment: future.toISOString() },
+        { name: "serverDate", appointment: serverTime.toISOString() },
+        {
+          name: "single user, session user",
+          single_user: JSON.stringify([currentUser]),
+        },
+        {
+          name: "single user",
+          single_user: JSON.stringify([globalUsers[0]]),
+        },
+        {
+          name: "multi user",
+          multi_user: JSON.stringify(globalUsers),
+        },
+        {
+          name: "multi user with session user",
+          multi_user: JSON.stringify([...globalUsers, currentUser]),
+        },
+      ]
+    }
+
+    beforeAll(async () => {
+      // Set up some global users
+      globalUsers = await Promise.all(
+        Array(2)
+          .fill(0)
+          .map(async () => {
+            const globalUser = await config.globalUser()
+            const userMedataId = globalUser._id
+              ? dbCore.generateUserMetadataID(globalUser._id)
+              : null
+            return {
+              _id: globalUser._id,
+              _meta: userMedataId,
+            }
+          })
+      )
+
+      await createTable({
+        name: { name: "name", type: FieldType.STRING },
+        appointment: { name: "appointment", type: FieldType.DATETIME },
+        single_user: {
+          name: "single_user",
+          type: FieldType.BB_REFERENCE,
+          subtype: BBReferenceFieldSubType.USER,
+        },
+        multi_user: {
+          name: "multi_user",
+          type: FieldType.BB_REFERENCE,
+          subtype: BBReferenceFieldSubType.USERS,
+        },
+      })
+      await createRows(rows(config.getUser()))
+    })
+
+    // !! Current User is auto generated per run
+    it("should return all rows matching the session user firstname", async () => {
+      await expectQuery({
+        equal: { name: "{{ [user].firstName }}" },
+      }).toContainExactly([
+        {
+          name: config.getUser().firstName,
+          appointment: future.toISOString(),
+        },
+      ])
+    })
+
+    it("should parse the date binding and return all rows after the resolved value", async () => {
+      await expectQuery({
+        range: {
+          appointment: {
+            low: "{{ [now] }}",
+            high: "9999-00-00T00:00:00.000Z",
+          },
+        },
+      }).toContainExactly([
+        {
+          name: config.getUser().firstName,
+          appointment: future.toISOString(),
+        },
+        { name: "serverDate", appointment: serverTime.toISOString() },
+      ])
+    })
+
+    it("should parse the date binding and return all rows before the resolved value", async () => {
+      await expectQuery({
+        range: {
+          appointment: {
+            low: "0000-00-00T00:00:00.000Z",
+            high: "{{ [now] }}",
+          },
+        },
+      }).toContainExactly([
+        { name: "foo", appointment: "1982-01-05T00:00:00.000Z" },
+        { name: "bar", appointment: "1995-05-06T00:00:00.000Z" },
+        { name: "serverDate", appointment: serverTime.toISOString() },
+      ])
+    })
+
+    it("should parse the encoded js snippet. Return rows with appointments up to 1 week in the past", async () => {
+      const jsBinding = "return snippets.WeeksAgo();"
+      const encodedBinding = encodeJSBinding(jsBinding)
+
+      await expectQuery({
+        range: {
+          appointment: {
+            low: "0000-00-00T00:00:00.000Z",
+            high: encodedBinding,
+          },
+        },
+      }).toContainExactly([
+        { name: "foo", appointment: "1982-01-05T00:00:00.000Z" },
+        { name: "bar", appointment: "1995-05-06T00:00:00.000Z" },
+      ])
+    })
+
+    it("should parse the encoded js binding. Return rows with appointments 2 weeks in the past", async () => {
+      const jsBinding =
+        "const currentTime = new Date()\ncurrentTime.setDate(currentTime.getDate()-14);\nreturn currentTime.toISOString();"
+      const encodedBinding = encodeJSBinding(jsBinding)
+
+      await expectQuery({
+        range: {
+          appointment: {
+            low: "0000-00-00T00:00:00.000Z",
+            high: encodedBinding,
+          },
+        },
+      }).toContainExactly([
+        { name: "foo", appointment: "1982-01-05T00:00:00.000Z" },
+        { name: "bar", appointment: "1995-05-06T00:00:00.000Z" },
+      ])
+    })
+
+    it("should match a single user row by the session user id", async () => {
+      await expectQuery({
+        equal: { single_user: "{{ [user]._id }}" },
+      }).toContainExactly([
+        {
+          name: "single user, session user",
+          single_user: [{ _id: config.getUser()._id }],
+        },
+      ])
+    })
+
+    // TODO(samwho): fix for SQS
+    !isSqs &&
+      it("should match the session user id in a multi user field", async () => {
+        const allUsers = [...globalUsers, config.getUser()].map((user: any) => {
+          return { _id: user._id }
+        })
+
+        await expectQuery({
+          contains: { multi_user: ["{{ [user]._id }}"] },
+        }).toContainExactly([
+          {
+            name: "multi user with session user",
+            multi_user: allUsers,
+          },
+        ])
+      })
+
+    // TODO(samwho): fix for SQS
+    !isSqs &&
+      it("should not match the session user id in a multi user field", async () => {
+        await expectQuery({
+          notContains: { multi_user: ["{{ [user]._id }}"] },
+          notEmpty: { multi_user: true },
+        }).toContainExactly([
+          {
+            name: "multi user",
+            multi_user: globalUsers.map((user: any) => {
+              return { _id: user._id }
+            }),
+          },
+        ])
+      })
+
+    it("should match the session user id and a user table row id using helpers, user binding and a static user id.", async () => {
+      await expectQuery({
+        oneOf: {
+          single_user: [
+            "{{ default [user]._id '_empty_' }}",
+            globalUsers[0]._id,
+          ],
+        },
+      }).toContainExactly([
+        {
+          name: "single user, session user",
+          single_user: [{ _id: config.getUser()._id }],
+        },
+        {
+          name: "single user",
+          single_user: [{ _id: globalUsers[0]._id }],
+        },
+      ])
+    })
+
+    it("should resolve 'default' helper to '_empty_' when binding resolves to nothing", async () => {
+      await expectQuery({
+        oneOf: {
+          single_user: [
+            "{{ default [user]._idx '_empty_' }}",
+            globalUsers[0]._id,
+          ],
+        },
+      }).toContainExactly([
+        {
+          name: "single user",
+          single_user: [{ _id: globalUsers[0]._id }],
+        },
+      ])
+    })
+  })
+
+  describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => {
     beforeAll(async () => {
       await createTable({
         name: { name: "name", type: FieldType.STRING },
@@ -250,6 +569,31 @@ describe.each([
         }).toFindNothing())
     })
 
+    describe("empty", () => {
+      it("finds no empty rows", () =>
+        expectQuery({ empty: { name: null } }).toFindNothing())
+
+      it("should not be affected by when filter empty behaviour", () =>
+        expectQuery({
+          empty: { name: null },
+          onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+        }).toFindNothing())
+    })
+
+    describe("notEmpty", () => {
+      it("finds all non-empty rows", () =>
+        expectQuery({ notEmpty: { name: null } }).toContainExactly([
+          { name: "foo" },
+          { name: "bar" },
+        ]))
+
+      it("should not be affected by when filter empty behaviour", () =>
+        expectQuery({
+          notEmpty: { name: null },
+          onEmptyFilter: EmptyFilterOption.RETURN_NONE,
+        }).toContainExactly([{ name: "foo" }, { name: "bar" }]))
+    })
+
     describe("sort", () => {
       it("sorts ascending", () =>
         expectSearch({
@@ -339,14 +683,14 @@ describe.each([
         }).toFindNothing())
 
       // We never implemented half-open ranges in Lucene.
-      !isInternal &&
+      !isLucene &&
         it("can search using just a low value", () =>
           expectQuery({
             range: { age: { low: 5 } },
           }).toContainExactly([{ age: 10 }]))
 
       // We never implemented half-open ranges in Lucene.
-      !isInternal &&
+      !isLucene &&
         it("can search using just a high value", () =>
           expectQuery({
             range: { age: { high: 5 } },
@@ -457,14 +801,14 @@ describe.each([
         }).toFindNothing())
 
       // We never implemented half-open ranges in Lucene.
-      !isInternal &&
+      !isLucene &&
         it("can search using just a low value", () =>
           expectQuery({
             range: { dob: { low: JAN_5TH } },
           }).toContainExactly([{ dob: JAN_10TH }]))
 
       // We never implemented half-open ranges in Lucene.
-      !isInternal &&
+      !isLucene &&
         it("can search using just a high value", () =>
           expectQuery({
             range: { dob: { high: JAN_5TH } },
@@ -506,7 +850,7 @@ describe.each([
     })
   })
 
-  describe("array of strings", () => {
+  describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
     beforeAll(async () => {
       await createTable({
         numbers: {
@@ -642,7 +986,7 @@ describe.each([
     // Range searches against bigints don't seem to work at all in Lucene, and I
     // couldn't figure out why. Given that we're replacing Lucene with SQS,
     // we've decided not to spend time on it.
-    !isInternal &&
+    !isLucene &&
       describe("range", () => {
         it("successfully finds a row", () =>
           expectQuery({
@@ -675,4 +1019,137 @@ describe.each([
           }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]))
       })
   })
+
+  isInternal &&
+    describe("auto", () => {
+      beforeAll(async () => {
+        await createTable({
+          auto: {
+            name: "auto",
+            type: FieldType.AUTO,
+            autocolumn: true,
+            subtype: AutoFieldSubType.AUTO_ID,
+          },
+        })
+        await createRows(new Array(10).fill({}))
+      })
+
+      describe("equal", () => {
+        it("successfully finds a row", () =>
+          expectQuery({ equal: { auto: 1 } }).toContainExactly([{ auto: 1 }]))
+
+        it("fails to find nonexistent row", () =>
+          expectQuery({ equal: { auto: 0 } }).toFindNothing())
+      })
+
+      describe("not equal", () => {
+        it("successfully finds a row", () =>
+          expectQuery({ notEqual: { auto: 1 } }).toContainExactly([
+            { auto: 2 },
+            { auto: 3 },
+            { auto: 4 },
+            { auto: 5 },
+            { auto: 6 },
+            { auto: 7 },
+            { auto: 8 },
+            { auto: 9 },
+            { auto: 10 },
+          ]))
+
+        it("fails to find nonexistent row", () =>
+          expectQuery({ notEqual: { auto: 0 } }).toContainExactly([
+            { auto: 1 },
+            { auto: 2 },
+            { auto: 3 },
+            { auto: 4 },
+            { auto: 5 },
+            { auto: 6 },
+            { auto: 7 },
+            { auto: 8 },
+            { auto: 9 },
+            { auto: 10 },
+          ]))
+      })
+
+      describe("oneOf", () => {
+        it("successfully finds a row", () =>
+          expectQuery({ oneOf: { auto: [1] } }).toContainExactly([{ auto: 1 }]))
+
+        it("fails to find nonexistent row", () =>
+          expectQuery({ oneOf: { auto: [0] } }).toFindNothing())
+      })
+
+      describe("range", () => {
+        it("successfully finds a row", () =>
+          expectQuery({
+            range: { auto: { low: 1, high: 1 } },
+          }).toContainExactly([{ auto: 1 }]))
+
+        it("successfully finds multiple rows", () =>
+          expectQuery({
+            range: { auto: { low: 1, high: 2 } },
+          }).toContainExactly([{ auto: 1 }, { auto: 2 }]))
+
+        it("successfully finds a row with a high bound", () =>
+          expectQuery({
+            range: { auto: { low: 2, high: 2 } },
+          }).toContainExactly([{ auto: 2 }]))
+
+        it("successfully finds no rows", () =>
+          expectQuery({
+            range: { auto: { low: 0, high: 0 } },
+          }).toFindNothing())
+
+        isSqs &&
+          it("can search using just a low value", () =>
+            expectQuery({
+              range: { auto: { low: 9 } },
+            }).toContainExactly([{ auto: 9 }, { auto: 10 }]))
+
+        isSqs &&
+          it("can search using just a high value", () =>
+            expectQuery({
+              range: { auto: { high: 2 } },
+            }).toContainExactly([{ auto: 1 }, { auto: 2 }]))
+      })
+
+      isSqs &&
+        describe("sort", () => {
+          it("sorts ascending", () =>
+            expectSearch({
+              query: {},
+              sort: "auto",
+              sortOrder: SortOrder.ASCENDING,
+            }).toMatchExactly([
+              { auto: 1 },
+              { auto: 2 },
+              { auto: 3 },
+              { auto: 4 },
+              { auto: 5 },
+              { auto: 6 },
+              { auto: 7 },
+              { auto: 8 },
+              { auto: 9 },
+              { auto: 10 },
+            ]))
+
+          it("sorts descending", () =>
+            expectSearch({
+              query: {},
+              sort: "auto",
+              sortOrder: SortOrder.DESCENDING,
+            }).toMatchExactly([
+              { auto: 10 },
+              { auto: 9 },
+              { auto: 8 },
+              { auto: 7 },
+              { auto: 6 },
+              { auto: 5 },
+              { auto: 4 },
+              { auto: 3 },
+              { auto: 2 },
+              { auto: 1 },
+            ]))
+        })
+    })
 })

packages/server/src/api/routes/tests/static.spec.js

@@ -1,3 +1,13 @@
+// Directly mock the AWS SDK
+jest.mock("aws-sdk", () => ({
+  S3: jest.fn(() => ({
+    getSignedUrl: jest.fn(
+      (operation, params) => `http://example.com/${params.Bucket}/${params.Key}`
+    ),
+    upload: jest.fn(() => ({ Contents: {} })),
+  })),
+}))
+
 const setup = require("./utilities")
 const { constants } = require("@budibase/backend-core")
 

packages/server/src/api/routes/tests/table.spec.ts

@@ -1,8 +1,8 @@
 import { context, events } from "@budibase/backend-core"
 import {
   AutoFieldSubType,
-  Datasource,
   BBReferenceFieldSubType,
+  Datasource,
   FieldType,
   INTERNAL_TABLE_SOURCE_ID,
   InternalTable,
@@ -149,58 +149,59 @@ describe.each([
       expect(res.name).toBeUndefined()
     })
 
-    it("updates only the passed fields", async () => {
+    isInternal &&
-      await timekeeper.withFreeze(new Date(2021, 1, 1), async () => {
+      it("updates only the passed fields", async () => {
-        const table = await config.api.table.save(
+        await timekeeper.withFreeze(new Date(2021, 1, 1), async () => {
-          tableForDatasource(datasource, {
+          const table = await config.api.table.save(
-            schema: {
+            tableForDatasource(datasource, {
-              autoId: {
+              schema: {
-                name: "id",
+                autoId: {
-                type: FieldType.NUMBER,
+                  name: "id",
-                subtype: AutoFieldSubType.AUTO_ID,
+                  type: FieldType.NUMBER,
-                autocolumn: true,
+                  subtype: AutoFieldSubType.AUTO_ID,
-                constraints: {
+                  autocolumn: true,
-                  type: "number",
+                  constraints: {
-                  presence: false,
+                    type: "number",
+                    presence: false,
+                  },
                 },
               },
-            },
+            })
+          )
+
+          const newName = generator.guid()
+
+          const updatedTable = await config.api.table.save({
+            ...table,
+            name: newName,
           })
-        )
 
-        const newName = generator.guid()
+          let expected: Table = {
+            ...table,
+            name: newName,
+            _id: expect.any(String),
+          }
+          if (isInternal) {
+            expected._rev = expect.stringMatching(/^2-.+/)
+          }
 
-        const updatedTable = await config.api.table.save({
+          expect(updatedTable).toEqual(expected)
-          ...table,
+
-          name: newName,
+          const persistedTable = await config.api.table.get(updatedTable._id!)
+          expected = {
+            ...table,
+            name: newName,
+            _id: updatedTable._id,
+          }
+          if (datasource?.isSQL) {
+            expected.sql = true
+          }
+          if (isInternal) {
+            expected._rev = expect.stringMatching(/^2-.+/)
+          }
+          expect(persistedTable).toEqual(expected)
         })
-
-        let expected: Table = {
-          ...table,
-          name: newName,
-          _id: expect.any(String),
-        }
-        if (isInternal) {
-          expected._rev = expect.stringMatching(/^2-.+/)
-        }
-
-        expect(updatedTable).toEqual(expected)
-
-        const persistedTable = await config.api.table.get(updatedTable._id!)
-        expected = {
-          ...table,
-          name: newName,
-          _id: updatedTable._id,
-        }
-        if (datasource?.isSQL) {
-          expected.sql = true
-        }
-        if (isInternal) {
-          expected._rev = expect.stringMatching(/^2-.+/)
-        }
-        expect(persistedTable).toEqual(expected)
       })
-    })
 
     describe("user table", () => {
       isInternal &&
@@ -214,6 +215,57 @@ describe.each([
         })
     })
 
+    describe("external table validation", () => {
+      !isInternal &&
+        it("should error if column is of type auto", async () => {
+          const table = basicTable(datasource)
+          await config.api.table.save(
+            {
+              ...table,
+              schema: {
+                ...table.schema,
+                auto: {
+                  name: "auto",
+                  autocolumn: true,
+                  type: FieldType.AUTO,
+                },
+              },
+            },
+            {
+              status: 400,
+              body: {
+                message: `Column "auto" has type "${FieldType.AUTO}" - this is not supported.`,
+              },
+            }
+          )
+        })
+
+      !isInternal &&
+        it("should error if column has auto subtype", async () => {
+          const table = basicTable(datasource)
+          await config.api.table.save(
+            {
+              ...table,
+              schema: {
+                ...table.schema,
+                auto: {
+                  name: "auto",
+                  autocolumn: true,
+                  type: FieldType.NUMBER,
+                  subtype: AutoFieldSubType.AUTO_ID,
+                },
+              },
+            },
+            {
+              status: 400,
+              body: {
+                message: `Column "auto" has subtype "${AutoFieldSubType.AUTO_ID}" - this is not supported.`,
+              },
+            }
+          )
+        })
+    })
+
     it("should add a new column for an internal DB table", async () => {
       const saveTableRequest: SaveTableRequest = {
         ...basicTable(),

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -24,8 +24,6 @@ import merge from "lodash/merge"
 import { quotas } from "@budibase/pro"
 import { roles } from "@budibase/backend-core"
 
-jest.unmock("mssql")
-
 describe.each([
   ["internal", undefined],
   [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],

packages/server/src/automations/automationUtils.ts

@@ -4,8 +4,11 @@ import {
   encodeJSBinding,
 } from "@budibase/string-templates"
 import sdk from "../sdk"
-import { Row } from "@budibase/types"
+import { AutomationAttachment, FieldType, Row } from "@budibase/types"
 import { LoopInput, LoopStepType } from "../definitions/automations"
+import { objectStore, context } from "@budibase/backend-core"
+import * as uuid from "uuid"
+import path from "path"
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -96,6 +99,98 @@ export function getError(err: any) {
   return typeof err !== "string" ? err.toString() : err
 }
 
+export async function sendAutomationAttachmentsToStorage(
+  tableId: string,
+  row: Row
+): Promise<Row> {
+  const table = await sdk.tables.getTable(tableId)
+  const attachmentRows: Record<
+    string,
+    AutomationAttachment[] | AutomationAttachment
+  > = {}
+
+  for (const [prop, value] of Object.entries(row)) {
+    const schema = table.schema[prop]
+    if (
+      schema?.type === FieldType.ATTACHMENTS ||
+      schema?.type === FieldType.ATTACHMENT_SINGLE
+    ) {
+      attachmentRows[prop] = value
+    }
+  }
+  for (const [prop, attachments] of Object.entries(attachmentRows)) {
+    if (Array.isArray(attachments)) {
+      if (attachments.length) {
+        row[prop] = await Promise.all(
+          attachments.map(attachment => generateAttachmentRow(attachment))
+        )
+      }
+    } else if (Object.keys(row[prop]).length > 0) {
+      row[prop] = await generateAttachmentRow(attachments)
+    }
+  }
+
+  return row
+}
+
+async function generateAttachmentRow(attachment: AutomationAttachment) {
+  const prodAppId = context.getProdAppId()
+
+  async function uploadToS3(
+    extension: string,
+    content: objectStore.StreamTypes
+  ) {
+    const fileName = `${uuid.v4()}${extension}`
+    const s3Key = `${prodAppId}/attachments/${fileName}`
+
+    await objectStore.streamUpload({
+      bucket: objectStore.ObjectStoreBuckets.APPS,
+      stream: content,
+      filename: s3Key,
+    })
+
+    return s3Key
+  }
+
+  async function getSize(s3Key: string) {
+    return (
+      await objectStore.getObjectMetadata(
+        objectStore.ObjectStoreBuckets.APPS,
+        s3Key
+      )
+    ).ContentLength
+  }
+
+  try {
+    const { filename } = attachment
+    const extension = path.extname(filename)
+    const attachmentResult = await objectStore.processAutomationAttachment(
+      attachment
+    )
+
+    let s3Key = ""
+    if (
+      "path" in attachmentResult &&
+      attachmentResult.path.startsWith(`${prodAppId}/attachments/`)
+    ) {
+      s3Key = attachmentResult.path
+    } else {
+      s3Key = await uploadToS3(extension, attachmentResult.content)
+    }
+
+    const size = await getSize(s3Key)
+
+    return {
+      size,
+      name: filename,
+      extension,
+      key: s3Key,
+    }
+  } catch (error) {
+    console.error("Failed to process attachment:", error)
+    throw error
+  }
+}
 export function substituteLoopStep(hbsString: string, substitute: string) {
   let checkForJS = isJSBinding(hbsString)
   let substitutedHbsString = ""

packages/server/src/automations/steps/createRow.ts

@@ -1,5 +1,9 @@
 import { save } from "../../api/controllers/row"
-import { cleanUpRow, getError } from "../automationUtils"
+import {
+  cleanUpRow,
+  getError,
+  sendAutomationAttachmentsToStorage,
+} from "../automationUtils"
 import { buildCtx } from "./utils"
 import {
   AutomationActionStepId,
@@ -89,6 +93,10 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) {
 
   try {
     inputs.row = await cleanUpRow(inputs.row.tableId, inputs.row)
+    inputs.row = await sendAutomationAttachmentsToStorage(
+      inputs.row.tableId,
+      inputs.row
+    )
     await save(ctx)
     return {
       row: inputs.row,

packages/server/src/automations/steps/updateRow.ts

@@ -108,7 +108,15 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) {
 
   try {
     if (tableId) {
-      inputs.row = await automationUtils.cleanUpRow(tableId, inputs.row)
+      inputs.row = await automationUtils.cleanUpRow(
+        inputs.row.tableId,
+        inputs.row
+      )
+
+      inputs.row = await automationUtils.sendAutomationAttachmentsToStorage(
+        inputs.row.tableId,
+        inputs.row
+      )
     }
     await rowController.patch(ctx)
     return {

packages/server/src/automations/tests/createRow.spec.ts

@@ -1,5 +1,18 @@
 import * as setup from "./utilities"
+import { basicTableWithAttachmentField } from "../../tests/utilities/structures"
+import { objectStore } from "@budibase/backend-core"
 
+async function uploadTestFile(filename: string) {
+  let bucket = "testbucket"
+  await objectStore.upload({
+    bucket,
+    filename,
+    body: Buffer.from("test data"),
+  })
+  let presignedUrl = await objectStore.getPresignedUrl(bucket, filename, 60000)
+
+  return presignedUrl
+}
 describe("test the create row action", () => {
   let table: any
   let row: any
@@ -43,4 +56,76 @@ describe("test the create row action", () => {
     const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {})
     expect(res.success).toEqual(false)
   })
+
+  it("should check that an attachment field is sent to storage and parsed", async () => {
+    let attachmentTable = await config.createTable(
+      basicTableWithAttachmentField()
+    )
+
+    let attachmentRow: any = {
+      tableId: attachmentTable._id,
+    }
+
+    let filename = "test1.txt"
+    let presignedUrl = await uploadTestFile(filename)
+    let attachmentObject = [
+      {
+        url: presignedUrl,
+        filename,
+      },
+    ]
+
+    attachmentRow.file_attachment = attachmentObject
+    const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {
+      row: attachmentRow,
+    })
+
+    expect(res.success).toEqual(true)
+    expect(res.row.file_attachment[0]).toHaveProperty("key")
+    let s3Key = res.row.file_attachment[0].key
+
+    const client = objectStore.ObjectStore(objectStore.ObjectStoreBuckets.APPS)
+
+    const objectData = await client
+      .headObject({ Bucket: objectStore.ObjectStoreBuckets.APPS, Key: s3Key })
+      .promise()
+
+    expect(objectData).toBeDefined()
+    expect(objectData.ContentLength).toBeGreaterThan(0)
+  })
+
+  it("should check that an single attachment field is sent to storage and parsed", async () => {
+    let attachmentTable = await config.createTable(
+      basicTableWithAttachmentField()
+    )
+
+    let attachmentRow: any = {
+      tableId: attachmentTable._id,
+    }
+
+    let filename = "test2.txt"
+    let presignedUrl = await uploadTestFile(filename)
+    let attachmentObject = {
+      url: presignedUrl,
+      filename,
+    }
+
+    attachmentRow.single_file_attachment = attachmentObject
+    const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {
+      row: attachmentRow,
+    })
+
+    expect(res.success).toEqual(true)
+    expect(res.row.single_file_attachment).toHaveProperty("key")
+    let s3Key = res.row.single_file_attachment.key
+
+    const client = objectStore.ObjectStore(objectStore.ObjectStoreBuckets.APPS)
+
+    const objectData = await client
+      .headObject({ Bucket: objectStore.ObjectStoreBuckets.APPS, Key: s3Key })
+      .promise()
+
+    expect(objectData).toBeDefined()
+    expect(objectData.ContentLength).toBeGreaterThan(0)
+  })
 })

packages/server/src/automations/tests/queryRows.spec.ts

@@ -1,14 +1,4 @@
-// lucene searching not supported in test due to use of PouchDB
+import { Table } from "@budibase/types"
-let rows: Row[] = []
-jest.mock("../../sdk/app/rows/search/internalSearch", () => ({
-  fullSearch: jest.fn(() => {
-    return {
-      rows,
-    }
-  }),
-  paginatedSearch: jest.fn(),
-}))
-import { Row, Table } from "@budibase/types"
 import * as setup from "./utilities"
 
 const NAME = "Test"
@@ -25,8 +15,8 @@ describe("Test a query step automation", () => {
       description: "original description",
       tableId: table._id,
     }
-    rows.push(await config.createRow(row))
+    await config.createRow(row)
-    rows.push(await config.createRow(row))
+    await config.createRow(row)
   })
 
   afterAll(setup.afterAll)

packages/server/src/integration-test/postgres.spec.ts

@@ -1200,4 +1200,38 @@ describe("postgres integrations", () => {
       expect(Object.keys(schema).sort()).toEqual(["id", "val1"])
     })
   })
+
+  describe("check custom column types", () => {
+    beforeAll(async () => {
+      await rawQuery(
+        rawDatasource,
+        `CREATE TABLE binaryTable (
+          id BYTEA PRIMARY KEY,
+          column1 TEXT,
+          column2 INT
+        );
+      `
+      )
+    })
+
+    it("should handle binary columns", async () => {
+      const response = await makeRequest(
+        "post",
+        `/api/datasources/${datasource._id}/schema`
+      )
+      expect(response.body).toBeDefined()
+      expect(response.body.datasource.entities).toBeDefined()
+      const table = response.body.datasource.entities["binarytable"]
+      expect(table).toBeDefined()
+      expect(table.schema.id.externalType).toBe("bytea")
+      const row = await config.api.row.save(table._id, {
+        id: "1111",
+        column1: "hello",
+        column2: 222,
+      })
+      expect(row._id).toBeDefined()
+      const decoded = decodeURIComponent(row._id!).replace(/'/g, '"')
+      expect(JSON.parse(decoded)[0]).toBe("1111")
+    })
+  })
 })

packages/server/src/integrations/base/sql.ts

@@ -150,6 +150,22 @@ function getTableName(table?: Table): string | undefined {
   }
 }
 
+function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] {
+  if (Array.isArray(query)) {
+    return query.map((q: SqlQuery) => convertBooleans(q) as SqlQuery)
+  } else {
+    if (query.bindings) {
+      query.bindings = query.bindings.map(binding => {
+        if (typeof binding === "boolean") {
+          return binding ? 1 : 0
+        }
+        return binding
+      })
+    }
+  }
+  return query
+}
+
 class InternalBuilder {
   private readonly client: string
 
@@ -654,7 +670,11 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     if (opts?.disableBindings) {
       return { sql: query.toString() }
     } else {
-      return getNativeSql(query)
+      let native = getNativeSql(query)
+      if (sqlClient === SqlClient.SQL_LITE) {
+        native = convertBooleans(native)
+      }
+      return native
     }
   }
 

packages/server/src/integrations/rest.ts

@@ -26,13 +26,13 @@ import { parse } from "content-disposition"
 import path from "path"
 import { Builder as XmlBuilder } from "xml2js"
 
-const BodyTypes = {
+enum BodyType {
-  NONE: "none",
+  NONE = "none",
-  FORM_DATA: "form",
+  FORM_DATA = "form",
-  XML: "xml",
+  XML = "xml",
-  ENCODED: "encoded",
+  ENCODED = "encoded",
-  JSON: "json",
+  JSON = "json",
-  TEXT: "text",
+  TEXT = "text",
 }
 
 const coreFields = {
@@ -54,7 +54,7 @@ const coreFields = {
   },
   bodyType: {
     type: DatasourceFieldType.STRING,
-    enum: Object.values(BodyTypes),
+    enum: Object.values(BodyType),
   },
   pagination: {
     type: DatasourceFieldType.OBJECT,
@@ -131,25 +131,28 @@ class RestIntegration implements IntegrationBase {
   }
 
   async parseResponse(response: any, pagination: PaginationConfig | null) {
-    let data, raw, headers, filename
+    let data: any[] | string | undefined,
+      raw: string | undefined,
+      headers: Record<string, string> = {},
+      filename: string | undefined
 
     const contentType = response.headers.get("content-type") || ""
     const contentDisposition = response.headers.get("content-disposition") || ""
     if (
+      contentDisposition.includes("filename") ||
       contentDisposition.includes("attachment") ||
       contentDisposition.includes("form-data")
     ) {
       filename =
         path.basename(parse(contentDisposition).parameters?.filename) || ""
     }
-
     try {
       if (filename) {
         return handleFileResponse(response, filename, this.startTimeMs)
       } else {
         if (response.status === 204) {
           data = []
-          raw = []
+          raw = ""
         } else if (contentType.includes("application/json")) {
           data = await response.json()
           raw = JSON.stringify(data)
@@ -162,16 +165,18 @@ class RestIntegration implements IntegrationBase {
           raw = xmlResponse.rawXml
         } else {
           data = await response.text()
-          raw = data
+          raw = data as string
         }
       }
     } catch (err) {
       throw `Failed to parse response body: ${err}`
     }
 
-    const size = formatBytes(
+    let contentLength: string = response.headers.get("content-length")
-      response.headers.get("content-length") || Buffer.byteLength(raw, "utf8")
+    if (!contentLength && raw) {
-    )
+      contentLength = Buffer.byteLength(raw, "utf8").toString()
+    }
+    const size = formatBytes(contentLength || "0")
     const time = `${Math.round(performance.now() - this.startTimeMs)}ms`
     headers = response.headers.raw()
     for (let [key, value] of Object.entries(headers)) {
@@ -255,7 +260,7 @@ class RestIntegration implements IntegrationBase {
     if (!input.headers) {
       input.headers = {}
     }
-    if (bodyType === BodyTypes.NONE) {
+    if (bodyType === BodyType.NONE) {
       return input
     }
     let error,
@@ -283,11 +288,11 @@ class RestIntegration implements IntegrationBase {
     }
 
     switch (bodyType) {
-      case BodyTypes.TEXT:
+      case BodyType.TEXT:
         // content type defaults to plaintext
         input.body = string
         break
-      case BodyTypes.ENCODED: {
+      case BodyType.ENCODED: {
         const params = new URLSearchParams()
         for (let [key, value] of Object.entries(object)) {
           params.append(key, value as string)
@@ -298,7 +303,7 @@ class RestIntegration implements IntegrationBase {
         input.body = params
         break
       }
-      case BodyTypes.FORM_DATA: {
+      case BodyType.FORM_DATA: {
         const form = new FormData()
         for (let [key, value] of Object.entries(object)) {
           form.append(key, value)
@@ -309,14 +314,14 @@ class RestIntegration implements IntegrationBase {
         input.body = form
         break
       }
-      case BodyTypes.XML:
+      case BodyType.XML:
         if (object != null && Object.keys(object).length) {
           string = new XmlBuilder().buildObject(object)
         }
         input.body = string
         input.headers["Content-Type"] = "application/xml"
         break
-      case BodyTypes.JSON:
+      case BodyType.JSON:
         // if JSON error, throw it
         if (error) {
           throw "Invalid JSON for request body"

packages/server/src/integrations/tests/aws-sdk.mock.ts

@@ -1,6 +1,3 @@
-import fs from "fs"
-import { join } from "path"
-
 const response = (body: any, extra?: any) => () => ({
   promise: () => body,
   ...extra,
@@ -62,9 +59,7 @@ class S3 {
         Body: "",
       },
       {
-        createReadStream: jest
+        createReadStream: jest.fn().mockReturnValue("stream"),
-          .fn()
-          .mockReturnValue(fs.createReadStream(join(__dirname, "aws-sdk.ts"))),
       }
     )
   )

packages/server/src/integrations/tests/dynamodb.spec.ts

@@ -1,7 +1,6 @@
+jest.mock("aws-sdk", () => require("./aws-sdk.mock"))
 import { default as DynamoDBIntegration } from "../dynamodb"
 
-jest.mock("aws-sdk")
-
 class TestConfiguration {
   integration: any
 

packages/server/src/integrations/tests/rest.spec.ts

@@ -28,7 +28,6 @@ jest.mock("uuid", () => ({ v4: () => "00000000-0000-0000-0000-000000000000" }))
 import { default as RestIntegration } from "../rest"
 import { RestAuthType } from "@budibase/types"
 import fetch from "node-fetch"
-import { objectStoreTestProviders } from "@budibase/backend-core/tests"
 import { Readable } from "stream"
 
 const FormData = require("form-data")
@@ -246,13 +245,13 @@ describe("REST Integration", () => {
       expect(output.extra.headers["content-type"]).toEqual("application/xml")
     })
 
-    test.each(contentTypes)(
+    test.each([...contentTypes, undefined])(
       "should not throw an error on 204 no content",
       async contentType => {
         const input = buildInput(undefined, null, contentType, 204)
         const output = await config.integration.parseResponse(input)
         expect(output.data).toEqual([])
-        expect(output.extra.raw).toEqual([])
+        expect(output.extra.raw).toEqual("")
         expect(output.info.code).toEqual(204)
         expect(output.extra.headers["content-type"]).toEqual(contentType)
       }
@@ -627,15 +626,6 @@ describe("REST Integration", () => {
   })
 
   describe("File Handling", () => {
-    beforeAll(async () => {
-      jest.unmock("aws-sdk")
-      await objectStoreTestProviders.minio.start()
-    })
-
-    afterAll(async () => {
-      await objectStoreTestProviders.minio.stop()
-    })
-
     it("uploads file to object store and returns signed URL", async () => {
       const responseData = Buffer.from("teest file contnt")
       const filename = "test.tar.gz"

packages/server/src/integrations/tests/s3.spec.ts

@@ -1,7 +1,6 @@
+jest.mock("aws-sdk", () => require("./aws-sdk.mock"))
 import { default as S3Integration } from "../s3"
 
-jest.mock("aws-sdk")
-
 class TestConfiguration {
   integration: any
 

packages/server/src/integrations/utils/utils.ts

@@ -192,6 +192,11 @@ export function generateRowIdField(keyProps: any[] = []) {
   if (!Array.isArray(keyProps)) {
     keyProps = [keyProps]
   }
+  for (let index in keyProps) {
+    if (keyProps[index] instanceof Buffer) {
+      keyProps[index] = keyProps[index].toString()
+    }
+  }
   // this conserves order and types
   // we have to swap the double quotes to single quotes for use in HBS statements
   // when using the literal helper the double quotes can break things

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -131,11 +131,6 @@ export async function search(
     },
     relationships,
   }
-  // make sure only rows returned
-  request.filters!.equal = {
-    ...request.filters?.equal,
-    type: "row",
-  }
 
   if (params.sort) {
     const sortField = table.schema[params.sort]
@@ -169,8 +164,8 @@ export async function search(
         throw new Error("SQS cannot currently handle multiple queries")
       }
 
-      let sql = query.sql,
+      let sql = query.sql
-        bindings = query.bindings
+      let bindings = query.bindings
 
       // quick hack for docIds
       sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")

packages/server/src/sdk/app/rows/search/tests/internal.spec.ts

@@ -1,117 +0,0 @@
-import {
-  FieldType,
-  Row,
-  Table,
-  RowSearchParams,
-  INTERNAL_TABLE_SOURCE_ID,
-  TableSourceType,
-} from "@budibase/types"
-import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
-import { search } from "../internal"
-import {
-  expectAnyInternalColsAttributes,
-  generator,
-} from "@budibase/backend-core/tests"
-
-describe("internal", () => {
-  const config = new TestConfiguration()
-
-  const tableData: Table = {
-    name: generator.word(),
-    type: "table",
-    sourceId: INTERNAL_TABLE_SOURCE_ID,
-    sourceType: TableSourceType.INTERNAL,
-    schema: {
-      name: {
-        name: "name",
-        type: FieldType.STRING,
-        constraints: {
-          type: FieldType.STRING,
-        },
-      },
-      surname: {
-        name: "surname",
-        type: FieldType.STRING,
-        constraints: {
-          type: FieldType.STRING,
-        },
-      },
-      age: {
-        name: "age",
-        type: FieldType.NUMBER,
-        constraints: {
-          type: FieldType.NUMBER,
-        },
-      },
-      address: {
-        name: "address",
-        type: FieldType.STRING,
-        constraints: {
-          type: FieldType.STRING,
-        },
-      },
-    },
-  }
-
-  beforeAll(async () => {
-    await config.init()
-  })
-
-  describe("search", () => {
-    const rows: Row[] = []
-    beforeAll(async () => {
-      await config.createTable(tableData)
-      for (let i = 0; i < 10; i++) {
-        rows.push(
-          await config.createRow({
-            name: generator.first(),
-            surname: generator.last(),
-            age: generator.age(),
-            address: generator.address(),
-          })
-        )
-      }
-    })
-
-    it("default search returns all the data", async () => {
-      await config.doInContext(config.appId, async () => {
-        const tableId = config.table!._id!
-
-        const searchParams: RowSearchParams = {
-          tableId,
-          query: {},
-        }
-        const result = await search(searchParams, config.table!)
-
-        expect(result.rows).toHaveLength(10)
-        expect(result.rows).toEqual(
-          expect.arrayContaining(rows.map(r => expect.objectContaining(r)))
-        )
-      })
-    })
-
-    it("querying by fields will always return data attribute columns", async () => {
-      await config.doInContext(config.appId, async () => {
-        const tableId = config.table!._id!
-
-        const searchParams: RowSearchParams = {
-          tableId,
-          query: {},
-          fields: ["name", "age"],
-        }
-        const result = await search(searchParams, config.table!)
-
-        expect(result.rows).toHaveLength(10)
-        expect(result.rows).toEqual(
-          expect.arrayContaining(
-            rows.map(r => ({
-              ...expectAnyInternalColsAttributes,
-              name: r.name,
-              age: r.age,
-            }))
-          )
-        )
-      })
-    })
-  })
-})

packages/server/src/sdk/app/rows/tests/internal.spec.ts

@@ -1,228 +0,0 @@
-import tk from "timekeeper"
-import * as internalSdk from "../internal"
-
-import { generator } from "@budibase/backend-core/tests"
-import {
-  INTERNAL_TABLE_SOURCE_ID,
-  TableSourceType,
-  FieldType,
-  Table,
-  AutoFieldSubType,
-  AutoColumnFieldMetadata,
-} from "@budibase/types"
-
-import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
-
-tk.freeze(Date.now())
-
-describe("sdk >> rows >> internal", () => {
-  const config = new TestConfiguration()
-
-  beforeAll(async () => {
-    await config.init()
-  })
-
-  function makeRow() {
-    return {
-      name: generator.first(),
-      surname: generator.last(),
-      age: generator.age(),
-      address: generator.address(),
-    }
-  }
-
-  describe("save", () => {
-    const tableData: Table = {
-      name: generator.word(),
-      type: "table",
-      sourceId: INTERNAL_TABLE_SOURCE_ID,
-      sourceType: TableSourceType.INTERNAL,
-      schema: {
-        name: {
-          name: "name",
-          type: FieldType.STRING,
-          constraints: {
-            type: FieldType.STRING,
-          },
-        },
-        surname: {
-          name: "surname",
-          type: FieldType.STRING,
-          constraints: {
-            type: FieldType.STRING,
-          },
-        },
-        age: {
-          name: "age",
-          type: FieldType.NUMBER,
-          constraints: {
-            type: FieldType.NUMBER,
-          },
-        },
-        address: {
-          name: "address",
-          type: FieldType.STRING,
-          constraints: {
-            type: FieldType.STRING,
-          },
-        },
-      },
-    }
-
-    beforeEach(() => {
-      jest.clearAllMocks()
-    })
-
-    it("save will persist the row properly", async () => {
-      const table = await config.createTable(tableData)
-      const row = makeRow()
-
-      await config.doInContext(config.appId, async () => {
-        const response = await internalSdk.save(
-          table._id!,
-          row,
-          config.getUser()._id
-        )
-
-        expect(response).toEqual({
-          table,
-          row: {
-            ...row,
-            type: "row",
-            _rev: expect.stringMatching("1-.*"),
-          },
-          squashed: {
-            ...row,
-            type: "row",
-            _rev: expect.stringMatching("1-.*"),
-          },
-        })
-
-        const persistedRow = await config.api.row.get(
-          table._id!,
-          response.row._id!
-        )
-        expect(persistedRow).toEqual({
-          ...row,
-          type: "row",
-          _rev: expect.stringMatching("1-.*"),
-          createdAt: expect.any(String),
-          updatedAt: expect.any(String),
-        })
-      })
-    })
-
-    it("auto ids will update when creating new rows", async () => {
-      const table = await config.createTable({
-        ...tableData,
-        schema: {
-          ...tableData.schema,
-          id: {
-            name: "id",
-            type: FieldType.AUTO,
-            subtype: AutoFieldSubType.AUTO_ID,
-            autocolumn: true,
-            lastID: 0,
-          },
-        },
-      })
-      const row = makeRow()
-
-      await config.doInContext(config.appId, async () => {
-        const response = await internalSdk.save(
-          table._id!,
-          row,
-          config.getUser()._id
-        )
-
-        expect(response).toEqual({
-          table: {
-            ...table,
-            schema: {
-              ...table.schema,
-              id: {
-                ...table.schema.id,
-                lastID: 1,
-              },
-            },
-          },
-          row: {
-            ...row,
-            id: 1,
-            type: "row",
-            _rev: expect.stringMatching("1-.*"),
-          },
-          squashed: {
-            ...row,
-            id: 1,
-            type: "row",
-            _rev: expect.stringMatching("1-.*"),
-          },
-        })
-
-        const persistedRow = await config.api.row.get(
-          table._id!,
-          response.row._id!
-        )
-        expect(persistedRow).toEqual({
-          ...row,
-          type: "row",
-          id: 1,
-          _rev: expect.stringMatching("1-.*"),
-          createdAt: expect.any(String),
-          updatedAt: expect.any(String),
-        })
-      })
-    })
-
-    it("auto ids will update when creating new rows in parallel", async () => {
-      function makeRows(count: number) {
-        return Array.from({ length: count }, () => makeRow())
-      }
-
-      const table = await config.createTable({
-        ...tableData,
-        schema: {
-          ...tableData.schema,
-          id: {
-            name: "id",
-            type: FieldType.AUTO,
-            subtype: AutoFieldSubType.AUTO_ID,
-            autocolumn: true,
-            lastID: 0,
-          },
-        },
-      })
-
-      await config.doInContext(config.appId, async () => {
-        for (const row of makeRows(5)) {
-          await internalSdk.save(table._id!, row, config.getUser()._id)
-        }
-        await Promise.all(
-          makeRows(10).map(row =>
-            internalSdk.save(table._id!, row, config.getUser()._id)
-          )
-        )
-        for (const row of makeRows(5)) {
-          await internalSdk.save(table._id!, row, config.getUser()._id)
-        }
-      })
-
-      const persistedRows = await config.getRows(table._id!)
-      expect(persistedRows).toHaveLength(20)
-      expect(persistedRows).toEqual(
-        expect.arrayContaining(
-          Array.from({ length: 20 }).map((_, i) =>
-            expect.objectContaining({ id: i + 1 })
-          )
-        )
-      )
-
-      const persistedTable = await config.getTable(table._id)
-      expect((table.schema.id as AutoColumnFieldMetadata).lastID).toBe(0)
-      expect((persistedTable.schema.id as AutoColumnFieldMetadata).lastID).toBe(
-        20
-      )
-    })
-  })
-})

packages/server/src/sdk/app/tables/external/index.ts

@@ -6,6 +6,7 @@ import {
   Table,
   TableRequest,
   ViewV2,
+  AutoFieldSubType,
 } from "@budibase/types"
 import { context } from "@budibase/backend-core"
 import { buildExternalTableId } from "../../../../integrations/utils"
@@ -29,6 +30,52 @@ import { populateExternalTableSchemas } from "../validation"
 import datasourceSdk from "../../datasources"
 import * as viewSdk from "../../views"
 
+const DEFAULT_PRIMARY_COLUMN = "id"
+
+function noPrimaryKey(table: Table) {
+  return table.primary == null || table.primary.length === 0
+}
+
+function validate(table: Table, oldTable?: Table) {
+  if (
+    !oldTable &&
+    table.schema[DEFAULT_PRIMARY_COLUMN] &&
+    noPrimaryKey(table)
+  ) {
+    throw new Error(
+      "External tables with no `primary` column set will define an `id` column, but we found an `id` column in the supplied schema. Either set a `primary` column or remove the `id` column."
+    )
+  }
+
+  if (hasTypeChanged(table, oldTable)) {
+    throw new Error("A column type has changed.")
+  }
+
+  const autoSubTypes = Object.values(AutoFieldSubType)
+  // check for auto columns, they are not allowed
+  for (let [key, column] of Object.entries(table.schema)) {
+    // this column is a special case, do not validate it
+    if (key === DEFAULT_PRIMARY_COLUMN) {
+      continue
+    }
+    // the auto-column type should never be used
+    if (column.type === FieldType.AUTO) {
+      throw new Error(
+        `Column "${key}" has type "${FieldType.AUTO}" - this is not supported.`
+      )
+    }
+
+    if (
+      column.subtype &&
+      autoSubTypes.includes(column.subtype as AutoFieldSubType)
+    ) {
+      throw new Error(
+        `Column "${key}" has subtype "${column.subtype}" - this is not supported.`
+      )
+    }
+  }
+}
+
 export async function save(
   datasourceId: string,
   update: Table,
@@ -47,28 +94,18 @@ export async function save(
     oldTable = await getTable(tableId)
   }
 
-  if (
+  // this will throw an error if something is wrong
-    !oldTable &&
+  validate(tableToSave, oldTable)
-    (tableToSave.primary == null || tableToSave.primary.length === 0)
-  ) {
-    if (tableToSave.schema.id) {
-      throw new Error(
-        "External tables with no `primary` column set will define an `id` column, but we found an `id` column in the supplied schema. Either set a `primary` column or remove the `id` column."
-      )
-    }
 
-    tableToSave.primary = ["id"]
+  if (!oldTable && noPrimaryKey(tableToSave)) {
-    tableToSave.schema.id = {
+    tableToSave.primary = [DEFAULT_PRIMARY_COLUMN]
+    tableToSave.schema[DEFAULT_PRIMARY_COLUMN] = {
       type: FieldType.NUMBER,
       autocolumn: true,
-      name: "id",
+      name: DEFAULT_PRIMARY_COLUMN,
     }
   }
 
-  if (hasTypeChanged(tableToSave, oldTable)) {
-    throw new Error("A column type has changed.")
-  }
-
   for (let view in tableToSave.views) {
     const tableView = tableToSave.views[view]
     if (!tableView || !viewSdk.isV2(tableView)) continue

packages/server/src/sdk/app/tables/internal/sqs.ts

@@ -33,7 +33,7 @@ const FieldTypeMap: Record<FieldType, SQLiteType> = {
   [FieldType.LONGFORM]: SQLiteType.TEXT,
   [FieldType.NUMBER]: SQLiteType.REAL,
   [FieldType.STRING]: SQLiteType.TEXT,
-  [FieldType.AUTO]: SQLiteType.TEXT,
+  [FieldType.AUTO]: SQLiteType.REAL,
   [FieldType.OPTIONS]: SQLiteType.TEXT,
   [FieldType.JSON]: SQLiteType.BLOB,
   [FieldType.INTERNAL]: SQLiteType.BLOB,