Merge branch 'master' of github.com:Budibase/budibase into feature/sqs-table-cleanup

packages/backend-core/src/context/mainContext.ts

@@ -281,7 +281,7 @@ export function doInScimContext(task: any) {
   return newContext(updates, task)
 }
 
-export async function ensureSnippetContext() {
+export async function ensureSnippetContext(enabled = !env.isTest()) {
   const ctx = getCurrentContext()
 
   // If we've already added snippets to context, continue
@@ -292,7 +292,7 @@ export async function ensureSnippetContext() {
   // Otherwise get snippets for this app and update context
   let snippets: Snippet[] | undefined
   const db = getAppDB()
-  if (db && !env.isTest()) {
+  if (db && enabled) {
     const app = await db.get<App>(DocumentType.APP_METADATA)
     snippets = app.snippets
   }

packages/backend-core/src/objectStore/objectStore.ts

@@ -13,13 +13,14 @@ import { bucketTTLConfig, budibaseTempDir } from "./utils"
 import { v4 } from "uuid"
 import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
 import fsp from "fs/promises"
+import { HeadObjectOutput } from "aws-sdk/clients/s3"
 
 const streamPipeline = promisify(stream.pipeline)
 // use this as a temporary store of buckets that are being created
 const STATE = {
   bucketCreationPromises: {},
 }
-const signedFilePrefix = "/files/signed"
+export const SIGNED_FILE_PREFIX = "/files/signed"
 
 type ListParams = {
   ContinuationToken?: string
@@ -40,8 +41,13 @@ type UploadParams = BaseUploadParams & {
   path?: string | PathLike
 }
 
-type StreamUploadParams = BaseUploadParams & {
+export type StreamTypes =
-  stream: ReadStream
+  | ReadStream
+  | NodeJS.ReadableStream
+  | ReadableStream<Uint8Array>
+
+export type StreamUploadParams = BaseUploadParams & {
+  stream?: StreamTypes
 }
 
 const CONTENT_TYPE_MAP: any = {
@@ -174,11 +180,9 @@ export async function upload({
   const objectStore = ObjectStore(bucketName)
   const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
 
-  if (ttl && (bucketCreated.created || bucketCreated.exists)) {
+  if (ttl && bucketCreated.created) {
     let ttlConfig = bucketTTLConfig(bucketName, ttl)
-    if (objectStore.putBucketLifecycleConfiguration) {
+    await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
-      await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
-    }
   }
 
   let contentType = type
@@ -222,11 +226,9 @@ export async function streamUpload({
   const objectStore = ObjectStore(bucketName)
   const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
 
-  if (ttl && (bucketCreated.created || bucketCreated.exists)) {
+  if (ttl && bucketCreated.created) {
     let ttlConfig = bucketTTLConfig(bucketName, ttl)
-    if (objectStore.putBucketLifecycleConfiguration) {
+    await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
-      await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
-    }
   }
 
   // Set content type for certain known extensions
@@ -333,7 +335,7 @@ export function getPresignedUrl(
     const signedUrl = new URL(url)
     const path = signedUrl.pathname
     const query = signedUrl.search
-    return `${signedFilePrefix}${path}${query}`
+    return `${SIGNED_FILE_PREFIX}${path}${query}`
   }
 }
 
@@ -521,6 +523,26 @@ export async function getReadStream(
   return client.getObject(params).createReadStream()
 }
 
+export async function getObjectMetadata(
+  bucket: string,
+  path: string
+): Promise<HeadObjectOutput> {
+  bucket = sanitizeBucket(bucket)
+  path = sanitizeKey(path)
+
+  const client = ObjectStore(bucket)
+  const params = {
+    Bucket: bucket,
+    Key: path,
+  }
+
+  try {
+    return await client.headObject(params).promise()
+  } catch (err: any) {
+    throw new Error("Unable to retrieve metadata from object")
+  }
+}
+
 /*
 Given a signed url like '/files/signed/tmp-files-attachments/app_123456/myfile.txt' extract
 the bucket and the path from it
@@ -530,7 +552,9 @@ export function extractBucketAndPath(
 ): { bucket: string; path: string } | null {
   const baseUrl = url.split("?")[0]
 
-  const regex = new RegExp(`^${signedFilePrefix}/(?<bucket>[^/]+)/(?<path>.+)$`)
+  const regex = new RegExp(
+    `^${SIGNED_FILE_PREFIX}/(?<bucket>[^/]+)/(?<path>.+)$`
+  )
   const match = baseUrl.match(regex)
 
   if (match && match.groups) {

packages/backend-core/src/objectStore/utils.ts

@@ -1,9 +1,14 @@
-import { join } from "path"
+import path, { join } from "path"
 import { tmpdir } from "os"
 import fs from "fs"
 import env from "../environment"
 import { PutBucketLifecycleConfigurationRequest } from "aws-sdk/clients/s3"
-
+import * as objectStore from "./objectStore"
+import {
+  AutomationAttachment,
+  AutomationAttachmentContent,
+  BucketedContent,
+} from "@budibase/types"
 /****************************************************
  *      NOTE: When adding a new bucket - name       *
  *     sure that S3 usages (like budibase-infra)    *
@@ -55,3 +60,50 @@ export const bucketTTLConfig = (
 
   return params
 }
+
+async function processUrlAttachment(
+  attachment: AutomationAttachment
+): Promise<AutomationAttachmentContent> {
+  const response = await fetch(attachment.url)
+  if (!response.ok || !response.body) {
+    throw new Error(`Unexpected response ${response.statusText}`)
+  }
+  const fallbackFilename = path.basename(new URL(attachment.url).pathname)
+  return {
+    filename: attachment.filename || fallbackFilename,
+    content: response.body,
+  }
+}
+
+export async function processObjectStoreAttachment(
+  attachment: AutomationAttachment
+): Promise<BucketedContent> {
+  const result = objectStore.extractBucketAndPath(attachment.url)
+
+  if (result === null) {
+    throw new Error("Invalid signed URL")
+  }
+
+  const { bucket, path: objectPath } = result
+  const readStream = await objectStore.getReadStream(bucket, objectPath)
+  const fallbackFilename = path.basename(objectPath)
+  return {
+    bucket,
+    path: objectPath,
+    filename: attachment.filename || fallbackFilename,
+    content: readStream,
+  }
+}
+
+export async function processAutomationAttachment(
+  attachment: AutomationAttachment
+): Promise<AutomationAttachmentContent | BucketedContent> {
+  const isFullyFormedUrl =
+    attachment.url?.startsWith("http://") ||
+    attachment.url?.startsWith("https://")
+  if (isFullyFormedUrl) {
+    return await processUrlAttachment(attachment)
+  } else {
+    return await processObjectStoreAttachment(attachment)
+  }
+}

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -358,7 +358,8 @@
       value.customType !== "cron" &&
       value.customType !== "triggerSchema" &&
       value.customType !== "automationFields" &&
-      value.type !== "attachment"
+      value.type !== "attachment" &&
+      value.type !== "attachment_single"
     )
   }
 

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -2,6 +2,8 @@
   import { tables } from "stores/builder"
   import { Select, Checkbox, Label } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
+  import { FieldType } from "@budibase/types"
+
   import RowSelectorTypes from "./RowSelectorTypes.svelte"
   import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
@@ -14,7 +16,6 @@
   export let bindings
   export let isTestModal
   export let isUpdateRow
-
   $: parsedBindings = bindings.map(binding => {
     let clone = Object.assign({}, binding)
     clone.icon = "ShareAndroid"
@@ -26,15 +27,19 @@
 
   $: {
     table = $tables.list.find(table => table._id === value?.tableId)
-    schemaFields = Object.entries(table?.schema ?? {})
+
-    // surface the schema so the user can see it in the json
+    // Just sorting attachment types to the bottom here for a cleaner UX
-    schemaFields.map(([, schema]) => {
+    schemaFields = Object.entries(table?.schema ?? {}).sort(
+      ([, schemaA], [, schemaB]) =>
+        (schemaA.type === "attachment") - (schemaB.type === "attachment")
+    )
+
+    schemaFields.forEach(([, schema]) => {
       if (!schema.autocolumn && !value[schema.name]) {
         value[schema.name] = ""
       }
     })
   }
-
   const onChangeTable = e => {
     value["tableId"] = e.detail
     dispatch("change", value)
@@ -114,10 +119,16 @@
 </div>
 {#if schemaFields.length}
   {#each schemaFields as [field, schema]}
-    {#if !schema.autocolumn && schema.type !== "attachment"}
+    {#if !schema.autocolumn}
-      <div class="schema-fields">
+      <div
+        class:schema-fields={schema.type !== FieldType.ATTACHMENTS &&
+          schema.type !== FieldType.ATTACHMENT_SINGLE}
+      >
         <Label>{field}</Label>
-        <div class="field-width">
+        <div
+          class:field-width={schema.type !== FieldType.ATTACHMENTS &&
+            schema.type !== FieldType.ATTACHMENT_SINGLE}
+        >
           {#if isTestModal}
             <RowSelectorTypes
               {isTestModal}

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -1,10 +1,12 @@
 <script>
   import { Select, DatePicker, Multiselect, TextArea } from "@budibase/bbui"
+  import { FieldType } from "@budibase/types"
   import LinkedRowSelector from "components/common/LinkedRowSelector.svelte"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
   import ModalBindableInput from "../../common/bindings/ModalBindableInput.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import Editor from "components/integration/QueryEditor.svelte"
+  import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
 
   export let onChange
   export let field
@@ -22,6 +24,27 @@
   function schemaHasOptions(schema) {
     return !!schema.constraints?.inclusion?.length
   }
+
+  const handleAttachmentParams = keyValuObj => {
+    let params = {}
+
+    if (
+      schema.type === FieldType.ATTACHMENT_SINGLE &&
+      Object.keys(keyValuObj).length === 0
+    ) {
+      return []
+    }
+    if (!Array.isArray(keyValuObj)) {
+      keyValuObj = [keyValuObj]
+    }
+
+    if (keyValuObj.length) {
+      for (let param of keyValuObj) {
+        params[param.url] = param.filename
+      }
+    }
+    return params
+  }
 </script>
 
 {#if schemaHasOptions(schema) && schema.type !== "array"}
@@ -77,6 +100,35 @@
     on:change={e => onChange(e, field)}
     useLabel={false}
   />
+{:else if schema.type === FieldType.ATTACHMENTS || schema.type === FieldType.ATTACHMENT_SINGLE}
+  <div class="attachment-field-spacinng">
+    <KeyValueBuilder
+      on:change={e =>
+        onChange(
+          {
+            detail:
+              schema.type === FieldType.ATTACHMENT_SINGLE
+                ? e.detail.length > 0
+                  ? { url: e.detail[0].name, filename: e.detail[0].value }
+                  : {}
+                : e.detail.map(({ name, value }) => ({
+                    url: name,
+                    filename: value,
+                  })),
+          },
+          field
+        )}
+      object={handleAttachmentParams(value[field])}
+      allowJS
+      {bindings}
+      keyBindings
+      customButtonText={"Add attachment"}
+      keyPlaceholder={"URL"}
+      valuePlaceholder={"Filename"}
+      actionButtonDisabled={schema.type === FieldType.ATTACHMENT_SINGLE &&
+        Object.keys(value[field]).length >= 1}
+    />
+  </div>
 {:else if ["string", "number", "bigint", "barcodeqr", "array"].includes(schema.type)}
   <svelte:component
     this={isTestModal ? ModalBindableInput : DrawerBindableInput}
@@ -90,3 +142,10 @@
     title={schema.name}
   />
 {/if}
+
+<style>
+  .attachment-field-spacinng {
+    margin-top: var(--spacing-s);
+    margin-bottom: var(--spacing-l);
+  }
+</style>

packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte

@@ -1,7 +1,9 @@
 <script>
   import { createEventDispatcher } from "svelte"
-  import { ActionButton, Modal, ModalContent } from "@budibase/bbui"
+  import { ActionButton, Drawer, DrawerContent, Button } from "@budibase/bbui"
   import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte"
+  import { getUserBindings } from "dataBinding"
+  import { makePropSafe } from "@budibase/string-templates"
 
   export let schema
   export let filters
@@ -10,7 +12,7 @@
 
   const dispatch = createEventDispatcher()
 
-  let modal
+  let drawer
 
   $: tempValue = filters || []
   $: schemaFields = Object.entries(schema || {}).map(
@@ -22,37 +24,53 @@
 
   $: text = getText(filters)
   $: selected = tempValue.filter(x => !x.onEmptyFilter)?.length > 0
-
+  $: bindings = [
+    {
+      type: "context",
+      runtimeBinding: `${makePropSafe("now")}`,
+      readableBinding: `Date`,
+      category: "Date",
+      icon: "Date",
+      display: {
+        name: "Server date",
+      },
+    },
+    ...getUserBindings(),
+  ]
   const getText = filters => {
     const count = filters?.filter(filter => filter.field)?.length
     return count ? `Filter (${count})` : "Filter"
   }
 </script>
 
-<ActionButton icon="Filter" quiet {disabled} on:click={modal.show} {selected}>
+<ActionButton icon="Filter" quiet {disabled} on:click={drawer.show} {selected}>
   {text}
 </ActionButton>
-<Modal bind:this={modal}>
-  <ModalContent
-    title="Filter"
-    confirmText="Save"
-    size="XL"
-    onConfirm={() => dispatch("change", tempValue)}
-  >
-    <div class="wrapper">
-      <FilterBuilder
-        allowBindings={false}
-        {filters}
-        {schemaFields}
-        datasource={{ type: "table", tableId }}
-        on:change={e => (tempValue = e.detail)}
-      />
-    </div>
-  </ModalContent>
-</Modal>
 
-<style>
+<Drawer
-  .wrapper :global(.main) {
+  bind:this={drawer}
-    padding: 0;
+  title="Filtering"
-  }
+  on:drawerHide
-</style>
+  on:drawerShow
+  forceModal
+>
+  <Button
+    cta
+    slot="buttons"
+    on:click={() => {
+      dispatch("change", tempValue)
+      drawer.hide()
+    }}
+  >
+    Save
+  </Button>
+  <DrawerContent slot="body">
+    <FilterBuilder
+      {filters}
+      {schemaFields}
+      datasource={{ type: "table", tableId }}
+      on:change={e => (tempValue = e.detail)}
+      {bindings}
+    />
+  </DrawerContent>
+</Drawer>

packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte

@@ -4,6 +4,7 @@
     readableToRuntimeBinding,
     runtimeToReadableBinding,
   } from "dataBinding"
+  import { FieldType } from "@budibase/types"
 
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
   import { createEventDispatcher, setContext } from "svelte"
@@ -102,6 +103,8 @@
     longform: value => !isJSBinding(value),
     json: value => !isJSBinding(value),
     boolean: isValidBoolean,
+    attachment: false,
+    attachment_single: false,
   }
 
   const isValid = value => {
@@ -116,7 +119,16 @@
     if (type === "json" && !isJSBinding(value)) {
       return "json-slot-icon"
     }
-    if (!["string", "number", "bigint", "barcodeqr"].includes(type)) {
+    if (
+      ![
+        "string",
+        "number",
+        "bigint",
+        "barcodeqr",
+        "attachment",
+        "attachment_single",
+      ].includes(type)
+    ) {
       return "slot-icon"
     }
     return ""
@@ -157,7 +169,7 @@
       {updateOnChange}
     />
   {/if}
-  {#if !disabled && type !== "formula"}
+  {#if !disabled && type !== "formula" && !disabled && type !== FieldType.ATTACHMENTS && !disabled && type !== FieldType.ATTACHMENT_SINGLE}
     <div
       class={`icon ${getIconClass(value, type)}`}
       on:click={() => {

packages/builder/src/components/integration/KeyValueBuilder.svelte

@@ -37,6 +37,7 @@
   export let customButtonText = null
   export let keyBindings = false
   export let allowJS = false
+  export let actionButtonDisabled = false
   export let compare = (option, value) => option === value
 
   let fields = Object.entries(object || {}).map(([name, value]) => ({
@@ -189,7 +190,14 @@
 {/if}
 {#if !readOnly && !noAddButton}
   <div>
-    <ActionButton icon="Add" secondary thin outline on:click={addEntry}>
+    <ActionButton
+      disabled={actionButtonDisabled}
+      icon="Add"
+      secondary
+      thin
+      outline
+      on:click={addEntry}
+    >
       {#if customButtonText}
         {customButtonText}
       {:else}

packages/frontend-core/src/components/FilterBuilder.svelte

@@ -289,6 +289,7 @@
                     OperatorOptions.ContainsAny.value,
                   ].includes(filter.operator)}
                   disabled={filter.noValue}
+                  type={filter.valueType}
                 />
               {:else}
                 <Input disabled />
@@ -325,8 +326,6 @@
 <style>
   .container {
     width: 100%;
-    max-width: 1000px;
-    margin: 0 auto;
   }
   .fields {
     display: grid;

packages/frontend-core/src/components/FilterUsers.svelte

@@ -4,6 +4,7 @@
   import { createAPIClient } from "../api"
 
   export let API = createAPIClient()
+
   export let value = null
   export let disabled
   export let multiselect = false
@@ -23,12 +24,14 @@
   $: component = multiselect ? Multiselect : Select
 </script>
 
-<svelte:component
+<div class="user-control">
-  this={component}
+  <svelte:component
-  bind:value
+    this={component}
-  autocomplete
+    bind:value
-  {options}
+    autocomplete
-  getOptionLabel={option => option.email}
+    {options}
-  getOptionValue={option => option._id}
+    getOptionLabel={option => option.email}
-  {disabled}
+    getOptionValue={option => option._id}
-/>
+    {disabled}
+  />
+</div>

packages/server/src/api/controllers/row/index.ts

@@ -2,7 +2,7 @@ import stream from "stream"
 import archiver from "archiver"
 
 import { quotas } from "@budibase/pro"
-import { objectStore } from "@budibase/backend-core"
+import { objectStore, context } from "@budibase/backend-core"
 import * as internal from "./internal"
 import * as external from "./external"
 import { isExternalTableID } from "../../../integrations/utils"
@@ -198,8 +198,18 @@ export async function destroy(ctx: UserCtx<DeleteRowRequest>) {
 export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
   const tableId = utils.getTableId(ctx)
 
+  await context.ensureSnippetContext(true)
+
+  const enrichedQuery = await utils.enrichSearchContext(
+    { ...ctx.request.body.query },
+    {
+      user: sdk.users.getUserContextBindings(ctx.user),
+    }
+  )
+
   const searchParams: RowSearchParams = {
     ...ctx.request.body,
+    query: enrichedQuery,
     tableId,
   }
 

packages/server/src/api/controllers/row/utils/basic.ts

@@ -73,12 +73,15 @@ export function basicProcessing({
   // filter the row down to what is actually the row (not joined)
   for (let field of Object.values(table.schema)) {
     const fieldName = field.name
-    const value = extractFieldValue({
+    let value = extractFieldValue({
       row,
       tableName: table.name,
       fieldName,
       isLinked,
     })
+    if (value instanceof Buffer) {
+      value = value.toString()
+    }
     // all responses include "select col as table.col" so that overlaps are handled
     if (value != null) {
       thisRow[fieldName] = value

packages/server/src/api/controllers/row/utils/utils.ts

@@ -22,7 +22,7 @@ import {
   getInternalRowId,
 } from "./basic"
 import sdk from "../../../../sdk"
-
+import { processStringSync } from "@budibase/string-templates"
 import validateJs from "validate.js"
 
 validateJs.extend(validateJs.validators.datetime, {
@@ -204,3 +204,63 @@ export async function sqlOutputProcessing(
 export function isUserMetadataTable(tableId: string) {
   return tableId === InternalTables.USER_METADATA
 }
+
+export async function enrichArrayContext(
+  fields: any[],
+  inputs = {},
+  helpers = true
+): Promise<any[]> {
+  const map: Record<string, any> = {}
+  for (let index in fields) {
+    map[index] = fields[index]
+  }
+  const output = await enrichSearchContext(map, inputs, helpers)
+  const outputArray: any[] = []
+  for (let [key, value] of Object.entries(output)) {
+    outputArray[parseInt(key)] = value
+  }
+  return outputArray
+}
+
+export async function enrichSearchContext(
+  fields: Record<string, any>,
+  inputs = {},
+  helpers = true
+): Promise<Record<string, any>> {
+  const enrichedQuery: Record<string, any> = {}
+  if (!fields || !inputs) {
+    return enrichedQuery
+  }
+  const parameters = { ...inputs }
+
+  if (Array.isArray(fields)) {
+    return enrichArrayContext(fields, inputs, helpers)
+  }
+
+  // enrich the fields with dynamic parameters
+  for (let key of Object.keys(fields)) {
+    if (fields[key] == null) {
+      enrichedQuery[key] = null
+      continue
+    }
+    if (typeof fields[key] === "object") {
+      // enrich nested fields object
+      enrichedQuery[key] = await enrichSearchContext(
+        fields[key],
+        parameters,
+        helpers
+      )
+    } else if (typeof fields[key] === "string") {
+      // enrich string value as normal
+      enrichedQuery[key] = processStringSync(fields[key], parameters, {
+        noEscaping: true,
+        noHelpers: !helpers,
+        escapeNewlines: true,
+      })
+    } else {
+      enrichedQuery[key] = fields[key]
+    }
+  }
+
+  return enrichedQuery
+}

packages/server/src/api/controllers/row/views.ts

@@ -9,7 +9,8 @@ import {
 } from "@budibase/types"
 import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../../sdk"
-import { db } from "@budibase/backend-core"
+import { db, context } from "@budibase/backend-core"
+import { enrichSearchContext } from "./utils"
 
 export async function searchView(
   ctx: UserCtx<SearchViewRowRequest, SearchRowResponse>
@@ -56,10 +57,16 @@ export async function searchView(
     })
   }
 
+  await context.ensureSnippetContext(true)
+
+  const enrichedQuery = await enrichSearchContext(query, {
+    user: sdk.users.getUserContextBindings(ctx.user),
+  })
+
   const searchOptions: RequiredKeys<SearchViewRowRequest> &
     RequiredKeys<Pick<RowSearchParams, "tableId" | "query" | "fields">> = {
     tableId: view.tableId,
-    query,
+    query: enrichedQuery,
     fields: viewFields,
     ...getSortOptions(body, view),
     limit: body.limit,

packages/server/src/api/routes/tests/row.spec.ts

@@ -32,8 +32,6 @@ import * as uuid from "uuid"
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
 
-jest.unmock("mssql")
-
 describe.each([
   ["internal", undefined],
   [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
@@ -131,7 +129,13 @@ describe.each([
 
   const assertRowUsage = async (expected: number) => {
     const usage = await getRowUsage()
-    expect(usage).toBe(expected)
+
+    // Because our quota tracking is not perfect, we allow a 10% margin of
+    // error.  This is to account for the fact that parallel writes can result
+    // in some quota updates getting lost. We don't have any need to solve this
+    // right now, so we just allow for some error.
+    expect(usage).toBeGreaterThan(expected * 0.9)
+    expect(usage).toBeLessThan(expected * 1.1)
   }
 
   const defaultRowFields = isInternal
@@ -194,39 +198,99 @@ describe.each([
       await assertRowUsage(rowUsage)
     })
 
-    it("increment row autoId per create row request", async () => {
+    isInternal &&
-      const rowUsage = await getRowUsage()
+      it("increment row autoId per create row request", async () => {
+        const rowUsage = await getRowUsage()
 
-      const newTable = await config.api.table.save(
+        const newTable = await config.api.table.save(
-        saveTableRequest({
+          saveTableRequest({
-          schema: {
+            schema: {
-            "Row ID": {
+              "Row ID": {
-              name: "Row ID",
+                name: "Row ID",
-              type: FieldType.NUMBER,
+                type: FieldType.NUMBER,
-              subtype: AutoFieldSubType.AUTO_ID,
+                subtype: AutoFieldSubType.AUTO_ID,
-              icon: "ri-magic-line",
+                icon: "ri-magic-line",
-              autocolumn: true,
+                autocolumn: true,
-              constraints: {
+                constraints: {
-                type: "number",
+                  type: "number",
-                presence: true,
+                  presence: true,
-                numericality: {
+                  numericality: {
-                  greaterThanOrEqualTo: "",
+                    greaterThanOrEqualTo: "",
-                  lessThanOrEqualTo: "",
+                    lessThanOrEqualTo: "",
+                  },
                 },
               },
             },
-          },
+          })
-        })
+        )
-      )
 
-      let previousId = 0
+        let previousId = 0
-      for (let i = 0; i < 10; i++) {
+        for (let i = 0; i < 10; i++) {
-        const row = await config.api.row.save(newTable._id!, {})
+          const row = await config.api.row.save(newTable._id!, {})
-        expect(row["Row ID"]).toBeGreaterThan(previousId)
+          expect(row["Row ID"]).toBeGreaterThan(previousId)
-        previousId = row["Row ID"]
+          previousId = row["Row ID"]
-      }
+        }
-      await assertRowUsage(rowUsage + 10)
+        await assertRowUsage(rowUsage + 10)
-    })
+      })
+
+    isInternal &&
+      it("should increment auto ID correctly when creating rows in parallel", async () => {
+        const table = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              "Row ID": {
+                name: "Row ID",
+                type: FieldType.NUMBER,
+                subtype: AutoFieldSubType.AUTO_ID,
+                icon: "ri-magic-line",
+                autocolumn: true,
+                constraints: {
+                  type: "number",
+                  presence: true,
+                  numericality: {
+                    greaterThanOrEqualTo: "",
+                    lessThanOrEqualTo: "",
+                  },
+                },
+              },
+            },
+          })
+        )
+
+        const sequence = Array(50)
+          .fill(0)
+          .map((_, i) => i + 1)
+
+        // This block of code is simulating users creating auto ID rows at the
+        // same time. It's expected that this operation will sometimes return
+        // a document conflict error (409), but the idea is to retry in those
+        // situations. The code below does this a large number of times with
+        // small, random delays between them to try and get through the list
+        // as quickly as possible.
+        await Promise.all(
+          sequence.map(async () => {
+            const attempts = 20
+            for (let attempt = 0; attempt < attempts; attempt++) {
+              try {
+                await config.api.row.save(table._id!, {})
+                return
+              } catch (e) {
+                await new Promise(r => setTimeout(r, Math.random() * 15))
+              }
+            }
+            throw new Error(`Failed to create row after ${attempts} attempts`)
+          })
+        )
+
+        const rows = await config.api.row.fetch(table._id!)
+        expect(rows).toHaveLength(50)
+
+        // The main purpose of this test is to ensure that even under pressure,
+        // we maintain data integrity. An auto ID column should hand out
+        // monotonically increasing unique integers no matter what.
+        const ids = rows.map(r => r["Row ID"])
+        expect(ids).toEqual(expect.arrayContaining(sequence))
+      })
 
     isInternal &&
       it("row values are coerced", async () => {

packages/server/src/api/routes/tests/search.spec.ts

@@ -1,11 +1,13 @@
 import { tableForDatasource } from "../../../tests/utilities/structures"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
+import { db as dbCore } from "@budibase/backend-core"
 
 import * as setup from "./utilities"
 import {
   AutoFieldSubType,
   Datasource,
   EmptyFilterOption,
+  BBReferenceFieldSubType,
   FieldType,
   RowSearchParams,
   SearchFilters,
@@ -13,10 +15,14 @@ import {
   SortType,
   Table,
   TableSchema,
+  User,
 } from "@budibase/types"
 import _ from "lodash"
+import tk from "timekeeper"
+import { encodeJSBinding } from "@budibase/string-templates"
 
-jest.unmock("mssql")
+const serverTime = new Date("2024-05-06T00:00:00.000Z")
+tk.freeze(serverTime)
 
 describe.each([
   ["lucene", undefined],
@@ -35,11 +41,25 @@ describe.each([
   let datasource: Datasource | undefined
   let table: Table
 
+  const snippets = [
+    {
+      name: "WeeksAgo",
+      code: "return function (weeks) {\n  const currentTime = new Date();\n  currentTime.setDate(currentTime.getDate()-(7 * (weeks || 1)));\n  return currentTime.toISOString();\n}",
+    },
+  ]
+
   beforeAll(async () => {
     if (isSqs) {
       envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" })
     }
     await config.init()
+
+    if (config.app?.appId) {
+      config.app = await config.api.application.update(config.app?.appId, {
+        snippets,
+      })
+    }
+
     if (dsProvider) {
       datasource = await config.createDatasource({
         datasource: await dsProvider,
@@ -227,6 +247,232 @@ describe.each([
     })
   })
 
+  // Ensure all bindings resolve and perform as expected
+  describe("bindings", () => {
+    let globalUsers: any = []
+
+    const future = new Date(serverTime.getTime())
+    future.setDate(future.getDate() + 30)
+
+    const rows = (currentUser: User) => {
+      return [
+        { name: "foo", appointment: "1982-01-05T00:00:00.000Z" },
+        { name: "bar", appointment: "1995-05-06T00:00:00.000Z" },
+        { name: currentUser.firstName, appointment: future.toISOString() },
+        { name: "serverDate", appointment: serverTime.toISOString() },
+        {
+          name: "single user, session user",
+          single_user: JSON.stringify([currentUser]),
+        },
+        {
+          name: "single user",
+          single_user: JSON.stringify([globalUsers[0]]),
+        },
+        {
+          name: "multi user",
+          multi_user: JSON.stringify(globalUsers),
+        },
+        {
+          name: "multi user with session user",
+          multi_user: JSON.stringify([...globalUsers, currentUser]),
+        },
+      ]
+    }
+
+    beforeAll(async () => {
+      // Set up some global users
+      globalUsers = await Promise.all(
+        Array(2)
+          .fill(0)
+          .map(async () => {
+            const globalUser = await config.globalUser()
+            const userMedataId = globalUser._id
+              ? dbCore.generateUserMetadataID(globalUser._id)
+              : null
+            return {
+              _id: globalUser._id,
+              _meta: userMedataId,
+            }
+          })
+      )
+
+      await createTable({
+        name: { name: "name", type: FieldType.STRING },
+        appointment: { name: "appointment", type: FieldType.DATETIME },
+        single_user: {
+          name: "single_user",
+          type: FieldType.BB_REFERENCE,
+          subtype: BBReferenceFieldSubType.USER,
+        },
+        multi_user: {
+          name: "multi_user",
+          type: FieldType.BB_REFERENCE,
+          subtype: BBReferenceFieldSubType.USERS,
+        },
+      })
+      await createRows(rows(config.getUser()))
+    })
+
+    // !! Current User is auto generated per run
+    it("should return all rows matching the session user firstname", async () => {
+      await expectQuery({
+        equal: { name: "{{ [user].firstName }}" },
+      }).toContainExactly([
+        {
+          name: config.getUser().firstName,
+          appointment: future.toISOString(),
+        },
+      ])
+    })
+
+    it("should parse the date binding and return all rows after the resolved value", async () => {
+      await expectQuery({
+        range: {
+          appointment: {
+            low: "{{ [now] }}",
+            high: "9999-00-00T00:00:00.000Z",
+          },
+        },
+      }).toContainExactly([
+        {
+          name: config.getUser().firstName,
+          appointment: future.toISOString(),
+        },
+        { name: "serverDate", appointment: serverTime.toISOString() },
+      ])
+    })
+
+    it("should parse the date binding and return all rows before the resolved value", async () => {
+      await expectQuery({
+        range: {
+          appointment: {
+            low: "0000-00-00T00:00:00.000Z",
+            high: "{{ [now] }}",
+          },
+        },
+      }).toContainExactly([
+        { name: "foo", appointment: "1982-01-05T00:00:00.000Z" },
+        { name: "bar", appointment: "1995-05-06T00:00:00.000Z" },
+        { name: "serverDate", appointment: serverTime.toISOString() },
+      ])
+    })
+
+    it("should parse the encoded js snippet. Return rows with appointments up to 1 week in the past", async () => {
+      const jsBinding = "return snippets.WeeksAgo();"
+      const encodedBinding = encodeJSBinding(jsBinding)
+
+      await expectQuery({
+        range: {
+          appointment: {
+            low: "0000-00-00T00:00:00.000Z",
+            high: encodedBinding,
+          },
+        },
+      }).toContainExactly([
+        { name: "foo", appointment: "1982-01-05T00:00:00.000Z" },
+        { name: "bar", appointment: "1995-05-06T00:00:00.000Z" },
+      ])
+    })
+
+    it("should parse the encoded js binding. Return rows with appointments 2 weeks in the past", async () => {
+      const jsBinding =
+        "const currentTime = new Date()\ncurrentTime.setDate(currentTime.getDate()-14);\nreturn currentTime.toISOString();"
+      const encodedBinding = encodeJSBinding(jsBinding)
+
+      await expectQuery({
+        range: {
+          appointment: {
+            low: "0000-00-00T00:00:00.000Z",
+            high: encodedBinding,
+          },
+        },
+      }).toContainExactly([
+        { name: "foo", appointment: "1982-01-05T00:00:00.000Z" },
+        { name: "bar", appointment: "1995-05-06T00:00:00.000Z" },
+      ])
+    })
+
+    it("should match a single user row by the session user id", async () => {
+      await expectQuery({
+        equal: { single_user: "{{ [user]._id }}" },
+      }).toContainExactly([
+        {
+          name: "single user, session user",
+          single_user: [{ _id: config.getUser()._id }],
+        },
+      ])
+    })
+
+    // TODO(samwho): fix for SQS
+    !isSqs &&
+      it("should match the session user id in a multi user field", async () => {
+        const allUsers = [...globalUsers, config.getUser()].map((user: any) => {
+          return { _id: user._id }
+        })
+
+        await expectQuery({
+          contains: { multi_user: ["{{ [user]._id }}"] },
+        }).toContainExactly([
+          {
+            name: "multi user with session user",
+            multi_user: allUsers,
+          },
+        ])
+      })
+
+    // TODO(samwho): fix for SQS
+    !isSqs &&
+      it("should not match the session user id in a multi user field", async () => {
+        await expectQuery({
+          notContains: { multi_user: ["{{ [user]._id }}"] },
+          notEmpty: { multi_user: true },
+        }).toContainExactly([
+          {
+            name: "multi user",
+            multi_user: globalUsers.map((user: any) => {
+              return { _id: user._id }
+            }),
+          },
+        ])
+      })
+
+    it("should match the session user id and a user table row id using helpers, user binding and a static user id.", async () => {
+      await expectQuery({
+        oneOf: {
+          single_user: [
+            "{{ default [user]._id '_empty_' }}",
+            globalUsers[0]._id,
+          ],
+        },
+      }).toContainExactly([
+        {
+          name: "single user, session user",
+          single_user: [{ _id: config.getUser()._id }],
+        },
+        {
+          name: "single user",
+          single_user: [{ _id: globalUsers[0]._id }],
+        },
+      ])
+    })
+
+    it("should resolve 'default' helper to '_empty_' when binding resolves to nothing", async () => {
+      await expectQuery({
+        oneOf: {
+          single_user: [
+            "{{ default [user]._idx '_empty_' }}",
+            globalUsers[0]._id,
+          ],
+        },
+      }).toContainExactly([
+        {
+          name: "single user",
+          single_user: [{ _id: globalUsers[0]._id }],
+        },
+      ])
+    })
+  })
+
   describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => {
     beforeAll(async () => {
       await createTable({

packages/server/src/api/routes/tests/table.spec.ts

@@ -1,8 +1,8 @@
 import { context, events } from "@budibase/backend-core"
 import {
   AutoFieldSubType,
-  Datasource,
   BBReferenceFieldSubType,
+  Datasource,
   FieldType,
   INTERNAL_TABLE_SOURCE_ID,
   InternalTable,
@@ -149,58 +149,59 @@ describe.each([
       expect(res.name).toBeUndefined()
     })
 
-    it("updates only the passed fields", async () => {
+    isInternal &&
-      await timekeeper.withFreeze(new Date(2021, 1, 1), async () => {
+      it("updates only the passed fields", async () => {
-        const table = await config.api.table.save(
+        await timekeeper.withFreeze(new Date(2021, 1, 1), async () => {
-          tableForDatasource(datasource, {
+          const table = await config.api.table.save(
-            schema: {
+            tableForDatasource(datasource, {
-              autoId: {
+              schema: {
-                name: "id",
+                autoId: {
-                type: FieldType.NUMBER,
+                  name: "id",
-                subtype: AutoFieldSubType.AUTO_ID,
+                  type: FieldType.NUMBER,
-                autocolumn: true,
+                  subtype: AutoFieldSubType.AUTO_ID,
-                constraints: {
+                  autocolumn: true,
-                  type: "number",
+                  constraints: {
-                  presence: false,
+                    type: "number",
+                    presence: false,
+                  },
                 },
               },
-            },
+            })
+          )
+
+          const newName = generator.guid()
+
+          const updatedTable = await config.api.table.save({
+            ...table,
+            name: newName,
           })
-        )
 
-        const newName = generator.guid()
+          let expected: Table = {
+            ...table,
+            name: newName,
+            _id: expect.any(String),
+          }
+          if (isInternal) {
+            expected._rev = expect.stringMatching(/^2-.+/)
+          }
 
-        const updatedTable = await config.api.table.save({
+          expect(updatedTable).toEqual(expected)
-          ...table,
+
-          name: newName,
+          const persistedTable = await config.api.table.get(updatedTable._id!)
+          expected = {
+            ...table,
+            name: newName,
+            _id: updatedTable._id,
+          }
+          if (datasource?.isSQL) {
+            expected.sql = true
+          }
+          if (isInternal) {
+            expected._rev = expect.stringMatching(/^2-.+/)
+          }
+          expect(persistedTable).toEqual(expected)
         })
-
-        let expected: Table = {
-          ...table,
-          name: newName,
-          _id: expect.any(String),
-        }
-        if (isInternal) {
-          expected._rev = expect.stringMatching(/^2-.+/)
-        }
-
-        expect(updatedTable).toEqual(expected)
-
-        const persistedTable = await config.api.table.get(updatedTable._id!)
-        expected = {
-          ...table,
-          name: newName,
-          _id: updatedTable._id,
-        }
-        if (datasource?.isSQL) {
-          expected.sql = true
-        }
-        if (isInternal) {
-          expected._rev = expect.stringMatching(/^2-.+/)
-        }
-        expect(persistedTable).toEqual(expected)
       })
-    })
 
     describe("user table", () => {
       isInternal &&
@@ -214,6 +215,57 @@ describe.each([
         })
     })
 
+    describe("external table validation", () => {
+      !isInternal &&
+        it("should error if column is of type auto", async () => {
+          const table = basicTable(datasource)
+          await config.api.table.save(
+            {
+              ...table,
+              schema: {
+                ...table.schema,
+                auto: {
+                  name: "auto",
+                  autocolumn: true,
+                  type: FieldType.AUTO,
+                },
+              },
+            },
+            {
+              status: 400,
+              body: {
+                message: `Column "auto" has type "${FieldType.AUTO}" - this is not supported.`,
+              },
+            }
+          )
+        })
+
+      !isInternal &&
+        it("should error if column has auto subtype", async () => {
+          const table = basicTable(datasource)
+          await config.api.table.save(
+            {
+              ...table,
+              schema: {
+                ...table.schema,
+                auto: {
+                  name: "auto",
+                  autocolumn: true,
+                  type: FieldType.NUMBER,
+                  subtype: AutoFieldSubType.AUTO_ID,
+                },
+              },
+            },
+            {
+              status: 400,
+              body: {
+                message: `Column "auto" has subtype "${AutoFieldSubType.AUTO_ID}" - this is not supported.`,
+              },
+            }
+          )
+        })
+    })
+
     it("should add a new column for an internal DB table", async () => {
       const saveTableRequest: SaveTableRequest = {
         ...basicTable(),

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -24,8 +24,6 @@ import merge from "lodash/merge"
 import { quotas } from "@budibase/pro"
 import { roles } from "@budibase/backend-core"
 
-jest.unmock("mssql")
-
 describe.each([
   ["internal", undefined],
   [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],

packages/server/src/automations/automationUtils.ts

@@ -4,8 +4,11 @@ import {
   encodeJSBinding,
 } from "@budibase/string-templates"
 import sdk from "../sdk"
-import { Row } from "@budibase/types"
+import { AutomationAttachment, FieldType, Row } from "@budibase/types"
 import { LoopInput, LoopStepType } from "../definitions/automations"
+import { objectStore, context } from "@budibase/backend-core"
+import * as uuid from "uuid"
+import path from "path"
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -96,6 +99,98 @@ export function getError(err: any) {
   return typeof err !== "string" ? err.toString() : err
 }
 
+export async function sendAutomationAttachmentsToStorage(
+  tableId: string,
+  row: Row
+): Promise<Row> {
+  const table = await sdk.tables.getTable(tableId)
+  const attachmentRows: Record<
+    string,
+    AutomationAttachment[] | AutomationAttachment
+  > = {}
+
+  for (const [prop, value] of Object.entries(row)) {
+    const schema = table.schema[prop]
+    if (
+      schema?.type === FieldType.ATTACHMENTS ||
+      schema?.type === FieldType.ATTACHMENT_SINGLE
+    ) {
+      attachmentRows[prop] = value
+    }
+  }
+  for (const [prop, attachments] of Object.entries(attachmentRows)) {
+    if (Array.isArray(attachments)) {
+      if (attachments.length) {
+        row[prop] = await Promise.all(
+          attachments.map(attachment => generateAttachmentRow(attachment))
+        )
+      }
+    } else if (Object.keys(row[prop]).length > 0) {
+      row[prop] = await generateAttachmentRow(attachments)
+    }
+  }
+
+  return row
+}
+
+async function generateAttachmentRow(attachment: AutomationAttachment) {
+  const prodAppId = context.getProdAppId()
+
+  async function uploadToS3(
+    extension: string,
+    content: objectStore.StreamTypes
+  ) {
+    const fileName = `${uuid.v4()}${extension}`
+    const s3Key = `${prodAppId}/attachments/${fileName}`
+
+    await objectStore.streamUpload({
+      bucket: objectStore.ObjectStoreBuckets.APPS,
+      stream: content,
+      filename: s3Key,
+    })
+
+    return s3Key
+  }
+
+  async function getSize(s3Key: string) {
+    return (
+      await objectStore.getObjectMetadata(
+        objectStore.ObjectStoreBuckets.APPS,
+        s3Key
+      )
+    ).ContentLength
+  }
+
+  try {
+    const { filename } = attachment
+    const extension = path.extname(filename)
+    const attachmentResult = await objectStore.processAutomationAttachment(
+      attachment
+    )
+
+    let s3Key = ""
+    if (
+      "path" in attachmentResult &&
+      attachmentResult.path.startsWith(`${prodAppId}/attachments/`)
+    ) {
+      s3Key = attachmentResult.path
+    } else {
+      s3Key = await uploadToS3(extension, attachmentResult.content)
+    }
+
+    const size = await getSize(s3Key)
+
+    return {
+      size,
+      name: filename,
+      extension,
+      key: s3Key,
+    }
+  } catch (error) {
+    console.error("Failed to process attachment:", error)
+    throw error
+  }
+}
 export function substituteLoopStep(hbsString: string, substitute: string) {
   let checkForJS = isJSBinding(hbsString)
   let substitutedHbsString = ""

packages/server/src/automations/steps/createRow.ts

@@ -1,5 +1,9 @@
 import { save } from "../../api/controllers/row"
-import { cleanUpRow, getError } from "../automationUtils"
+import {
+  cleanUpRow,
+  getError,
+  sendAutomationAttachmentsToStorage,
+} from "../automationUtils"
 import { buildCtx } from "./utils"
 import {
   AutomationActionStepId,
@@ -89,6 +93,10 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) {
 
   try {
     inputs.row = await cleanUpRow(inputs.row.tableId, inputs.row)
+    inputs.row = await sendAutomationAttachmentsToStorage(
+      inputs.row.tableId,
+      inputs.row
+    )
     await save(ctx)
     return {
       row: inputs.row,

packages/server/src/automations/steps/updateRow.ts

@@ -108,7 +108,15 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) {
 
   try {
     if (tableId) {
-      inputs.row = await automationUtils.cleanUpRow(tableId, inputs.row)
+      inputs.row = await automationUtils.cleanUpRow(
+        inputs.row.tableId,
+        inputs.row
+      )
+
+      inputs.row = await automationUtils.sendAutomationAttachmentsToStorage(
+        inputs.row.tableId,
+        inputs.row
+      )
     }
     await rowController.patch(ctx)
     return {

packages/server/src/automations/tests/createRow.spec.ts

@@ -1,5 +1,18 @@
 import * as setup from "./utilities"
+import { basicTableWithAttachmentField } from "../../tests/utilities/structures"
+import { objectStore } from "@budibase/backend-core"
 
+async function uploadTestFile(filename: string) {
+  let bucket = "testbucket"
+  await objectStore.upload({
+    bucket,
+    filename,
+    body: Buffer.from("test data"),
+  })
+  let presignedUrl = await objectStore.getPresignedUrl(bucket, filename, 60000)
+
+  return presignedUrl
+}
 describe("test the create row action", () => {
   let table: any
   let row: any
@@ -43,4 +56,76 @@ describe("test the create row action", () => {
     const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {})
     expect(res.success).toEqual(false)
   })
+
+  it("should check that an attachment field is sent to storage and parsed", async () => {
+    let attachmentTable = await config.createTable(
+      basicTableWithAttachmentField()
+    )
+
+    let attachmentRow: any = {
+      tableId: attachmentTable._id,
+    }
+
+    let filename = "test1.txt"
+    let presignedUrl = await uploadTestFile(filename)
+    let attachmentObject = [
+      {
+        url: presignedUrl,
+        filename,
+      },
+    ]
+
+    attachmentRow.file_attachment = attachmentObject
+    const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {
+      row: attachmentRow,
+    })
+
+    expect(res.success).toEqual(true)
+    expect(res.row.file_attachment[0]).toHaveProperty("key")
+    let s3Key = res.row.file_attachment[0].key
+
+    const client = objectStore.ObjectStore(objectStore.ObjectStoreBuckets.APPS)
+
+    const objectData = await client
+      .headObject({ Bucket: objectStore.ObjectStoreBuckets.APPS, Key: s3Key })
+      .promise()
+
+    expect(objectData).toBeDefined()
+    expect(objectData.ContentLength).toBeGreaterThan(0)
+  })
+
+  it("should check that an single attachment field is sent to storage and parsed", async () => {
+    let attachmentTable = await config.createTable(
+      basicTableWithAttachmentField()
+    )
+
+    let attachmentRow: any = {
+      tableId: attachmentTable._id,
+    }
+
+    let filename = "test2.txt"
+    let presignedUrl = await uploadTestFile(filename)
+    let attachmentObject = {
+      url: presignedUrl,
+      filename,
+    }
+
+    attachmentRow.single_file_attachment = attachmentObject
+    const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {
+      row: attachmentRow,
+    })
+
+    expect(res.success).toEqual(true)
+    expect(res.row.single_file_attachment).toHaveProperty("key")
+    let s3Key = res.row.single_file_attachment.key
+
+    const client = objectStore.ObjectStore(objectStore.ObjectStoreBuckets.APPS)
+
+    const objectData = await client
+      .headObject({ Bucket: objectStore.ObjectStoreBuckets.APPS, Key: s3Key })
+      .promise()
+
+    expect(objectData).toBeDefined()
+    expect(objectData.ContentLength).toBeGreaterThan(0)
+  })
 })

packages/server/src/integration-test/postgres.spec.ts

@@ -1200,4 +1200,38 @@ describe("postgres integrations", () => {
       expect(Object.keys(schema).sort()).toEqual(["id", "val1"])
     })
   })
+
+  describe("check custom column types", () => {
+    beforeAll(async () => {
+      await rawQuery(
+        rawDatasource,
+        `CREATE TABLE binaryTable (
+          id BYTEA PRIMARY KEY,
+          column1 TEXT,
+          column2 INT
+        );
+      `
+      )
+    })
+
+    it("should handle binary columns", async () => {
+      const response = await makeRequest(
+        "post",
+        `/api/datasources/${datasource._id}/schema`
+      )
+      expect(response.body).toBeDefined()
+      expect(response.body.datasource.entities).toBeDefined()
+      const table = response.body.datasource.entities["binarytable"]
+      expect(table).toBeDefined()
+      expect(table.schema.id.externalType).toBe("bytea")
+      const row = await config.api.row.save(table._id, {
+        id: "1111",
+        column1: "hello",
+        column2: 222,
+      })
+      expect(row._id).toBeDefined()
+      const decoded = decodeURIComponent(row._id!).replace(/'/g, '"')
+      expect(JSON.parse(decoded)[0]).toBe("1111")
+    })
+  })
 })

packages/server/src/integrations/rest.ts

@@ -139,13 +139,13 @@ class RestIntegration implements IntegrationBase {
     const contentType = response.headers.get("content-type") || ""
     const contentDisposition = response.headers.get("content-disposition") || ""
     if (
+      contentDisposition.includes("filename") ||
       contentDisposition.includes("attachment") ||
       contentDisposition.includes("form-data")
     ) {
       filename =
         path.basename(parse(contentDisposition).parameters?.filename) || ""
     }
-
     try {
       if (filename) {
         return handleFileResponse(response, filename, this.startTimeMs)

packages/server/src/integrations/utils/utils.ts

@@ -192,6 +192,11 @@ export function generateRowIdField(keyProps: any[] = []) {
   if (!Array.isArray(keyProps)) {
     keyProps = [keyProps]
   }
+  for (let index in keyProps) {
+    if (keyProps[index] instanceof Buffer) {
+      keyProps[index] = keyProps[index].toString()
+    }
+  }
   // this conserves order and types
   // we have to swap the double quotes to single quotes for use in HBS statements
   // when using the literal helper the double quotes can break things

packages/server/src/sdk/app/rows/search/tests/internal.spec.ts

@@ -1,117 +0,0 @@
-import {
-  FieldType,
-  Row,
-  Table,
-  RowSearchParams,
-  INTERNAL_TABLE_SOURCE_ID,
-  TableSourceType,
-} from "@budibase/types"
-import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
-import { search } from "../internal"
-import {
-  expectAnyInternalColsAttributes,
-  generator,
-} from "@budibase/backend-core/tests"
-
-describe("internal", () => {
-  const config = new TestConfiguration()
-
-  const tableData: Table = {
-    name: generator.word(),
-    type: "table",
-    sourceId: INTERNAL_TABLE_SOURCE_ID,
-    sourceType: TableSourceType.INTERNAL,
-    schema: {
-      name: {
-        name: "name",
-        type: FieldType.STRING,
-        constraints: {
-          type: FieldType.STRING,
-        },
-      },
-      surname: {
-        name: "surname",
-        type: FieldType.STRING,
-        constraints: {
-          type: FieldType.STRING,
-        },
-      },
-      age: {
-        name: "age",
-        type: FieldType.NUMBER,
-        constraints: {
-          type: FieldType.NUMBER,
-        },
-      },
-      address: {
-        name: "address",
-        type: FieldType.STRING,
-        constraints: {
-          type: FieldType.STRING,
-        },
-      },
-    },
-  }
-
-  beforeAll(async () => {
-    await config.init()
-  })
-
-  describe("search", () => {
-    const rows: Row[] = []
-    beforeAll(async () => {
-      await config.createTable(tableData)
-      for (let i = 0; i < 10; i++) {
-        rows.push(
-          await config.createRow({
-            name: generator.first(),
-            surname: generator.last(),
-            age: generator.age(),
-            address: generator.address(),
-          })
-        )
-      }
-    })
-
-    it("default search returns all the data", async () => {
-      await config.doInContext(config.appId, async () => {
-        const tableId = config.table!._id!
-
-        const searchParams: RowSearchParams = {
-          tableId,
-          query: {},
-        }
-        const result = await search(searchParams, config.table!)
-
-        expect(result.rows).toHaveLength(10)
-        expect(result.rows).toEqual(
-          expect.arrayContaining(rows.map(r => expect.objectContaining(r)))
-        )
-      })
-    })
-
-    it("querying by fields will always return data attribute columns", async () => {
-      await config.doInContext(config.appId, async () => {
-        const tableId = config.table!._id!
-
-        const searchParams: RowSearchParams = {
-          tableId,
-          query: {},
-          fields: ["name", "age"],
-        }
-        const result = await search(searchParams, config.table!)
-
-        expect(result.rows).toHaveLength(10)
-        expect(result.rows).toEqual(
-          expect.arrayContaining(
-            rows.map(r => ({
-              ...expectAnyInternalColsAttributes,
-              name: r.name,
-              age: r.age,
-            }))
-          )
-        )
-      })
-    })
-  })
-})

packages/server/src/sdk/app/rows/tests/internal.spec.ts

@@ -1,230 +0,0 @@
-import tk from "timekeeper"
-import * as internalSdk from "../internal"
-
-import { generator } from "@budibase/backend-core/tests"
-import {
-  INTERNAL_TABLE_SOURCE_ID,
-  TableSourceType,
-  FieldType,
-  Table,
-  AutoFieldSubType,
-  AutoColumnFieldMetadata,
-} from "@budibase/types"
-
-import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
-
-tk.freeze(Date.now())
-
-describe("sdk >> rows >> internal", () => {
-  const config = new TestConfiguration()
-
-  beforeAll(async () => {
-    await config.init()
-  })
-
-  function makeRow() {
-    return {
-      name: generator.first(),
-      surname: generator.last(),
-      age: generator.age(),
-      address: generator.address(),
-    }
-  }
-
-  describe("save", () => {
-    const tableData: Table = {
-      name: generator.word(),
-      type: "table",
-      sourceId: INTERNAL_TABLE_SOURCE_ID,
-      sourceType: TableSourceType.INTERNAL,
-      schema: {
-        name: {
-          name: "name",
-          type: FieldType.STRING,
-          constraints: {
-            type: FieldType.STRING,
-          },
-        },
-        surname: {
-          name: "surname",
-          type: FieldType.STRING,
-          constraints: {
-            type: FieldType.STRING,
-          },
-        },
-        age: {
-          name: "age",
-          type: FieldType.NUMBER,
-          constraints: {
-            type: FieldType.NUMBER,
-          },
-        },
-        address: {
-          name: "address",
-          type: FieldType.STRING,
-          constraints: {
-            type: FieldType.STRING,
-          },
-        },
-      },
-    }
-
-    beforeEach(() => {
-      jest.clearAllMocks()
-    })
-
-    it("save will persist the row properly", async () => {
-      const table = await config.createTable(tableData)
-      const row = makeRow()
-
-      await config.doInContext(config.appId, async () => {
-        const response = await internalSdk.save(
-          table._id!,
-          row,
-          config.getUser()._id
-        )
-
-        expect(response).toEqual({
-          table,
-          row: {
-            ...row,
-            type: "row",
-            _rev: expect.stringMatching("1-.*"),
-          },
-          squashed: {
-            ...row,
-            type: "row",
-            _rev: expect.stringMatching("1-.*"),
-          },
-        })
-
-        const persistedRow = await config.api.row.get(
-          table._id!,
-          response.row._id!
-        )
-        expect(persistedRow).toEqual({
-          ...row,
-          type: "row",
-          _rev: expect.stringMatching("1-.*"),
-          createdAt: expect.any(String),
-          updatedAt: expect.any(String),
-        })
-      })
-    })
-
-    it("auto ids will update when creating new rows", async () => {
-      const table = await config.createTable({
-        ...tableData,
-        schema: {
-          ...tableData.schema,
-          id: {
-            name: "id",
-            type: FieldType.AUTO,
-            subtype: AutoFieldSubType.AUTO_ID,
-            autocolumn: true,
-            lastID: 0,
-          },
-        },
-      })
-      const row = makeRow()
-
-      await config.doInContext(config.appId, async () => {
-        const response = await internalSdk.save(
-          table._id!,
-          row,
-          config.getUser()._id
-        )
-
-        expect(response).toEqual({
-          table: {
-            ...table,
-            schema: {
-              ...table.schema,
-              id: {
-                ...table.schema.id,
-                lastID: 1,
-              },
-            },
-            _rev: expect.stringMatching("2-.*"),
-          },
-          row: {
-            ...row,
-            id: 1,
-            type: "row",
-            _rev: expect.stringMatching("1-.*"),
-          },
-          squashed: {
-            ...row,
-            id: 1,
-            type: "row",
-            _rev: expect.stringMatching("1-.*"),
-          },
-        })
-
-        const persistedRow = await config.api.row.get(
-          table._id!,
-          response.row._id!
-        )
-        expect(persistedRow).toEqual({
-          ...row,
-          type: "row",
-          id: 1,
-          _rev: expect.stringMatching("1-.*"),
-          createdAt: expect.any(String),
-          updatedAt: expect.any(String),
-        })
-      })
-    })
-
-    it("auto ids will update when creating new rows in parallel", async () => {
-      function makeRows(count: number) {
-        return Array.from({ length: count }, () => makeRow())
-      }
-
-      const table = await config.createTable({
-        ...tableData,
-        schema: {
-          ...tableData.schema,
-          id: {
-            name: "id",
-            type: FieldType.AUTO,
-            subtype: AutoFieldSubType.AUTO_ID,
-            autocolumn: true,
-          },
-        },
-      })
-
-      await config.doInContext(config.appId, async () => {
-        for (const row of makeRows(5)) {
-          await internalSdk.save(table._id!, row, config.getUser()._id)
-        }
-        await Promise.all(
-          makeRows(20).map(row =>
-            internalSdk.save(table._id!, row, config.getUser()._id)
-          )
-        )
-        for (const row of makeRows(5)) {
-          await internalSdk.save(table._id!, row, config.getUser()._id)
-        }
-      })
-
-      const persistedRows = await config.getRows(table._id!)
-      expect(persistedRows).toHaveLength(30)
-      expect(persistedRows).toEqual(
-        expect.arrayContaining(
-          Array.from({ length: 30 }).map((_, i) =>
-            expect.objectContaining({ id: i + 1 })
-          )
-        )
-      )
-
-      const persistedTable = await config.getTable(table._id)
-      expect(
-        (table.schema.id as AutoColumnFieldMetadata).lastID
-      ).toBeUndefined()
-      expect((persistedTable.schema.id as AutoColumnFieldMetadata).lastID).toBe(
-        30
-      )
-    })
-  })
-})

packages/server/src/sdk/app/tables/external/index.ts

@@ -6,6 +6,7 @@ import {
   Table,
   TableRequest,
   ViewV2,
+  AutoFieldSubType,
 } from "@budibase/types"
 import { context } from "@budibase/backend-core"
 import { buildExternalTableId } from "../../../../integrations/utils"
@@ -29,6 +30,52 @@ import { populateExternalTableSchemas } from "../validation"
 import datasourceSdk from "../../datasources"
 import * as viewSdk from "../../views"
 
+const DEFAULT_PRIMARY_COLUMN = "id"
+
+function noPrimaryKey(table: Table) {
+  return table.primary == null || table.primary.length === 0
+}
+
+function validate(table: Table, oldTable?: Table) {
+  if (
+    !oldTable &&
+    table.schema[DEFAULT_PRIMARY_COLUMN] &&
+    noPrimaryKey(table)
+  ) {
+    throw new Error(
+      "External tables with no `primary` column set will define an `id` column, but we found an `id` column in the supplied schema. Either set a `primary` column or remove the `id` column."
+    )
+  }
+
+  if (hasTypeChanged(table, oldTable)) {
+    throw new Error("A column type has changed.")
+  }
+
+  const autoSubTypes = Object.values(AutoFieldSubType)
+  // check for auto columns, they are not allowed
+  for (let [key, column] of Object.entries(table.schema)) {
+    // this column is a special case, do not validate it
+    if (key === DEFAULT_PRIMARY_COLUMN) {
+      continue
+    }
+    // the auto-column type should never be used
+    if (column.type === FieldType.AUTO) {
+      throw new Error(
+        `Column "${key}" has type "${FieldType.AUTO}" - this is not supported.`
+      )
+    }
+
+    if (
+      column.subtype &&
+      autoSubTypes.includes(column.subtype as AutoFieldSubType)
+    ) {
+      throw new Error(
+        `Column "${key}" has subtype "${column.subtype}" - this is not supported.`
+      )
+    }
+  }
+}
+
 export async function save(
   datasourceId: string,
   update: Table,
@@ -47,28 +94,18 @@ export async function save(
     oldTable = await getTable(tableId)
   }
 
-  if (
+  // this will throw an error if something is wrong
-    !oldTable &&
+  validate(tableToSave, oldTable)
-    (tableToSave.primary == null || tableToSave.primary.length === 0)
-  ) {
-    if (tableToSave.schema.id) {
-      throw new Error(
-        "External tables with no `primary` column set will define an `id` column, but we found an `id` column in the supplied schema. Either set a `primary` column or remove the `id` column."
-      )
-    }
 
-    tableToSave.primary = ["id"]
+  if (!oldTable && noPrimaryKey(tableToSave)) {
-    tableToSave.schema.id = {
+    tableToSave.primary = [DEFAULT_PRIMARY_COLUMN]
+    tableToSave.schema[DEFAULT_PRIMARY_COLUMN] = {
       type: FieldType.NUMBER,
       autocolumn: true,
-      name: "id",
+      name: DEFAULT_PRIMARY_COLUMN,
     }
   }
 
-  if (hasTypeChanged(tableToSave, oldTable)) {
-    throw new Error("A column type has changed.")
-  }
-
   for (let view in tableToSave.views) {
     const tableView = tableToSave.views[view]
     if (!tableView || !viewSdk.isV2(tableView)) continue

packages/server/src/sdk/users/utils.ts

@@ -124,3 +124,12 @@ export async function syncGlobalUsers() {
     await db.bulkDocs(toWrite)
   }
 }
+
+export function getUserContextBindings(user: ContextUser) {
+  if (!user) {
+    return {}
+  }
+  // Current user context for bindable search
+  const { _id, _rev, firstName, lastName, email, status, roleId } = user
+  return { _id, _rev, firstName, lastName, email, status, roleId }
+}

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -81,6 +81,12 @@ mocks.licenses.useUnlimited()
 
 dbInit()
 
+export interface CreateAppRequest {
+  appName: string
+  url?: string
+  snippets?: any[]
+}
+
 export interface TableToBuild extends Omit<Table, "sourceId" | "sourceType"> {
   sourceId?: string
   sourceType?: TableSourceType
@@ -580,8 +586,6 @@ export default class TestConfiguration {
 
   // APP
   async createApp(appName: string, url?: string): Promise<App> {
-    // create dev app
-    // clear any old app
     this.appId = undefined
     this.app = await context.doInTenant(
       this.tenantId!,
@@ -592,6 +596,7 @@ export default class TestConfiguration {
         })) as App
     )
     this.appId = this.app.appId
+
     return await context.doInAppContext(this.app.appId!, async () => {
       // create production app
       this.prodApp = await this.publish()

packages/server/src/tests/utilities/structures.ts

@@ -78,6 +78,32 @@ export function basicTable(
   )
 }
 
+export function basicTableWithAttachmentField(
+  datasource?: Datasource,
+  ...extra: Partial<Table>[]
+): Table {
+  return tableForDatasource(
+    datasource,
+    {
+      name: "TestTable",
+      schema: {
+        file_attachment: {
+          type: FieldType.ATTACHMENTS,
+          name: "description",
+          constraints: {
+            type: "array",
+          },
+        },
+        single_file_attachment: {
+          type: FieldType.ATTACHMENT_SINGLE,
+          name: "description",
+        },
+      },
+    },
+    ...extra
+  )
+}
+
 export function basicView(tableId: string) {
   return {
     tableId,

packages/server/src/utilities/rowProcessor/index.ts

@@ -1,6 +1,6 @@
 import * as linkRows from "../../db/linkedRows"
 import { processFormulas, fixAutoColumnSubType } from "./utils"
-import { context, objectStore, utils } from "@budibase/backend-core"
+import { objectStore, utils } from "@budibase/backend-core"
 import { InternalTables } from "../../db/utils"
 import { TYPE_TRANSFORM_MAP } from "./map"
 import {
@@ -25,45 +25,6 @@ type AutoColumnProcessingOpts = {
   noAutoRelationships?: boolean
 }
 
-// Returns the next auto ID for a column in a table. On success, the table will
-// be updated which is why it gets returned. The nextID returned is guaranteed
-// to be given only to you, and if you don't use it it's gone forever (a gap
-// will be left in the auto ID sequence).
-//
-// This function can throw if it fails to generate an auto ID after so many
-// attempts.
-async function getNextAutoId(
-  table: Table,
-  column: string
-): Promise<{ table: Table; nextID: number }> {
-  const db = context.getAppDB()
-  for (let attempt = 0; attempt < 5; attempt++) {
-    const schema = table.schema[column]
-    if (schema.type !== FieldType.NUMBER && schema.type !== FieldType.AUTO) {
-      throw new Error(`Column ${column} is not an auto column`)
-    }
-    schema.lastID = (schema.lastID || 0) + 1
-    try {
-      const resp = await db.put(table)
-      table._rev = resp.rev
-      return { table, nextID: schema.lastID }
-    } catch (e: any) {
-      if (e.status !== 409) {
-        throw e
-      }
-      // We wait for a random amount of time before retrying. The randomness
-      // makes it less likely for multiple requests modifying this table to
-      // collide.
-      await new Promise(resolve =>
-        setTimeout(resolve, Math.random() * 1.2 ** attempt * 1000)
-      )
-      table = await db.get(table._id)
-    }
-  }
-
-  throw new Error("Failed to generate an auto ID")
-}
-
 /**
  * This will update any auto columns that are found on the row/table with the correct information based on
  * time now and the current logged in user making the request.
@@ -116,9 +77,10 @@ export async function processAutoColumn(
         break
       case AutoFieldSubType.AUTO_ID:
         if (creating) {
-          const { table: newTable, nextID } = await getNextAutoId(table, key)
+          schema.lastID = schema.lastID || 0
-          table = newTable
+          row[key] = schema.lastID + 1
-          row[key] = nextID
+          schema.lastID++
+          table.schema[key] = schema
         }
         break
     }
@@ -272,7 +234,7 @@ export async function outputProcessing<T extends Row[] | Row>(
       }
     } else if (column.type === FieldType.ATTACHMENT_SINGLE) {
       for (let row of enriched) {
-        if (!row[property]) {
+        if (!row[property] || Object.keys(row[property]).length === 0) {
           continue
         }
 

packages/types/src/documents/app/automation.ts

@@ -1,6 +1,7 @@
 import { Document } from "../document"
 import { EventEmitter } from "events"
 import { User } from "../global"
+import { ReadStream } from "fs"
 
 export enum AutomationIOType {
   OBJECT = "object",
@@ -235,3 +236,18 @@ export interface AutomationMetadata extends Document {
   errorCount?: number
   automationChainCount?: number
 }
+
+export type AutomationAttachment = {
+  url: string
+  filename: string
+}
+
+export type AutomationAttachmentContent = {
+  filename: string
+  content: ReadStream | NodeJS.ReadableStream | ReadableStream<Uint8Array>
+}
+
+export type BucketedContent = AutomationAttachmentContent & {
+  bucket: string
+  path: string
+}

packages/worker/src/api/routes/global/tests/realEmail.spec.ts

@@ -1,5 +1,4 @@
 jest.unmock("node-fetch")
-jest.unmock("aws-sdk")
 import { TestConfiguration } from "../../../../tests"
 import { EmailTemplatePurpose } from "../../../../constants"
 import { objectStore } from "@budibase/backend-core"

packages/worker/src/utilities/email.ts

@@ -6,8 +6,7 @@ import { processString } from "@budibase/string-templates"
 import { User, SendEmailOpts, SMTPInnerConfig } from "@budibase/types"
 import { configs, cache, objectStore } from "@budibase/backend-core"
 import ical from "ical-generator"
-import fetch from "node-fetch"
+import _ from "lodash"
-import path from "path"
 
 const nodemailer = require("nodemailer")
 
@@ -165,39 +164,12 @@ export async function sendEmail(
     }),
   }
   if (opts?.attachments) {
-    const attachments = await Promise.all(
+    let attachments = await Promise.all(
-      opts.attachments?.map(async attachment => {
+      opts.attachments?.map(objectStore.processAutomationAttachment)
-        const isFullyFormedUrl =
-          attachment.url.startsWith("http://") ||
-          attachment.url.startsWith("https://")
-        if (isFullyFormedUrl) {
-          const response = await fetch(attachment.url)
-          if (!response.ok) {
-            throw new Error(`unexpected response ${response.statusText}`)
-          }
-          const fallbackFilename = path.basename(
-            new URL(attachment.url).pathname
-          )
-          return {
-            filename: attachment.filename || fallbackFilename,
-            content: response?.body,
-          }
-        } else {
-          const url = attachment.url
-          const result = objectStore.extractBucketAndPath(url)
-          if (result === null) {
-            throw new Error("Invalid signed URL")
-          }
-          const { bucket, path } = result
-          const readStream = await objectStore.getReadStream(bucket, path)
-          const fallbackFilename = path.split("/").pop() || ""
-          return {
-            filename: attachment.filename || fallbackFilename,
-            content: readStream,
-          }
-        }
-      })
     )
+    attachments = attachments.map(attachment => {
+      return _.omit(attachment, "path")
+    })
     message = { ...message, attachments }
   }