Merge branch 'master' into new-table-selected-rows

.eslintrc.json

@@ -42,6 +42,8 @@
       },
       "rules": {
         "no-unused-vars": "off",
+        "local-rules/no-budibase-imports": "error",
+        "local-rules/no-console-error": "error",
         "@typescript-eslint/no-unused-vars": [
           "error",
           {

.vscode/settings.json

@@ -24,5 +24,8 @@
   },
   "[svelte]": {
     "editor.defaultFormatter": "svelte.svelte-vscode"
+  },
+  "[handlebars]": {
+    "editor.formatOnSave": false
   }
 }

charts/budibase/templates/app-service-deployment.yaml

@@ -106,6 +106,8 @@ spec:
           value: {{ .Values.services.objectStore.globalBucketName | quote }}
         - name: BACKUPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.backupsBucketName | quote }}
+        - name: TEMP_BUCKET_NAME
+          value: {{ .Values.globals.tempBucketName | quote }}
         - name: PORT
           value: {{ .Values.services.apps.port | quote }}
         {{ if .Values.services.worker.publicApiRateLimitPerSecond }}

charts/budibase/templates/automation-worker-service-deployment.yaml

@@ -107,6 +107,8 @@ spec:
           value: {{ .Values.services.objectStore.globalBucketName | quote }}
         - name: BACKUPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.backupsBucketName | quote }}
+        - name: TEMP_BUCKET_NAME
+          value: {{ .Values.globals.tempBucketName | quote }}
         - name: PORT
           value: {{ .Values.services.automationWorkers.port | quote }}
         {{ if .Values.services.worker.publicApiRateLimitPerSecond }}

charts/budibase/templates/worker-service-deployment.yaml

@@ -106,6 +106,8 @@ spec:
           value: {{ .Values.services.objectStore.globalBucketName | quote }}
         - name: BACKUPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.backupsBucketName | quote }}
+        - name: TEMP_BUCKET_NAME
+          value: {{ .Values.globals.tempBucketName | quote }}
         - name: PORT
           value: {{ .Values.services.worker.port | quote }}
         - name: MULTI_TENANCY

charts/budibase/values.yaml

@@ -121,6 +121,9 @@ globals:
   # to the old value for the duration of the rotation.
   jwtSecretFallback: ""
 
+  ## -- If using S3 the bucket name to be used for storing temporary files
+  tempBucketName: ""
+
   smtp:
     # -- Whether to enable SMTP or not.
     enabled: false

eslint-local-rules/index.js

@@ -1,4 +1,25 @@
 module.exports = {
+  "no-console-error": {
+    create: function(context) {
+      return {
+        CallExpression(node) {
+          if (
+            node.callee.type === "MemberExpression" &&
+            node.callee.object.name === "console" &&
+            node.callee.property.name === "error" &&
+            node.arguments.length === 1 &&
+            node.arguments[0].name &&
+            node.arguments[0].name.startsWith("err")
+          ) {
+            context.report({
+              node,
+              message: 'Using console.error(err) on its own is not allowed. Either provide context to the error (console.error(msg, err)) or throw it.',
+            })
+          }
+        },
+      };
+    },
+  },
   "no-budibase-imports": {
     create: function (context) {
       return {

hosting/single/Dockerfile

@@ -19,9 +19,6 @@ RUN chmod +x ./scripts/removeWorkspaceDependencies.sh
 RUN ./scripts/removeWorkspaceDependencies.sh packages/server/package.json
 RUN ./scripts/removeWorkspaceDependencies.sh packages/worker/package.json
 
-
-# We will never want to sync pro, but the script is still required
-RUN echo '' > scripts/syncProPackage.js
 RUN jq 'del(.scripts.postinstall)' package.json > temp.json && mv temp.json package.json
 RUN ./scripts/removeWorkspaceDependencies.sh package.json
 RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production --frozen-lockfile

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.23.7",
+  "version": "2.23.12",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

package.json

@@ -32,7 +32,6 @@
     "yargs": "^17.7.2"
   },
   "scripts": {
-    "preinstall": "node scripts/syncProPackage.js",
     "get-past-client-version": "node scripts/getPastClientVersion.js",
     "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev",
     "build": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream",
@@ -107,6 +106,7 @@
     "@budibase/shared-core": "0.0.0",
     "@budibase/string-templates": "0.0.0",
     "@budibase/types": "0.0.0",
+    "@budibase/pro": "npm:@budibase/pro@latest",
     "tough-cookie": "4.1.3",
     "node-fetch": "2.6.7",
     "semver": "7.5.3",

packages/account-portal

@@ -1 +1 @@
-Subproject commit eb7d5da233885c5cffd9c255d3e954d0cd39185e
+Subproject commit c167c331ff9b8161fc18e2ecbaaf1ea5815ba964

packages/backend-core/src/auth/auth.ts

@@ -64,7 +64,6 @@ async function refreshOIDCAccessToken(
     }
     strategy = await oidc.strategyFactory(enrichedConfig, ssoSaveUserNoOp)
   } catch (err) {
-    console.error(err)
     throw new Error("Could not refresh OAuth Token")
   }
 
@@ -99,7 +98,6 @@ async function refreshGoogleAccessToken(
       ssoSaveUserNoOp
     )
   } catch (err: any) {
-    console.error(err)
     throw new Error(
       `Error constructing OIDC refresh strategy: message=${err.message}`
     )

packages/backend-core/src/environment.ts

@@ -29,6 +29,7 @@ const DefaultBucketName = {
   TEMPLATES: "templates",
   GLOBAL: "global",
   PLUGINS: "plugins",
+  TEMP: "tmp-file-attachments",
 }
 
 const selfHosted = !!parseInt(process.env.SELF_HOSTED || "")
@@ -146,6 +147,7 @@ const environment = {
     process.env.GLOBAL_BUCKET_NAME || DefaultBucketName.GLOBAL,
   PLUGIN_BUCKET_NAME:
     process.env.PLUGIN_BUCKET_NAME || DefaultBucketName.PLUGINS,
+  TEMP_BUCKET_NAME: process.env.TEMP_BUCKET_NAME || DefaultBucketName.TEMP,
   USE_COUCH: process.env.USE_COUCH || true,
   MOCK_REDIS: process.env.MOCK_REDIS,
   DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,

packages/backend-core/src/middleware/authenticated.ts

@@ -138,7 +138,6 @@ export default function (
         } catch (err: any) {
           authenticated = false
           console.error(`Auth Error: ${err.message}`)
-          console.error(err)
           // remove the cookie as the user does not exist anymore
           clearCookie(ctx, Cookie.Auth)
         }
@@ -187,7 +186,6 @@ export default function (
       }
     } catch (err: any) {
       console.error(`Auth Error: ${err.message}`)
-      console.error(err)
       // invalid token, clear the cookie
       if (err?.name === "JsonWebTokenError") {
         clearCookie(ctx, Cookie.Auth)

packages/backend-core/src/middleware/errorHandling.ts

@@ -12,7 +12,7 @@ export async function errorHandling(ctx: any, next: any) {
     if (status >= 400 && status < 500) {
       console.warn(err)
     } else {
-      console.error(err)
+      console.error("Got 400 response code", err)
     }
 
     let error: APIError = {

packages/backend-core/src/middleware/passport/sso/google.ts

@@ -68,7 +68,6 @@ export async function strategyFactory(
       verify
     )
   } catch (err: any) {
-    console.error(err)
     throw new Error(`Error constructing google authentication strategy: ${err}`)
   }
 }

packages/backend-core/src/middleware/passport/sso/oidc.ts

@@ -103,7 +103,6 @@ export async function strategyFactory(
     strategy.name = "oidc"
     return strategy
   } catch (err: any) {
-    console.error(err)
     throw new Error(`Error constructing OIDC authentication strategy - ${err}`)
   }
 }
@@ -142,7 +141,6 @@ export async function fetchStrategyConfig(
       callbackURL: callbackUrl,
     }
   } catch (err) {
-    console.error(err)
     throw new Error(
       `Error constructing OIDC authentication configuration - ${err}`
     )

packages/backend-core/src/migrations/migrations.ts

@@ -26,7 +26,6 @@ export const getMigrationsDoc = async (db: any) => {
     if (err.status && err.status === 404) {
       return { _id: DocumentType.MIGRATIONS }
     } else {
-      console.error(err)
       throw err
     }
   }

packages/backend-core/src/objectStore/objectStore.ts

@@ -7,31 +7,41 @@ import tar from "tar-fs"
 import zlib from "zlib"
 import { promisify } from "util"
 import { join } from "path"
-import fs, { ReadStream } from "fs"
+import fs, { PathLike, ReadStream } from "fs"
 import env from "../environment"
-import { budibaseTempDir } from "./utils"
+import { bucketTTLConfig, budibaseTempDir } from "./utils"
 import { v4 } from "uuid"
 import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
+import fsp from "fs/promises"
 
 const streamPipeline = promisify(stream.pipeline)
 // use this as a temporary store of buckets that are being created
 const STATE = {
   bucketCreationPromises: {},
 }
+const signedFilePrefix = "/files/signed"
 
 type ListParams = {
   ContinuationToken?: string
 }
 
-type UploadParams = {
+type BaseUploadParams = {
   bucket: string
   filename: string
-  path: string
   type?: string | null
-  // can be undefined, we will remove it
+  metadata?: { [key: string]: string | undefined }
-  metadata?: {
+  body?: ReadableStream | Buffer
-    [key: string]: string | undefined
+  ttl?: number
+  addTTL?: boolean
+  extra?: any
 }
+
+type UploadParams = BaseUploadParams & {
+  path?: string | PathLike
+}
+
+type StreamUploadParams = BaseUploadParams & {
+  stream: ReadStream
 }
 
 const CONTENT_TYPE_MAP: any = {
@@ -41,6 +51,8 @@ const CONTENT_TYPE_MAP: any = {
   js: "application/javascript",
   json: "application/json",
   gz: "application/gzip",
+  svg: "image/svg+xml",
+  form: "multipart/form-data",
 }
 
 const STRING_CONTENT_TYPES = [
@@ -105,7 +117,10 @@ export function ObjectStore(
  * Given an object store and a bucket name this will make sure the bucket exists,
  * if it does not exist then it will create it.
  */
-export async function makeSureBucketExists(client: any, bucketName: string) {
+export async function createBucketIfNotExists(
+  client: any,
+  bucketName: string
+): Promise<{ created: boolean; exists: boolean }> {
   bucketName = sanitizeBucket(bucketName)
   try {
     await client
@@ -113,15 +128,16 @@ export async function makeSureBucketExists(client: any, bucketName: string) {
         Bucket: bucketName,
       })
       .promise()
+    return { created: false, exists: true }
   } catch (err: any) {
     const promises: any = STATE.bucketCreationPromises
     const doesntExist = err.statusCode === 404,
       noAccess = err.statusCode === 403
     if (promises[bucketName]) {
       await promises[bucketName]
+      return { created: false, exists: true }
     } else if (doesntExist || noAccess) {
       if (doesntExist) {
-        // bucket doesn't exist create it
         promises[bucketName] = client
           .createBucket({
             Bucket: bucketName,
@@ -129,13 +145,15 @@ export async function makeSureBucketExists(client: any, bucketName: string) {
           .promise()
         await promises[bucketName]
         delete promises[bucketName]
+        return { created: true, exists: false }
+      } else {
+        throw new Error("Access denied to object store bucket." + err)
       }
     } else {
       throw new Error("Unable to write to object store bucket.")
     }
   }
 }
-
 /**
  * Uploads the contents of a file given the required parameters, useful when
  * temp files in use (for example file uploaded as an attachment).
@@ -146,12 +164,22 @@ export async function upload({
   path,
   type,
   metadata,
+  body,
+  ttl,
 }: UploadParams) {
   const extension = filename.split(".").pop()
-  const fileBytes = fs.readFileSync(path)
+
+  const fileBytes = path ? (await fsp.open(path)).createReadStream() : body
 
   const objectStore = ObjectStore(bucketName)
-  await makeSureBucketExists(objectStore, bucketName)
+  const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
+
+  if (ttl && (bucketCreated.created || bucketCreated.exists)) {
+    let ttlConfig = bucketTTLConfig(bucketName, ttl)
+    if (objectStore.putBucketLifecycleConfiguration) {
+      await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
+    }
+  }
 
   let contentType = type
   if (!contentType) {
@@ -174,6 +202,7 @@ export async function upload({
     }
     config.Metadata = metadata
   }
+
   return objectStore.upload(config).promise()
 }
 
@@ -181,14 +210,24 @@ export async function upload({
  * Similar to the upload function but can be used to send a file stream
  * through to the object store.
  */
-export async function streamUpload(
+export async function streamUpload({
-  bucketName: string,
+  bucket: bucketName,
-  filename: string,
+  stream,
-  stream: ReadStream | ReadableStream,
+  filename,
-  extra = {}
+  type,
-) {
+  extra,
+  ttl,
+}: StreamUploadParams) {
+  const extension = filename.split(".").pop()
   const objectStore = ObjectStore(bucketName)
-  await makeSureBucketExists(objectStore, bucketName)
+  const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
+
+  if (ttl && (bucketCreated.created || bucketCreated.exists)) {
+    let ttlConfig = bucketTTLConfig(bucketName, ttl)
+    if (objectStore.putBucketLifecycleConfiguration) {
+      await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
+    }
+  }
 
   // Set content type for certain known extensions
   if (filename?.endsWith(".js")) {
@@ -203,10 +242,18 @@ export async function streamUpload(
     }
   }
 
+  let contentType = type
+  if (!contentType) {
+    contentType = extension
+      ? CONTENT_TYPE_MAP[extension.toLowerCase()]
+      : CONTENT_TYPE_MAP.txt
+  }
+
   const params = {
     Bucket: sanitizeBucket(bucketName),
     Key: sanitizeKey(filename),
     Body: stream,
+    ContentType: contentType,
     ...extra,
   }
   return objectStore.upload(params).promise()
@@ -286,7 +333,7 @@ export function getPresignedUrl(
     const signedUrl = new URL(url)
     const path = signedUrl.pathname
     const query = signedUrl.search
-    return `/files/signed${path}${query}`
+    return `${signedFilePrefix}${path}${query}`
   }
 }
 
@@ -341,7 +388,7 @@ export async function retrieveDirectory(bucketName: string, path: string) {
  */
 export async function deleteFile(bucketName: string, filepath: string) {
   const objectStore = ObjectStore(bucketName)
-  await makeSureBucketExists(objectStore, bucketName)
+  await createBucketIfNotExists(objectStore, bucketName)
   const params = {
     Bucket: bucketName,
     Key: sanitizeKey(filepath),
@@ -351,7 +398,7 @@ export async function deleteFile(bucketName: string, filepath: string) {
 
 export async function deleteFiles(bucketName: string, filepaths: string[]) {
   const objectStore = ObjectStore(bucketName)
-  await makeSureBucketExists(objectStore, bucketName)
+  await createBucketIfNotExists(objectStore, bucketName)
   const params = {
     Bucket: bucketName,
     Delete: {
@@ -412,7 +459,13 @@ export async function uploadDirectory(
     if (file.isDirectory()) {
       uploads.push(uploadDirectory(bucketName, local, path))
     } else {
-      uploads.push(streamUpload(bucketName, path, fs.createReadStream(local)))
+      uploads.push(
+        streamUpload({
+          bucket: bucketName,
+          filename: path,
+          stream: fs.createReadStream(local),
+        })
+      )
     }
   }
   await Promise.all(uploads)
@@ -467,3 +520,23 @@ export async function getReadStream(
   }
   return client.getObject(params).createReadStream()
 }
+
+/*
+Given a signed url like '/files/signed/tmp-files-attachments/app_123456/myfile.txt' extract
+the bucket and the path from it
+*/
+export function extractBucketAndPath(
+  url: string
+): { bucket: string; path: string } | null {
+  const baseUrl = url.split("?")[0]
+
+  const regex = new RegExp(`^${signedFilePrefix}/(?<bucket>[^/]+)/(?<path>.+)$`)
+  const match = baseUrl.match(regex)
+
+  if (match && match.groups) {
+    const { bucket, path } = match.groups
+    return { bucket, path }
+  }
+
+  return null
+}

packages/backend-core/src/objectStore/utils.ts

@@ -2,6 +2,7 @@ import { join } from "path"
 import { tmpdir } from "os"
 import fs from "fs"
 import env from "../environment"
+import { PutBucketLifecycleConfigurationRequest } from "aws-sdk/clients/s3"
 
 /****************************************************
  *      NOTE: When adding a new bucket - name       *
@@ -15,6 +16,7 @@ export const ObjectStoreBuckets = {
   TEMPLATES: env.TEMPLATES_BUCKET_NAME,
   GLOBAL: env.GLOBAL_BUCKET_NAME,
   PLUGINS: env.PLUGIN_BUCKET_NAME,
+  TEMP: env.TEMP_BUCKET_NAME,
 }
 
 const bbTmp = join(tmpdir(), ".budibase")
@@ -29,3 +31,27 @@ try {
 export function budibaseTempDir() {
   return bbTmp
 }
+
+export const bucketTTLConfig = (
+  bucketName: string,
+  days: number
+): PutBucketLifecycleConfigurationRequest => {
+  const lifecycleRule = {
+    ID: `${bucketName}-ExpireAfter${days}days`,
+    Prefix: "",
+    Status: "Enabled",
+    Expiration: {
+      Days: days,
+    },
+  }
+  const lifecycleConfiguration = {
+    Rules: [lifecycleRule],
+  }
+
+  const params = {
+    Bucket: bucketName,
+    LifecycleConfiguration: lifecycleConfiguration,
+  }
+
+  return params
+}

packages/backend-core/src/users/db.ts

@@ -50,6 +50,8 @@ type CreateAdminUserOpts = {
   hashPassword?: boolean
   requirePassword?: boolean
   skipPasswordValidation?: boolean
+  firstName?: string
+  lastName?: string
 }
 type FeatureFns = { isSSOEnforced: FeatureFn; isAppBuildersEnabled: FeatureFn }
 
@@ -517,6 +519,8 @@ export class UserDB {
         global: true,
       },
       tenantId,
+      firstName: opts?.firstName,
+      lastName: opts?.lastName,
     }
     if (opts?.ssoId) {
       user.ssoId = opts.ssoId

packages/backend-core/tests/core/utilities/index.ts

@@ -4,3 +4,6 @@ export { generator } from "./structures"
 export * as testContainerUtils from "./testContainerUtils"
 export * as utils from "./utils"
 export * from "./jestUtils"
+import * as minio from "./minio"
+
+export const objectStoreTestProviders = { minio }

packages/backend-core/tests/core/utilities/minio.ts

@@ -0,0 +1,34 @@
+import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
+import env from "../../../src/environment"
+
+let container: StartedTestContainer | undefined
+
+class ObjectStoreWaitStrategy extends AbstractWaitStrategy {
+  async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
+    const logs = Wait.forListeningPorts()
+    await logs.waitUntilReady(container, boundPorts, startTime)
+  }
+}
+
+export async function start(): Promise<void> {
+  container = await new GenericContainer("minio/minio")
+    .withExposedPorts(9000)
+    .withCommand(["server", "/data"])
+    .withEnvironment({
+      MINIO_ACCESS_KEY: "budibase",
+      MINIO_SECRET_KEY: "budibase",
+    })
+    .withWaitStrategy(new ObjectStoreWaitStrategy().withStartupTimeout(30000))
+    .start()
+
+  const port = container.getMappedPort(9000)
+  env._set("MINIO_URL", `http://0.0.0.0:${port}`)
+}
+
+export async function stop() {
+  if (container) {
+    await container.stop()
+    container = undefined
+  }
+}

packages/bbui/package.json

@@ -21,7 +21,7 @@
     "@rollup/plugin-json": "^4.1.0",
     "@rollup/plugin-node-resolve": "^11.2.1",
     "postcss": "^8.2.9",
-    "rollup": "^2.45.2",
+    "rollup": "^4.9.6",
     "rollup-plugin-postcss": "^4.0.0",
     "rollup-plugin-svelte": "^7.1.0",
     "rollup-plugin-terser": "^7.0.2"

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -32,6 +32,7 @@
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import CodeEditor from "components/common/CodeEditor/CodeEditor.svelte"
   import BindingSidePanel from "components/common/bindings/BindingSidePanel.svelte"
+  import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
   import { BindingHelpers, BindingType } from "components/common/bindings/utils"
   import {
     bindingsToCompletions,
@@ -356,7 +357,8 @@
       value.customType !== "queryParams" &&
       value.customType !== "cron" &&
       value.customType !== "triggerSchema" &&
-      value.customType !== "automationFields"
+      value.customType !== "automationFields" &&
+      value.type !== "attachment"
     )
   }
 
@@ -372,6 +374,15 @@
       console.error(error)
     }
   })
+  const handleAttachmentParams = keyValuObj => {
+    let params = {}
+    if (keyValuObj?.length) {
+      for (let param of keyValuObj) {
+        params[param.url] = param.filename
+      }
+    }
+    return params
+  }
 </script>
 
 <div class="fields">
@@ -437,6 +448,33 @@
               value={inputData[key]}
               options={Object.keys(table?.schema || {})}
             />
+          {:else if value.type === "attachment"}
+            <div class="attachment-field-wrapper">
+              <div class="label-wrapper">
+                <Label>{label}</Label>
+              </div>
+              <div class="attachment-field-width">
+                <KeyValueBuilder
+                  on:change={e =>
+                    onChange(
+                      {
+                        detail: e.detail.map(({ name, value }) => ({
+                          url: name,
+                          filename: value,
+                        })),
+                      },
+                      key
+                    )}
+                  object={handleAttachmentParams(inputData[key])}
+                  allowJS
+                  {bindings}
+                  keyBindings
+                  customButtonText={"Add attachment"}
+                  keyPlaceholder={"URL"}
+                  valuePlaceholder={"Filename"}
+                />
+              </div>
+            </div>
           {:else if value.customType === "filters"}
             <ActionButton on:click={drawer.show}>Define filters</ActionButton>
             <Drawer bind:this={drawer} title="Filtering">
@@ -651,14 +689,22 @@
   }
 
   .block-field {
-    display: flex; /* Use Flexbox */
+    display: flex;
     justify-content: space-between;
-    flex-direction: row; /* Arrange label and field side by side */
+    flex-direction: row;
-    align-items: center; /* Align vertically in the center */
+    align-items: center;
-    gap: 10px; /* Add some space between label and field */
+    gap: 10px;
     flex: 1;
   }
 
+  .attachment-field-width {
+    margin-top: var(--spacing-xs);
+  }
+
+  .label-wrapper {
+    margin-top: var(--spacing-s);
+  }
+
   .test :global(.drawer) {
     width: 10000px !important;
   }

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -29,7 +29,11 @@
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import { getBindings } from "components/backend/DataTable/formula"
   import JSONSchemaModal from "./JSONSchemaModal.svelte"
-  import { FieldType, FieldSubtype, SourceName } from "@budibase/types"
+  import {
+    FieldType,
+    BBReferenceFieldSubType,
+    SourceName,
+  } from "@budibase/types"
   import RelationshipSelector from "components/common/RelationshipSelector.svelte"
   import { RowUtils } from "@budibase/frontend-core"
   import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
@@ -41,8 +45,6 @@
   const NUMBER_TYPE = FieldType.NUMBER
   const JSON_TYPE = FieldType.JSON
   const DATE_TYPE = FieldType.DATETIME
-  const USER_TYPE = FieldSubtype.USER
-  const USERS_TYPE = FieldSubtype.USERS
 
   const dispatch = createEventDispatcher()
   const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
@@ -263,9 +265,9 @@
       delete saveColumn.fieldName
     }
     if (isUsersColumn(saveColumn)) {
-      if (saveColumn.subtype === USER_TYPE) {
+      if (saveColumn.subtype === BBReferenceFieldSubType.USER) {
         saveColumn.relationshipType = RelationshipType.ONE_TO_MANY
-      } else if (saveColumn.subtype === USERS_TYPE) {
+      } else if (saveColumn.subtype === BBReferenceFieldSubType.USERS) {
         saveColumn.relationshipType = RelationshipType.MANY_TO_MANY
       }
     }
@@ -375,7 +377,7 @@
 
     const isUsers =
       editableColumn.type === FieldType.BB_REFERENCE &&
-      editableColumn.subtype === FieldSubtype.USERS
+      editableColumn.subtype === BBReferenceFieldSubType.USERS
 
     if (!externalTable) {
       return [
@@ -485,7 +487,9 @@
   function isUsersColumn(column) {
     return (
       column.type === FieldType.BB_REFERENCE &&
-      [FieldSubtype.USER, FieldSubtype.USERS].includes(column.subtype)
+      [BBReferenceFieldSubType.USER, BBReferenceFieldSubType.USERS].includes(
+        column.subtype
+      )
     )
   }
 
@@ -688,12 +692,14 @@
     >
   {:else if isUsersColumn(editableColumn) && datasource?.source !== SourceName.GOOGLE_SHEETS}
     <Toggle
-      value={editableColumn.subtype === FieldSubtype.USERS}
+      value={editableColumn.subtype === BBReferenceFieldSubType.USERS}
       on:change={e =>
         handleTypeChange(
           makeFieldId(
             FieldType.BB_REFERENCE,
-            e.detail ? FieldSubtype.USERS : FieldSubtype.USER
+            e.detail
+              ? BBReferenceFieldSubType.USERS
+              : BBReferenceFieldSubType.USER
           )
         )}
       disabled={!isCreating}

packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { FieldType, FieldSubtype } from "@budibase/types"
+  import { FieldType, BBReferenceFieldSubType } from "@budibase/types"
   import { Select, Toggle, Multiselect } from "@budibase/bbui"
   import { DB_TYPE_INTERNAL } from "constants/backend"
   import { API } from "api"
@@ -60,11 +60,11 @@
     },
     {
       label: "User",
-      value: `${FieldType.BB_REFERENCE}${FieldSubtype.USER}`,
+      value: `${FieldType.BB_REFERENCE}${BBReferenceFieldSubType.USER}`,
     },
     {
       label: "Users",
-      value: `${FieldType.BB_REFERENCE}${FieldSubtype.USERS}`,
+      value: `${FieldType.BB_REFERENCE}${BBReferenceFieldSubType.USERS}`,
     },
   ]
 

packages/builder/src/components/integration/KeyValueBuilder.svelte

@@ -35,6 +35,8 @@
   export let bindingDrawerLeft
   export let allowHelpers = true
   export let customButtonText = null
+  export let keyBindings = false
+  export let allowJS = false
   export let compare = (option, value) => option === value
 
   let fields = Object.entries(object || {}).map(([name, value]) => ({
@@ -116,12 +118,23 @@
     class:readOnly-menu={readOnly && showMenu}
   >
     {#each fields as field, idx}
-      <Input
+      {#if keyBindings}
+        <DrawerBindableInput
+          {bindings}
           placeholder={keyPlaceholder}
-        readonly={readOnly}
+          on:blur={e => {
-        bind:value={field.name}
+            field.name = e.detail
-        on:blur={changed}
+            changed()
+          }}
+          disabled={readOnly}
+          value={field.name}
+          {allowJS}
+          {allowHelpers}
+          drawerLeft={bindingDrawerLeft}
         />
+      {:else}
+        <Input readonly={readOnly} bind:value={field.name} on:blur={changed} />
+      {/if}
       {#if isJsonArray(field.value)}
         <Select readonly={true} value="Array" options={["Array"]} />
       {:else if options}
@@ -134,14 +147,14 @@
       {:else if bindings && bindings.length}
         <DrawerBindableInput
           {bindings}
-          placeholder="Value"
+          placeholder={valuePlaceholder}
           on:blur={e => {
             field.value = e.detail
             changed()
           }}
           disabled={readOnly}
           value={field.value}
-          allowJS={false}
+          {allowJS}
           {allowHelpers}
           drawerLeft={bindingDrawerLeft}
         />

packages/builder/src/constants/backend/index.js

@@ -1,6 +1,6 @@
 import {
   FieldType,
-  FieldSubtype,
+  BBReferenceFieldSubType,
   INTERNAL_TABLE_SOURCE_ID,
   AutoFieldSubType,
   Hosting,
@@ -160,13 +160,13 @@ export const FIELDS = {
   USER: {
     name: "User",
     type: FieldType.BB_REFERENCE,
-    subtype: FieldSubtype.USER,
+    subtype: BBReferenceFieldSubType.USER,
     icon: TypeIconMap[FieldType.USER],
   },
   USERS: {
     name: "Users",
     type: FieldType.BB_REFERENCE,
-    subtype: FieldSubtype.USERS,
+    subtype: BBReferenceFieldSubType.USERS,
     icon: TypeIconMap[FieldType.USERS],
     constraints: {
       type: "array",

packages/builder/src/dataBinding.js

@@ -22,6 +22,7 @@ import {
   isJSBinding,
   decodeJSBinding,
   encodeJSBinding,
+  getJsHelperList,
 } from "@budibase/string-templates"
 import { TableNames } from "./constants"
 import { JSONUtils, Constants } from "@budibase/frontend-core"
@@ -1210,9 +1211,32 @@ const shouldReplaceBinding = (currentValue, from, convertTo, binding) => {
   if (!currentValue?.includes(from)) {
     return false
   }
-  if (convertTo === "readableBinding") {
+  // some cases we have the same binding for readable/runtime, specific logic for this
-    // Dont replace if the value already matches the readable binding
+  const sameBindings = binding.runtimeBinding.includes(binding.readableBinding)
+  const convertingToReadable = convertTo === "readableBinding"
+  const helperNames = Object.keys(getJsHelperList())
+  const matchedHelperNames = helperNames.filter(
+    name => name.includes(from) && currentValue.includes(name)
+  )
+  // edge case - if the binding is part of a helper it may accidentally replace it
+  if (matchedHelperNames.length > 0) {
+    const indexStart = currentValue.indexOf(from),
+      indexEnd = indexStart + from.length
+    for (let helperName of matchedHelperNames) {
+      const helperIndexStart = currentValue.indexOf(helperName),
+        helperIndexEnd = helperIndexStart + helperName.length
+      if (indexStart >= helperIndexStart && indexEnd <= helperIndexEnd) {
+        return false
+      }
+    }
+  }
+
+  if (convertingToReadable && !sameBindings) {
+    // Don't replace if the value already matches the readable binding
     return currentValue.indexOf(binding.readableBinding) === -1
+  } else if (convertingToReadable) {
+    // if the runtime and readable bindings are very similar we have to assume it should be replaced
+    return true
   }
   // remove all the spaces, if the input is surrounded by spaces e.g. [ Auto ID ] then
   // this makes sure it is detected

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUIDrawer.svelte

@@ -189,6 +189,7 @@
                 <Select
                   options={settingOptions}
                   bind:value={condition.setting}
+                  on:change={() => delete condition.settingValue}
                 />
                 <div>TO</div>
                 {#if definition}

packages/cli/src/backups/objectStore.ts

@@ -9,7 +9,7 @@ const {
   ObjectStore,
   retrieve,
   uploadDirectory,
-  makeSureBucketExists,
+  createBucketIfNotExists,
 } = objectStore
 
 const bucketList = Object.values(ObjectStoreBuckets)
@@ -61,7 +61,7 @@ export async function importObjects() {
   let count = 0
   for (let bucket of buckets) {
     const client = ObjectStore(bucket)
-    await makeSureBucketExists(client, bucket)
+    await createBucketIfNotExists(client, bucket)
     const files = await uploadDirectory(bucket, join(path, bucket), "/")
     count += files.length
     bar.update(count)

packages/cli/src/hosting/utils.ts

@@ -54,11 +54,9 @@ export async function downloadDockerCompose() {
 
 export async function checkDockerConfigured() {
   const error =
-    "docker/docker-compose has not been installed, please follow instructions at: https://docs.budibase.com/docs/docker-compose"
+    "docker has not been installed, please follow instructions at: https://docs.budibase.com/docs/docker-compose"
   const docker = await lookpath("docker")
-  const compose = await lookpath("docker-compose")
+  if (!docker) {
-  const composeV2 = await lookpath("docker compose")
-  if (!docker || (!compose && !composeV2)) {
     throw error
   }
 }

packages/client/manifest.json

@@ -6973,6 +6973,12 @@
         "key": "stripeRows",
         "defaultValue": false
       },
+      {
+        "type": "boolean",
+        "label": "Quiet",
+        "key": "quiet",
+        "defaultValue": false
+      },
       {
         "section": true,
         "name": "Columns",

packages/client/src/components/ClientApp.svelte

@@ -119,6 +119,7 @@
   {/if}
 </svelte:head>
 
+{#if dataLoaded}
   <div
     id="spectrum-root"
     lang="en"
@@ -253,6 +254,7 @@
     {/if}
   </div>
   <KeyboardManager />
+{/if}
 
 <style>
   #spectrum-root {

packages/client/src/components/app/GridBlock.svelte

@@ -11,6 +11,7 @@
   export let allowEditRows = true
   export let allowDeleteRows = true
   export let stripeRows = false
+  export let quiet = false
   export let initialFilter = null
   export let initialSortColumn = null
   export let initialSortOrder = null
@@ -141,6 +142,7 @@
       datasource={table}
       {API}
       {stripeRows}
+      {quiet}
       {initialFilter}
       {initialSortColumn}
       {initialSortOrder}

packages/frontend-core/src/components/FilterBuilder.svelte

@@ -67,6 +67,11 @@
 
   const removeFilter = id => {
     filters = filters.filter(field => field.id !== id)
+
+    // Clear all filters when no fields are specified
+    if (filters.length === 1 && filters[0].onEmptyFilter) {
+      filters = []
+    }
   }
 
   const duplicateFilter = id => {

packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte

@@ -1,7 +1,7 @@
 <script>
   import { getContext } from "svelte"
   import RelationshipCell from "./RelationshipCell.svelte"
-  import { FieldSubtype, RelationshipType } from "@budibase/types"
+  import { BBReferenceFieldSubType, RelationshipType } from "@budibase/types"
 
   export let api
 
@@ -13,13 +13,16 @@
     // This is not really used, just adding some content to be able to render the relationship cell
     tableId: "external",
     relationshipType:
-      subtype === FieldSubtype.USER
+      subtype === BBReferenceFieldSubType.USER
         ? RelationshipType.ONE_TO_MANY
         : RelationshipType.MANY_TO_MANY,
   }
 
   async function searchFunction(searchParams) {
-    if (subtype !== FieldSubtype.USER && subtype !== FieldSubtype.USERS) {
+    if (
+      subtype !== BBReferenceFieldSubType.USER &&
+      subtype !== BBReferenceFieldSubType.USERS
+    ) {
       throw `Search for '${subtype}' not implemented`
     }
 

packages/frontend-core/src/components/grid/cells/GridCell.svelte

@@ -43,6 +43,9 @@
   on:mouseup
   on:click
   on:contextmenu
+  on:touchstart
+  on:touchend
+  on:touchcancel
   {style}
 >
   {#if error}

packages/frontend-core/src/components/grid/cells/HeaderCell.svelte

@@ -18,7 +18,6 @@
 
   export let column
   export let idx
-  export let orderable = true
 
   const {
     reorder,
@@ -66,6 +65,7 @@
   $: resetSearchValue(column.name)
   $: searching = searchValue != null
   $: debouncedUpdateFilter(searchValue)
+  $: orderable = !column.primaryDisplay
 
   const getSortingLabels = type => {
     switch (type) {
@@ -112,16 +112,17 @@
   }
 
   const onMouseDown = e => {
-    if (e.button === 0 && orderable) {
+    if ((e.touches?.length || e.button === 0) && orderable) {
       timeout = setTimeout(() => {
         reorder.actions.startReordering(column.name, e)
       }, 200)
     }
   }
 
-  const onMouseUp = e => {
+  const onMouseUp = () => {
-    if (e.button === 0 && orderable) {
+    if (timeout) {
       clearTimeout(timeout)
+      timeout = null
     }
   }
 
@@ -258,6 +259,9 @@
   <GridCell
     on:mousedown={onMouseDown}
     on:mouseup={onMouseUp}
+    on:touchstart={onMouseDown}
+    on:touchend={onMouseUp}
+    on:touchcancel={onMouseUp}
     on:contextmenu={onContextMenu}
     width={column.width}
     left={column.left}
@@ -347,7 +351,8 @@
       <MenuItem
         icon="Label"
         on:click={makeDisplayColumn}
-        disabled={idx === "sticky" || !canBeDisplayColumn(column.schema.type)}
+        disabled={column.primaryDisplay ||
+          !canBeDisplayColumn(column.schema.type)}
       >
         Use as display column
       </MenuItem>
@@ -378,7 +383,7 @@
         Move right
       </MenuItem>
       <MenuItem
-        disabled={idx === "sticky" || !$config.showControls}
+        disabled={column.primaryDisplay || !$config.showControls}
         icon="VisibilityOff"
         on:click={hideColumn}
       >

packages/frontend-core/src/components/grid/controls/HideColumnsButton.svelte

@@ -81,6 +81,7 @@
           size="S"
           value={column.visible}
           on:change={e => toggleVisibility(column, e.detail)}
+          disabled={column.primaryDisplay}
         />
       {/each}
     </div>

packages/frontend-core/src/components/grid/controls/MigrationModal.svelte

@@ -7,7 +7,11 @@
   } from "@budibase/bbui"
   import { getContext } from "svelte"
   import { ValidColumnNameRegex } from "@budibase/shared-core"
-  import { FieldSubtype, FieldType, RelationshipType } from "@budibase/types"
+  import {
+    BBReferenceFieldSubType,
+    FieldType,
+    RelationshipType,
+  } from "@budibase/types"
 
   const { API, definition, rows } = getContext("grid")
 
@@ -29,9 +33,9 @@
   }
 
   const migrateUserColumn = async () => {
-    let subtype = FieldSubtype.USERS
+    let subtype = BBReferenceFieldSubType.USERS
     if (column.schema.relationshipType === RelationshipType.ONE_TO_MANY) {
-      subtype = FieldSubtype.USER
+      subtype = BBReferenceFieldSubType.USER
     }
 
     try {

packages/frontend-core/src/components/grid/layout/Grid.svelte

@@ -40,6 +40,7 @@
   export let canSaveSchema = true
   export let canSelectRows = false
   export let stripeRows = false
+  export let quiet = false
   export let collaboration = true
   export let showAvatars = true
   export let showControls = true
@@ -93,6 +94,7 @@
     canSaveSchema,
     canSelectRows,
     stripeRows,
+    quiet,
     collaboration,
     showAvatars,
     showControls,
@@ -126,6 +128,7 @@
   class:is-resizing={$isResizing}
   class:is-reordering={$isReordering}
   class:stripe={stripeRows}
+  class:quiet
   on:mouseenter={() => gridFocused.set(true)}
   on:mouseleave={() => gridFocused.set(false)}
   style="--row-height:{$rowHeight}px; --default-row-height:{DefaultRowHeight}px; --gutter-width:{GutterWidth}px; --max-cell-render-height:{MaxCellRenderHeight}px; --max-cell-render-width-overflow:{MaxCellRenderWidthOverflow}px; --content-lines:{$contentLines};"
@@ -333,4 +336,9 @@
   .grid-data-outer :global(.spectrum-Checkbox-partialCheckmark) {
     transition: none;
   }
+
+  /* Overrides */
+  .grid.quiet :global(.grid-data-content .row > .cell:not(:last-child)) {
+    border-right: none;
+  }
 </style>

packages/frontend-core/src/components/grid/layout/NewRow.svelte

@@ -30,6 +30,7 @@
     refreshing,
     config,
     filter,
+    inlineFilters,
     columnRenderMap,
   } = getContext("grid")
 
@@ -157,7 +158,11 @@
 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <TempTooltip
   text="Click here to create your first row"
-  condition={hasNoRows && $loaded && !$filter?.length && !$refreshing}
+  condition={hasNoRows &&
+    $loaded &&
+    !$filter?.length &&
+    !$inlineFilters?.length &&
+    !$refreshing}
   type={TooltipType.Info}
 >
   {#if !visible && !selectedRowCount && $config.canAddRows}

packages/frontend-core/src/components/grid/lib/utils.js

@@ -20,3 +20,10 @@ export const getColumnIcon = column => {
 
   return result || "Text"
 }
+
+export const parseEventLocation = e => {
+  return {
+    x: e.clientX ?? e.touches?.[0]?.clientX,
+    y: e.clientY ?? e.touches?.[0]?.clientY,
+  }
+}

packages/frontend-core/src/components/grid/overlays/ResizeOverlay.svelte

@@ -21,6 +21,7 @@
       class="resize-slider"
       class:visible={activeColumn === $stickyColumn.name}
       on:mousedown={e => resize.actions.startResizing($stickyColumn, e)}
+      on:touchstart={e => resize.actions.startResizing($stickyColumn, e)}
       on:dblclick={() => resize.actions.resetSize($stickyColumn)}
       style="left:{GutterWidth + $stickyColumn.width}px;"
     >
@@ -32,6 +33,7 @@
       class="resize-slider"
       class:visible={activeColumn === column.name}
       on:mousedown={e => resize.actions.startResizing(column, e)}
+      on:touchstart={e => resize.actions.startResizing(column, e)}
       on:dblclick={() => resize.actions.resetSize(column)}
       style={getStyle(column, offset, $scrollLeft)}
     >

packages/frontend-core/src/components/grid/overlays/ScrollOverlay.svelte

@@ -2,6 +2,7 @@
   import { getContext } from "svelte"
   import { domDebounce } from "../../../utils/utils"
   import { DefaultRowHeight, ScrollBarSize } from "../lib/constants"
+  import { parseEventLocation } from "../lib/utils"
 
   const {
     scroll,
@@ -53,17 +54,10 @@
     }
   }
 
-  const getLocation = e => {
-    return {
-      y: e.touches?.[0]?.clientY ?? e.clientY,
-      x: e.touches?.[0]?.clientX ?? e.clientX,
-    }
-  }
-
   // V scrollbar drag handlers
   const startVDragging = e => {
     e.preventDefault()
-    initialMouse = getLocation(e).y
+    initialMouse = parseEventLocation(e).y
     initialScroll = $scrollTop
     document.addEventListener("mousemove", moveVDragging)
     document.addEventListener("touchmove", moveVDragging)
@@ -73,7 +67,7 @@
     closeMenu()
   }
   const moveVDragging = domDebounce(e => {
-    const delta = getLocation(e).y - initialMouse
+    const delta = parseEventLocation(e).y - initialMouse
     const weight = delta / availHeight
     const newScrollTop = initialScroll + weight * $maxScrollTop
     scroll.update(state => ({
@@ -92,7 +86,7 @@
   // H scrollbar drag handlers
   const startHDragging = e => {
     e.preventDefault()
-    initialMouse = getLocation(e).x
+    initialMouse = parseEventLocation(e).x
     initialScroll = $scrollLeft
     document.addEventListener("mousemove", moveHDragging)
     document.addEventListener("touchmove", moveHDragging)
@@ -102,7 +96,7 @@
     closeMenu()
   }
   const moveHDragging = domDebounce(e => {
-    const delta = getLocation(e).x - initialMouse
+    const delta = parseEventLocation(e).x - initialMouse
     const weight = delta / availWidth
     const newScrollLeft = initialScroll + weight * $maxScrollLeft
     scroll.update(state => ({

packages/frontend-core/src/components/grid/stores/columns.js

@@ -48,22 +48,28 @@ export const createStores = () => {
 export const deriveStores = context => {
   const { columns, stickyColumn } = context
 
-  // Derive if we have any normal columns
+  // Quick access to all columns
-  const hasNonAutoColumn = derived(
+  const allColumns = derived(
     [columns, stickyColumn],
     ([$columns, $stickyColumn]) => {
       let allCols = $columns || []
       if ($stickyColumn) {
         allCols = [...allCols, $stickyColumn]
       }
-      const normalCols = allCols.filter(column => {
+      return allCols
-        return !column.schema?.autocolumn
-      })
-      return normalCols.length > 0
     }
   )
 
+  // Derive if we have any normal columns
+  const hasNonAutoColumn = derived(allColumns, $allColumns => {
+    const normalCols = $allColumns.filter(column => {
+      return !column.schema?.autocolumn
+    })
+    return normalCols.length > 0
+  })
+
   return {
+    allColumns,
     hasNonAutoColumn,
   }
 }
@@ -142,24 +148,26 @@ export const createActions = context => {
 }
 
 export const initialise = context => {
-  const { definition, columns, stickyColumn, enrichedSchema } = context
+  const {
+    definition,
+    columns,
+    stickyColumn,
+    allColumns,
+    enrichedSchema,
+    compact,
+  } = context
 
   // Merge new schema fields with existing schema in order to preserve widths
-  enrichedSchema.subscribe($enrichedSchema => {
+  const processColumns = $enrichedSchema => {
     if (!$enrichedSchema) {
       columns.set([])
       stickyColumn.set(null)
       return
     }
     const $definition = get(definition)
-    const $columns = get(columns)
+    const $allColumns = get(allColumns)
     const $stickyColumn = get(stickyColumn)
-
+    const $compact = get(compact)
-    // Generate array of all columns to easily find pre-existing columns
-    let allColumns = $columns || []
-    if ($stickyColumn) {
-      allColumns.push($stickyColumn)
-    }
 
     // Find primary display
     let primaryDisplay
@@ -171,7 +179,7 @@ export const initialise = context => {
     // Get field list
     let fields = []
     Object.keys($enrichedSchema).forEach(field => {
-      if (field !== primaryDisplay) {
+      if ($compact || field !== primaryDisplay) {
         fields.push(field)
       }
     })
@@ -181,7 +189,7 @@ export const initialise = context => {
       fields
         .map(field => {
           const fieldSchema = $enrichedSchema[field]
-          const oldColumn = allColumns?.find(x => x.name === field)
+          const oldColumn = $allColumns?.find(x => x.name === field)
           return {
             name: field,
             label: fieldSchema.displayName || field,
@@ -189,9 +197,18 @@ export const initialise = context => {
             width: fieldSchema.width || oldColumn?.width || DefaultColumnWidth,
             visible: fieldSchema.visible ?? true,
             order: fieldSchema.order ?? oldColumn?.order,
+            primaryDisplay: field === primaryDisplay,
           }
         })
         .sort((a, b) => {
+          // If we don't have a pinned column then primary display will be in
+          // the normal columns list, and should be first
+          if (a.name === primaryDisplay) {
+            return -1
+          } else if (b.name === primaryDisplay) {
+            return 1
+          }
+
           // Sort by order first
           const orderA = a.order
           const orderB = b.order
@@ -214,12 +231,12 @@ export const initialise = context => {
     )
 
     // Update sticky column
-    if (!primaryDisplay) {
+    if ($compact || !primaryDisplay) {
       stickyColumn.set(null)
       return
     }
     const stickySchema = $enrichedSchema[primaryDisplay]
-    const oldStickyColumn = allColumns?.find(x => x.name === primaryDisplay)
+    const oldStickyColumn = $allColumns?.find(x => x.name === primaryDisplay)
     stickyColumn.set({
       name: primaryDisplay,
       label: stickySchema.displayName || primaryDisplay,
@@ -228,6 +245,13 @@ export const initialise = context => {
       visible: true,
       order: 0,
       left: GutterWidth,
-    })
+      primaryDisplay: true,
     })
   }
+
+  // Process columns when schema changes
+  enrichedSchema.subscribe(processColumns)
+
+  // Process columns when compact flag changes
+  compact.subscribe(() => processColumns(get(enrichedSchema)))
+}

packages/frontend-core/src/components/grid/stores/reorder.js

@@ -1,4 +1,5 @@
 import { get, writable, derived } from "svelte/store"
+import { parseEventLocation } from "../lib/utils"
 
 const reorderInitialState = {
   sourceColumn: null,
@@ -33,6 +34,7 @@ export const createActions = context => {
     stickyColumn,
     ui,
     maxScrollLeft,
+    width,
   } = context
 
   let autoScrollInterval
@@ -55,6 +57,11 @@ export const createActions = context => {
         x: 0,
         column: $stickyColumn.name,
       })
+    } else if (!$visibleColumns[0].primaryDisplay) {
+      breakpoints.unshift({
+        x: 0,
+        column: null,
+      })
     }
 
     // Update state
@@ -69,6 +76,9 @@ export const createActions = context => {
     // Add listeners to handle mouse movement
     document.addEventListener("mousemove", onReorderMouseMove)
     document.addEventListener("mouseup", stopReordering)
+    document.addEventListener("touchmove", onReorderMouseMove)
+    document.addEventListener("touchend", stopReordering)
+    document.addEventListener("touchcancel", stopReordering)
 
     // Trigger a move event immediately so ensure a candidate column is chosen
     onReorderMouseMove(e)
@@ -77,7 +87,7 @@ export const createActions = context => {
   // Callback when moving the mouse when reordering columns
   const onReorderMouseMove = e => {
     // Immediately handle the current position
-    const x = e.clientX
+    const { x } = parseEventLocation(e)
     reorder.update(state => ({
       ...state,
       latestX: x,
@@ -86,7 +96,7 @@ export const createActions = context => {
 
     // Check if we need to start auto-scrolling
     const $reorder = get(reorder)
-    const proximityCutoff = 140
+    const proximityCutoff = Math.min(140, get(width) / 6)
     const speedFactor = 8
     const rightProximity = Math.max(0, $reorder.gridLeft + $reorder.width - x)
     const leftProximity = Math.max(0, x - $reorder.gridLeft)
@@ -158,21 +168,24 @@ export const createActions = context => {
     // Ensure auto-scrolling is stopped
     stopAutoScroll()
 
-    // Swap position of columns
-    let { sourceColumn, targetColumn } = get(reorder)
-    moveColumn(sourceColumn, targetColumn)
-
-    // Reset state
-    reorder.set(reorderInitialState)
-
     // Remove event handlers
     document.removeEventListener("mousemove", onReorderMouseMove)
     document.removeEventListener("mouseup", stopReordering)
+    document.removeEventListener("touchmove", onReorderMouseMove)
+    document.removeEventListener("touchend", stopReordering)
+    document.removeEventListener("touchcancel", stopReordering)
 
-    // Save column changes
+    // Ensure there's actually a change
+    let { sourceColumn, targetColumn } = get(reorder)
+    if (sourceColumn !== targetColumn) {
+      moveColumn(sourceColumn, targetColumn)
       await columns.actions.saveChanges()
     }
 
+    // Reset state
+    reorder.set(reorderInitialState)
+  }
+
   // Moves a column after another columns.
   // An undefined target column will move the source to index 0.
   const moveColumn = (sourceColumn, targetColumn) => {
@@ -185,8 +198,7 @@ export const createActions = context => {
       if (--targetIdx < sourceIdx) {
         targetIdx++
       }
-      state.splice(targetIdx, 0, removed[0])
+      return state.toSpliced(targetIdx, 0, removed[0])
-      return state.slice()
     })
   }
 

packages/frontend-core/src/components/grid/stores/resize.js

@@ -1,5 +1,6 @@
 import { writable, get, derived } from "svelte/store"
 import { MinColumnWidth, DefaultColumnWidth } from "../lib/constants"
+import { parseEventLocation } from "../lib/utils"
 
 const initialState = {
   initialMouseX: null,
@@ -24,8 +25,11 @@ export const createActions = context => {
 
   // Starts resizing a certain column
   const startResizing = (column, e) => {
+    const { x } = parseEventLocation(e)
+
     // Prevent propagation to stop reordering triggering
     e.stopPropagation()
+    e.preventDefault()
     ui.actions.blur()
 
     // Find and cache index
@@ -39,7 +43,7 @@ export const createActions = context => {
       width: column.width,
       left: column.left,
       initialWidth: column.width,
-      initialMouseX: e.clientX,
+      initialMouseX: x,
       column: column.name,
       columnIdx,
     })
@@ -47,12 +51,16 @@ export const createActions = context => {
     // Add mouse event listeners to handle resizing
     document.addEventListener("mousemove", onResizeMouseMove)
     document.addEventListener("mouseup", stopResizing)
+    document.addEventListener("touchmove", onResizeMouseMove)
+    document.addEventListener("touchend", stopResizing)
+    document.addEventListener("touchcancel", stopResizing)
   }
 
   // Handler for moving the mouse to resize columns
   const onResizeMouseMove = e => {
     const { initialMouseX, initialWidth, width, columnIdx } = get(resize)
-    const dx = e.clientX - initialMouseX
+    const { x } = parseEventLocation(e)
+    const dx = x - initialMouseX
     const newWidth = Math.round(Math.max(MinColumnWidth, initialWidth + dx))
 
     // Ignore small changes
@@ -87,6 +95,9 @@ export const createActions = context => {
     resize.set(initialState)
     document.removeEventListener("mousemove", onResizeMouseMove)
     document.removeEventListener("mouseup", stopResizing)
+    document.removeEventListener("touchmove", onResizeMouseMove)
+    document.removeEventListener("touchend", stopResizing)
+    document.removeEventListener("touchcancel", stopResizing)
 
     // Persist width if it changed
     if ($resize.width !== $resize.initialWidth) {

packages/frontend-core/src/components/grid/stores/ui.js

@@ -98,7 +98,7 @@ export const deriveStores = context => {
 
   // Derive whether we should use the compact UI, depending on width
   const compact = derived([stickyColumn, width], ([$stickyColumn, $width]) => {
-    return ($stickyColumn?.width || 0) + $width + GutterWidth < 1100
+    return ($stickyColumn?.width || 0) + $width + GutterWidth < 800
   })
 
   return {

packages/frontend-core/src/constants.js

@@ -4,7 +4,7 @@
 export { OperatorOptions, SqlNumberTypeRangeMap } from "@budibase/shared-core"
 export { Feature as Features } from "@budibase/types"
 import { BpmCorrelationKey } from "@budibase/shared-core"
-import { FieldType, FieldTypeSubtypes } from "@budibase/types"
+import { FieldType, BBReferenceFieldSubType } from "@budibase/types"
 
 // Cookie names
 export const Cookies = {
@@ -134,7 +134,7 @@ export const TypeIconMap = {
   [FieldType.USER]: "User",
   [FieldType.USERS]: "UserGroup",
   [FieldType.BB_REFERENCE]: {
-    [FieldTypeSubtypes.BB_REFERENCE.USER]: "User",
+    [BBReferenceFieldSubType.USER]: "User",
-    [FieldTypeSubtypes.BB_REFERENCE.USERS]: "UserGroup",
+    [BBReferenceFieldSubType.USERS]: "UserGroup",
   },
 }

packages/pro

@@ -1 +1 @@
-Subproject commit 06b1064f7e2f7cac5d4bef2ee999796a2a1f0f2c
+Subproject commit 479879246aac5dd3073cc695945c62c41fae5b0e

packages/server/package.json

@@ -61,14 +61,17 @@
     "@google-cloud/firestore": "6.8.0",
     "@koa/router": "8.0.8",
     "@socket.io/redis-adapter": "^8.2.1",
+    "@types/xml2js": "^0.4.14",
     "airtable": "0.10.1",
     "arangojs": "7.2.0",
     "archiver": "7.0.1",
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
+    "bl": "^6.0.12",
     "bull": "4.10.1",
     "chokidar": "3.5.3",
+    "content-disposition": "^0.5.4",
     "cookies": "0.8.0",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",
@@ -95,7 +98,7 @@
     "memorystream": "0.3.1",
     "mongodb": "^6.3.0",
     "mssql": "10.0.1",
-    "mysql2": "3.5.2",
+    "mysql2": "3.9.7",
     "node-fetch": "2.6.7",
     "object-sizeof": "2.6.1",
     "open": "8.4.0",

packages/server/scripts/integrations/mssql/docker-compose.yaml

@@ -4,6 +4,7 @@ services:
   # user: sa
   # database: master
   mssql:
+    # platform: linux/amd64
     image: bb/mssql
     build:
       context: .

packages/server/specs/generate.ts

@@ -76,7 +76,7 @@ function writeFile(output: any, filename: string) {
     console.log(`Wrote spec to ${path}`)
     return path
   } catch (err) {
-    console.error(err)
+    console.error("Error writing spec file", err)
   }
 }
 

packages/server/src/api/routes/tests/datasource.spec.ts

@@ -6,7 +6,19 @@ import sdk from "../../../sdk"
 
 import tk from "timekeeper"
 import { mocks } from "@budibase/backend-core/tests"
-import { QueryPreview, SourceName } from "@budibase/types"
+import {
+  Datasource,
+  FieldSchema,
+  BBReferenceFieldSubType,
+  FieldType,
+  QueryPreview,
+  RelationshipType,
+  SourceName,
+  Table,
+  TableSchema,
+} from "@budibase/types"
+import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
+import { tableForDatasource } from "../../../tests/utilities/structures"
 
 tk.freeze(mocks.date.MOCK_DATE)
 
@@ -223,4 +235,152 @@ describe("/datasources", () => {
       })
     })
   })
+
+  describe.each([
+    [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+    [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+    [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+    [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  ])("fetch schema (%s)", (_, dsProvider) => {
+    beforeAll(async () => {
+      datasource = await config.api.datasource.create(await dsProvider)
+    })
+
+    it("fetching schema will not drop tables or columns", async () => {
+      const datasourceId = datasource!._id!
+
+      const simpleTable = await config.api.table.save(
+        tableForDatasource(datasource, {
+          name: "simple",
+          schema: {
+            name: {
+              name: "name",
+              type: FieldType.STRING,
+            },
+          },
+        })
+      )
+
+      type SupportedSqlTypes =
+        | FieldType.STRING
+        | FieldType.BARCODEQR
+        | FieldType.LONGFORM
+        | FieldType.OPTIONS
+        | FieldType.DATETIME
+        | FieldType.NUMBER
+        | FieldType.BOOLEAN
+        | FieldType.FORMULA
+        | FieldType.BIGINT
+        | FieldType.BB_REFERENCE
+        | FieldType.LINK
+        | FieldType.ARRAY
+
+      const fullSchema: {
+        [type in SupportedSqlTypes]: FieldSchema & { type: type }
+      } = {
+        [FieldType.STRING]: {
+          name: "string",
+          type: FieldType.STRING,
+          constraints: {
+            presence: true,
+          },
+        },
+        [FieldType.LONGFORM]: {
+          name: "longform",
+          type: FieldType.LONGFORM,
+        },
+        [FieldType.OPTIONS]: {
+          name: "options",
+          type: FieldType.OPTIONS,
+          constraints: {
+            presence: { allowEmpty: false },
+          },
+        },
+        [FieldType.NUMBER]: {
+          name: "number",
+          type: FieldType.NUMBER,
+        },
+        [FieldType.BOOLEAN]: {
+          name: "boolean",
+          type: FieldType.BOOLEAN,
+        },
+        [FieldType.ARRAY]: {
+          name: "array",
+          type: FieldType.ARRAY,
+        },
+        [FieldType.DATETIME]: {
+          name: "datetime",
+          type: FieldType.DATETIME,
+          dateOnly: true,
+          timeOnly: false,
+        },
+        [FieldType.LINK]: {
+          name: "link",
+          type: FieldType.LINK,
+          tableId: simpleTable._id!,
+          relationshipType: RelationshipType.ONE_TO_MANY,
+          fieldName: "link",
+        },
+        [FieldType.FORMULA]: {
+          name: "formula",
+          type: FieldType.FORMULA,
+          formula: "any formula",
+        },
+        [FieldType.BARCODEQR]: {
+          name: "barcodeqr",
+          type: FieldType.BARCODEQR,
+        },
+        [FieldType.BIGINT]: {
+          name: "bigint",
+          type: FieldType.BIGINT,
+        },
+        [FieldType.BB_REFERENCE]: {
+          name: "bb_reference",
+          type: FieldType.BB_REFERENCE,
+          subtype: BBReferenceFieldSubType.USERS,
+        },
+      }
+
+      await config.api.table.save(
+        tableForDatasource(datasource, {
+          name: "full",
+          schema: fullSchema,
+        })
+      )
+
+      const persisted = await config.api.datasource.get(datasourceId)
+      await config.api.datasource.fetchSchema(datasourceId)
+
+      const updated = await config.api.datasource.get(datasourceId)
+      const expected: Datasource = {
+        ...persisted,
+        entities:
+          persisted?.entities &&
+          Object.entries(persisted.entities).reduce<Record<string, Table>>(
+            (acc, [tableName, table]) => {
+              acc[tableName] = {
+                ...table,
+                primaryDisplay: expect.not.stringMatching(
+                  new RegExp(`^${table.primaryDisplay || ""}$`)
+                ),
+                schema: Object.entries(table.schema).reduce<TableSchema>(
+                  (acc, [fieldName, field]) => {
+                    acc[fieldName] = expect.objectContaining({
+                      ...field,
+                    })
+                    return acc
+                  },
+                  {}
+                ),
+              }
+              return acc
+            },
+            {}
+          ),
+
+        _rev: expect.any(String),
+      }
+      expect(updated).toEqual(expected)
+    })
+  })
 })

packages/server/src/api/routes/tests/row.spec.ts

@@ -13,7 +13,7 @@ import {
   DeleteRow,
   FieldSchema,
   FieldType,
-  FieldTypeSubtypes,
+  BBReferenceFieldSubType,
   FormulaType,
   INTERNAL_TABLE_SOURCE_ID,
   NumberFieldMetadata,
@@ -1015,12 +1015,12 @@ describe.each([
         user: {
           name: "user",
           type: FieldType.BB_REFERENCE,
-          subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+          subtype: BBReferenceFieldSubType.USER,
         },
         users: {
           name: "users",
           type: FieldType.BB_REFERENCE,
-          subtype: FieldTypeSubtypes.BB_REFERENCE.USERS,
+          subtype: BBReferenceFieldSubType.USERS,
         },
       }),
       () => config.createUser(),

packages/server/src/api/routes/tests/table.spec.ts

@@ -2,7 +2,7 @@ import { context, events } from "@budibase/backend-core"
 import {
   AutoFieldSubType,
   Datasource,
-  FieldSubtype,
+  BBReferenceFieldSubType,
   FieldType,
   INTERNAL_TABLE_SOURCE_ID,
   InternalTable,
@@ -34,7 +34,7 @@ describe.each([
   [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
   [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
 ])("/tables (%s)", (_, dsProvider) => {
-  let isInternal: boolean
+  const isInternal: boolean = !dsProvider
   let datasource: Datasource | undefined
   let config = setup.getConfig()
 
@@ -44,9 +44,6 @@ describe.each([
     await config.init()
     if (dsProvider) {
       datasource = await config.api.datasource.create(await dsProvider)
-      isInternal = false
-    } else {
-      isInternal = true
     }
   })
 
@@ -500,7 +497,7 @@ describe.each([
         newColumn: {
           name: "user column",
           type: FieldType.BB_REFERENCE,
-          subtype: FieldSubtype.USER,
+          subtype: BBReferenceFieldSubType.USER,
         },
       })
 
@@ -565,7 +562,7 @@ describe.each([
         newColumn: {
           name: "user column",
           type: FieldType.BB_REFERENCE,
-          subtype: FieldSubtype.USERS,
+          subtype: BBReferenceFieldSubType.USERS,
         },
       })
 
@@ -617,7 +614,7 @@ describe.each([
         newColumn: {
           name: "user column",
           type: FieldType.BB_REFERENCE,
-          subtype: FieldSubtype.USERS,
+          subtype: BBReferenceFieldSubType.USERS,
         },
       })
 
@@ -672,7 +669,7 @@ describe.each([
         newColumn: {
           name: "user column",
           type: FieldType.BB_REFERENCE,
-          subtype: FieldSubtype.USERS,
+          subtype: BBReferenceFieldSubType.USERS,
         },
       })
 
@@ -731,7 +728,7 @@ describe.each([
             newColumn: {
               name: "",
               type: FieldType.BB_REFERENCE,
-              subtype: FieldSubtype.USERS,
+              subtype: BBReferenceFieldSubType.USERS,
             },
           },
           { status: 400 }
@@ -746,7 +743,7 @@ describe.each([
             newColumn: {
               name: "_id",
               type: FieldType.BB_REFERENCE,
-              subtype: FieldSubtype.USERS,
+              subtype: BBReferenceFieldSubType.USERS,
             },
           },
           { status: 400 }
@@ -761,7 +758,7 @@ describe.each([
             newColumn: {
               name: "num",
               type: FieldType.BB_REFERENCE,
-              subtype: FieldSubtype.USERS,
+              subtype: BBReferenceFieldSubType.USERS,
             },
           },
           { status: 400 }
@@ -775,12 +772,12 @@ describe.each([
             oldColumn: {
               name: "not a column",
               type: FieldType.BB_REFERENCE,
-              subtype: FieldSubtype.USERS,
+              subtype: BBReferenceFieldSubType.USERS,
             },
             newColumn: {
               name: "new column",
               type: FieldType.BB_REFERENCE,
-              subtype: FieldSubtype.USERS,
+              subtype: BBReferenceFieldSubType.USERS,
             },
           },
           { status: 400 }

packages/server/src/automations/steps/sendSmtpEmail.ts

@@ -7,6 +7,7 @@ import {
   AutomationStepType,
   AutomationIOType,
   AutomationFeature,
+  AutomationCustomIOType,
 } from "@budibase/types"
 
 export const definition: AutomationStepSchema = {
@@ -72,10 +73,10 @@ export const definition: AutomationStepSchema = {
           title: "Location",
           dependsOn: "addInvite",
         },
-        url: {
+        attachments: {
-          type: AutomationIOType.STRING,
+          type: AutomationIOType.ATTACHMENT,
-          title: "URL",
+          customType: AutomationCustomIOType.MULTI_ATTACHMENTS,
-          dependsOn: "addInvite",
+          title: "Attachments",
         },
       },
       required: ["to", "from", "subject", "contents"],
@@ -110,11 +111,13 @@ export async function run({ inputs }: AutomationStepInput) {
     summary,
     location,
     url,
+    attachments,
   } = inputs
   if (!contents) {
     contents = "<h1>No content</h1>"
   }
   to = to || undefined
+
   try {
     let response = await sendSmtpEmail({
       to,
@@ -124,6 +127,7 @@ export async function run({ inputs }: AutomationStepInput) {
       cc,
       bcc,
       automation: true,
+      attachments,
       invite: addInvite
         ? {
             startTime,

packages/server/src/automations/tests/sendSmtpEmail.spec.ts

@@ -50,6 +50,10 @@ describe("test the outgoing webhook action", () => {
       cc: "cc",
       bcc: "bcc",
       addInvite: true,
+      attachments: [
+        { url: "attachment1", filename: "attachment1.txt" },
+        { url: "attachment2", filename: "attachment2.txt" },
+      ],
       ...invite,
     }
     let resp = generateResponse(inputs.to, inputs.from)
@@ -69,6 +73,10 @@ describe("test the outgoing webhook action", () => {
       bcc: "bcc",
       invite,
       automation: true,
+      attachments: [
+        { url: "attachment1", filename: "attachment1.txt" },
+        { url: "attachment2", filename: "attachment2.txt" },
+      ],
     })
   })
 })

packages/server/src/integrations/base/sql.ts

@@ -12,7 +12,7 @@ import SqlTableQueryBuilder from "./sqlTable"
 import {
   BBReferenceFieldMetadata,
   FieldSchema,
-  FieldSubtype,
+  BBReferenceFieldSubType,
   FieldType,
   JsonFieldMetadata,
   Operation,
@@ -472,14 +472,13 @@ class InternalBuilder {
   ): Knex.QueryBuilder {
     const tableName = endpoint.entityId
     const tableAlias = aliases?.[tableName]
-    let table: string | Record<string, string> = tableName
+
-    if (tableAlias) {
+    const query = knex(
-      table = { [tableAlias]: tableName }
+      this.tableNameWithSchema(tableName, {
-    }
+        alias: tableAlias,
-    let query = knex(table)
+        schema: endpoint.schema,
-    if (endpoint.schema) {
+      })
-      query = query.withSchema(endpoint.schema)
+    )
-    }
     return query
   }
 
@@ -768,7 +767,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     return (
       field.type === FieldType.JSON ||
       (field.type === FieldType.BB_REFERENCE &&
-        field.subtype === FieldSubtype.USERS)
+        field.subtype === BBReferenceFieldSubType.USERS)
     )
   }
 

packages/server/src/integrations/base/sqlTable.ts

@@ -1,6 +1,6 @@
 import { Knex, knex } from "knex"
 import {
-  FieldSubtype,
+  BBReferenceFieldSubType,
   FieldType,
   NumberFieldMetadata,
   Operation,
@@ -64,10 +64,10 @@ function generateSchema(
       case FieldType.BB_REFERENCE: {
         const subtype = column.subtype
         switch (subtype) {
-          case FieldSubtype.USER:
+          case BBReferenceFieldSubType.USER:
             schema.text(key)
             break
-          case FieldSubtype.USERS:
+          case BBReferenceFieldSubType.USERS:
             schema.json(key)
             break
           default:

packages/server/src/integrations/oracle.ts

@@ -378,7 +378,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
         try {
           await connection.close()
         } catch (err) {
-          console.error(err)
+          console.error("Error connecting to Oracle", err)
         }
       }
     }

packages/server/src/integrations/rest.ts

@@ -21,6 +21,10 @@ import { performance } from "perf_hooks"
 import FormData from "form-data"
 import { URLSearchParams } from "url"
 import { blacklist } from "@budibase/backend-core"
+import { handleFileResponse, handleXml } from "./utils"
+import { parse } from "content-disposition"
+import path from "path"
+import { Builder as XmlBuilder } from "xml2js"
 
 const BodyTypes = {
   NONE: "none",
@@ -57,8 +61,6 @@ const coreFields = {
   },
 }
 
-const { parseStringPromise: xmlParser, Builder: XmlBuilder } = require("xml2js")
-
 const SCHEMA: Integration = {
   docs: "https://github.com/node-fetch/node-fetch",
   description:
@@ -129,9 +131,22 @@ class RestIntegration implements IntegrationBase {
   }
 
   async parseResponse(response: any, pagination: PaginationConfig | null) {
-    let data, raw, headers
+    let data, raw, headers, filename
+
     const contentType = response.headers.get("content-type") || ""
+    const contentDisposition = response.headers.get("content-disposition") || ""
+    if (
+      contentDisposition.includes("attachment") ||
+      contentDisposition.includes("form-data")
+    ) {
+      filename =
+        path.basename(parse(contentDisposition).parameters?.filename) || ""
+    }
+
     try {
+      if (filename) {
+        return handleFileResponse(response, filename, this.startTimeMs)
+      } else {
         if (response.status === 204) {
           data = []
           raw = []
@@ -142,29 +157,18 @@ class RestIntegration implements IntegrationBase {
           contentType.includes("text/xml") ||
           contentType.includes("application/xml")
         ) {
-        const rawXml = await response.text()
+          let xmlResponse = await handleXml(response)
-        data =
+          data = xmlResponse.data
-          (await xmlParser(rawXml, {
+          raw = xmlResponse.rawXml
-            explicitArray: false,
-            trim: true,
-            explicitRoot: false,
-          })) || {}
-        // there is only one structure, its an array, return the array so it appears as rows
-        const keys = Object.keys(data)
-        if (keys.length === 1 && Array.isArray(data[keys[0]])) {
-          data = data[keys[0]]
-        }
-        raw = rawXml
-      } else if (contentType.includes("application/pdf")) {
-        data = await response.arrayBuffer() // Save PDF as ArrayBuffer
-        raw = Buffer.from(data)
         } else {
           data = await response.text()
           raw = data
         }
-    } catch (err) {
-      throw "Failed to parse response body."
       }
+    } catch (err) {
+      throw `Failed to parse response body: ${err}`
+    }
+
     const size = formatBytes(
       response.headers.get("content-length") || Buffer.byteLength(raw, "utf8")
     )

packages/server/src/integrations/tests/rest.spec.ts

@@ -13,9 +13,23 @@ jest.mock("node-fetch", () => {
   }))
 })
 
-import fetch from "node-fetch"
+jest.mock("@budibase/backend-core", () => {
+  const core = jest.requireActual("@budibase/backend-core")
+  return {
+    ...core,
+    context: {
+      ...core.context,
+      getProdAppId: jest.fn(() => "app-id"),
+    },
+  }
+})
+jest.mock("uuid", () => ({ v4: () => "00000000-0000-0000-0000-000000000000" }))
+
 import { default as RestIntegration } from "../rest"
 import { RestAuthType } from "@budibase/types"
+import fetch from "node-fetch"
+import { objectStoreTestProviders } from "@budibase/backend-core/tests"
+import { Readable } from "stream"
 
 const FormData = require("form-data")
 const { URLSearchParams } = require("url")
@@ -611,4 +625,104 @@ describe("REST Integration", () => {
     expect(calledConfig.headers).toEqual({})
     expect(calledConfig.agent.options.rejectUnauthorized).toBe(false)
   })
+
+  describe("File Handling", () => {
+    beforeAll(async () => {
+      jest.unmock("aws-sdk")
+      await objectStoreTestProviders.minio.start()
+    })
+
+    afterAll(async () => {
+      await objectStoreTestProviders.minio.stop()
+    })
+
+    it("uploads file to object store and returns signed URL", async () => {
+      const responseData = Buffer.from("teest file contnt")
+      const filename = "test.tar.gz"
+      const contentType = "application/gzip"
+      const mockReadable = new Readable()
+      mockReadable.push(responseData)
+      mockReadable.push(null)
+      ;(fetch as unknown as jest.Mock).mockImplementationOnce(() =>
+        Promise.resolve({
+          headers: {
+            raw: () => ({
+              "content-type": [contentType],
+              "content-disposition": [`attachment; filename="${filename}"`],
+            }),
+            get: (header: any) => {
+              if (header === "content-type") return contentType
+              if (header === "content-disposition")
+                return `attachment; filename="${filename}"`
+            },
+          },
+          body: mockReadable,
+        })
+      )
+
+      const query = {
+        path: "api",
+      }
+
+      const response = await config.integration.read(query)
+
+      expect(response.data).toEqual({
+        size: responseData.byteLength,
+        name: "00000000-0000-0000-0000-000000000000.tar.gz",
+        url: expect.stringContaining(
+          "/files/signed/tmp-file-attachments/app-id/00000000-0000-0000-0000-000000000000.tar.gz"
+        ),
+        extension: "tar.gz",
+        key: expect.stringContaining(
+          "app-id/00000000-0000-0000-0000-000000000000.tar.gz"
+        ),
+      })
+    })
+
+    it("uploads file with non ascii filename to object store and returns signed URL", async () => {
+      const responseData = Buffer.from("teest file contnt")
+      const contentType = "text/plain"
+      const mockReadable = new Readable()
+      mockReadable.push(responseData)
+      mockReadable.push(null)
+      ;(fetch as unknown as jest.Mock).mockImplementationOnce(() =>
+        Promise.resolve({
+          headers: {
+            raw: () => ({
+              "content-type": [contentType],
+              "content-disposition": [
+                // eslint-disable-next-line no-useless-escape
+                `attachment; filename="£ and ? rates.pdf"; filename*=UTF-8'\'%C2%A3%20and%20%E2%82%AC%20rates.pdf`,
+              ],
+            }),
+            get: (header: any) => {
+              if (header === "content-type") return contentType
+              if (header === "content-disposition")
+                // eslint-disable-next-line no-useless-escape
+                return `attachment; filename="£ and ? rates.pdf"; filename*=UTF-8'\'%C2%A3%20and%20%E2%82%AC%20rates.pdf`
+            },
+          },
+          body: mockReadable,
+        })
+      )
+
+      const query = {
+        path: "api",
+      }
+
+      const response = await config.integration.read(query)
+
+      expect(response.data).toEqual({
+        size: responseData.byteLength,
+        name: "00000000-0000-0000-0000-000000000000.pdf",
+        url: expect.stringContaining(
+          "/files/signed/tmp-file-attachments/app-id/00000000-0000-0000-0000-000000000000.pdf"
+        ),
+        extension: "pdf",
+        key: expect.stringContaining(
+          "app-id/00000000-0000-0000-0000-000000000000.pdf"
+        ),
+      })
+    })
+  })
 })

packages/server/src/integrations/utils/utils.ts

@@ -4,18 +4,45 @@ import {
   Datasource,
   FieldType,
   TableSourceType,
+  FieldSchema,
 } from "@budibase/types"
+import { context, objectStore } from "@budibase/backend-core"
+import { v4 } from "uuid"
+import { parseStringPromise as xmlParser } from "xml2js"
+import { formatBytes } from "../../utilities"
+import bl from "bl"
+import env from "../../environment"
 import { DocumentType, SEPARATOR } from "../../db/utils"
 import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../../constants"
-import { SWITCHABLE_TYPES, helpers } from "@budibase/shared-core"
+import { helpers, utils } from "@budibase/shared-core"
-import env from "../../environment"
 import { Knex } from "knex"
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
 const ENCODED_SPACE = encodeURIComponent(" ")
 
-const SQL_NUMBER_TYPE_MAP = {
+type PrimitiveTypes =
+  | FieldType.STRING
+  | FieldType.NUMBER
+  | FieldType.BOOLEAN
+  | FieldType.DATETIME
+  | FieldType.JSON
+  | FieldType.BIGINT
+  | FieldType.OPTIONS
+
+function isPrimitiveType(type: FieldType): type is PrimitiveTypes {
+  return [
+    FieldType.STRING,
+    FieldType.NUMBER,
+    FieldType.BOOLEAN,
+    FieldType.DATETIME,
+    FieldType.JSON,
+    FieldType.BIGINT,
+    FieldType.OPTIONS,
+  ].includes(type)
+}
+
+const SQL_NUMBER_TYPE_MAP: Record<string, PrimitiveTypes> = {
   integer: FieldType.NUMBER,
   int: FieldType.NUMBER,
   decimal: FieldType.NUMBER,
@@ -35,7 +62,7 @@ const SQL_NUMBER_TYPE_MAP = {
   smallmoney: FieldType.NUMBER,
 }
 
-const SQL_DATE_TYPE_MAP = {
+const SQL_DATE_TYPE_MAP: Record<string, PrimitiveTypes> = {
   timestamp: FieldType.DATETIME,
   time: FieldType.DATETIME,
   datetime: FieldType.DATETIME,
@@ -46,7 +73,7 @@ const SQL_DATE_TYPE_MAP = {
 const SQL_DATE_ONLY_TYPES = ["date"]
 const SQL_TIME_ONLY_TYPES = ["time"]
 
-const SQL_STRING_TYPE_MAP = {
+const SQL_STRING_TYPE_MAP: Record<string, PrimitiveTypes> = {
   varchar: FieldType.STRING,
   char: FieldType.STRING,
   nchar: FieldType.STRING,
@@ -58,22 +85,22 @@ const SQL_STRING_TYPE_MAP = {
   text: FieldType.STRING,
 }
 
-const SQL_BOOLEAN_TYPE_MAP = {
+const SQL_BOOLEAN_TYPE_MAP: Record<string, PrimitiveTypes> = {
   boolean: FieldType.BOOLEAN,
   bit: FieldType.BOOLEAN,
   tinyint: FieldType.BOOLEAN,
 }
 
-const SQL_OPTIONS_TYPE_MAP = {
+const SQL_OPTIONS_TYPE_MAP: Record<string, PrimitiveTypes> = {
   "user-defined": FieldType.OPTIONS,
 }
 
-const SQL_MISC_TYPE_MAP = {
+const SQL_MISC_TYPE_MAP: Record<string, PrimitiveTypes> = {
   json: FieldType.JSON,
   bigint: FieldType.BIGINT,
 }
 
-const SQL_TYPE_MAP = {
+const SQL_TYPE_MAP: Record<string, PrimitiveTypes> = {
   ...SQL_NUMBER_TYPE_MAP,
   ...SQL_DATE_TYPE_MAP,
   ...SQL_STRING_TYPE_MAP,
@@ -239,14 +266,14 @@ export function generateColumnDefinition(config: {
     constraints.inclusion = options
   }
 
-  const schema: any = {
+  const schema: FieldSchema = {
     type: foundType,
     externalType,
     autocolumn,
     name,
     constraints,
   }
-  if (foundType === FieldType.DATETIME) {
+  if (schema.type === FieldType.DATETIME) {
     schema.dateOnly = SQL_DATE_ONLY_TYPES.includes(lowerCaseType)
     schema.timeOnly = SQL_TIME_ONLY_TYPES.includes(lowerCaseType)
   }
@@ -274,49 +301,6 @@ export function isIsoDateString(str: string) {
   return d.toISOString() === trimmedValue
 }
 
-/**
- * This function will determine whether a column is a relationship and whether it
- * is currently valid. The reason for the validity check is that tables can be deleted
- * outside of Budibase control and if this is the case it will break Budibase relationships.
- * The tableIds is a list passed down from the main finalise tables function, which is
- * based on the tables that have just been fetched. This will only really be used on subsequent
- * fetches to the first one - if the user is periodically refreshing Budibase knowledge of tables.
- * @param column The column to check, to see if it is a valid relationship.
- * @param tableIds The IDs of the tables which currently exist.
- */
-function shouldCopyRelationship(
-  column: { type: FieldType.LINK; tableId?: string },
-  tableIds: string[]
-) {
-  return (
-    column.type === FieldType.LINK &&
-    column.tableId &&
-    tableIds.includes(column.tableId)
-  )
-}
-
-/**
- * Similar function to the shouldCopyRelationship function, but instead this looks for options and boolean
- * types. It is possible to switch a string -> options and a number -> boolean (and vice versus) need to make
- * sure that these get copied over when tables are fetched. Also checks whether they are still valid, if a
- * column has changed type in the external database then copying it over may not be possible.
- * @param column The column to check for options or boolean type.
- * @param fetchedColumn The fetched column to check for the type in the external database.
- */
-function shouldCopySpecialColumn(
-  column: { type: FieldType },
-  fetchedColumn: { type: FieldType } | undefined
-) {
-  const isFormula = column.type === FieldType.FORMULA
-  // column has been deleted, remove - formulas will never exist, always copy
-  if (!isFormula && column && !fetchedColumn) {
-    return false
-  }
-  const fetchedIsNumber =
-    !fetchedColumn || fetchedColumn.type === FieldType.NUMBER
-  return fetchedIsNumber && column.type === FieldType.BOOLEAN
-}
-
 /**
  * Looks for columns which need to be copied over into the new table definitions, like relationships,
  * options types and views.
@@ -338,6 +322,9 @@ function copyExistingPropsOver(
     if (entities[tableName]?.created) {
       table.created = entities[tableName]?.created
     }
+    if (entities[tableName]?.constrained) {
+      table.constrained = entities[tableName]?.constrained
+    }
 
     table.views = entities[tableName].views
 
@@ -346,45 +333,73 @@ function copyExistingPropsOver(
       if (!Object.prototype.hasOwnProperty.call(existingTableSchema, key)) {
         continue
       }
+
       const column = existingTableSchema[key]
 
       const existingColumnType = column?.type
       const updatedColumnType = table.schema[key]?.type
 
-      // If the db column type changed to a non-compatible one, we want to re-fetch it
+      const keepIfType = (...validTypes: PrimitiveTypes[]) => {
-      if (
+        return (
-        updatedColumnType !== existingColumnType &&
+          isPrimitiveType(updatedColumnType) &&
-        !SWITCHABLE_TYPES[updatedColumnType]?.includes(existingColumnType)
+          table.schema[key] &&
-      ) {
+          validTypes.includes(updatedColumnType)
-        continue
+        )
       }
 
-      if (
+      let shouldKeepSchema = false
-        column.type === FieldType.LINK &&
+      switch (existingColumnType) {
-        !shouldCopyRelationship(column, tableIds)
+        case FieldType.FORMULA:
-      ) {
+        case FieldType.AUTO:
-        continue
+        case FieldType.INTERNAL:
-      }
+          shouldKeepSchema = true
-
+          break
-      const specialTypes = [
+
-        FieldType.OPTIONS,
+        case FieldType.LINK:
-        FieldType.LONGFORM,
+          shouldKeepSchema =
-        FieldType.ARRAY,
+            existingColumnType === FieldType.LINK &&
-        FieldType.FORMULA,
+            tableIds.includes(column.tableId)
-        FieldType.BB_REFERENCE,
+          break
-      ]
+
-      if (
+        case FieldType.STRING:
-        specialTypes.includes(column.type) &&
+        case FieldType.OPTIONS:
-        !shouldCopySpecialColumn(column, table.schema[key])
+        case FieldType.LONGFORM:
-      ) {
+        case FieldType.BARCODEQR:
-        continue
+          shouldKeepSchema = keepIfType(FieldType.STRING)
+          break
+
+        case FieldType.NUMBER:
+        case FieldType.BOOLEAN:
+          shouldKeepSchema = keepIfType(FieldType.BOOLEAN, FieldType.NUMBER)
+          break
+
+        case FieldType.ARRAY:
+        case FieldType.ATTACHMENTS:
+        case FieldType.ATTACHMENT_SINGLE:
+        case FieldType.JSON:
+        case FieldType.BB_REFERENCE:
+          shouldKeepSchema = keepIfType(FieldType.JSON, FieldType.STRING)
+          break
+
+        case FieldType.DATETIME:
+          shouldKeepSchema = keepIfType(FieldType.DATETIME, FieldType.STRING)
+          break
+
+        case FieldType.BIGINT:
+          shouldKeepSchema = keepIfType(FieldType.BIGINT, FieldType.NUMBER)
+          break
+
+        default:
+          utils.unreachable(existingColumnType)
       }
 
+      if (shouldKeepSchema) {
         table.schema[key] = {
           ...existingTableSchema[key],
           externalType:
             existingTableSchema[key].externalType ||
-          table.schema[key].externalType,
+            table.schema[key]?.externalType,
+        }
       }
     }
   }
@@ -457,3 +472,74 @@ export function getPrimaryDisplay(testValue: unknown): string | undefined {
 export function isValidFilter(value: any) {
   return value != null && value !== ""
 }
+
+export async function handleXml(response: any) {
+  let data,
+    rawXml = await response.text()
+  data =
+    (await xmlParser(rawXml, {
+      explicitArray: false,
+      trim: true,
+      explicitRoot: false,
+    })) || {}
+  // there is only one structure, its an array, return the array so it appears as rows
+  const keys = Object.keys(data)
+  if (keys.length === 1 && Array.isArray(data[keys[0]])) {
+    data = data[keys[0]]
+  }
+  return { data, rawXml }
+}
+
+export async function handleFileResponse(
+  response: any,
+  filename: string,
+  startTime: number
+) {
+  let presignedUrl,
+    size = 0
+  const fileExtension = filename.includes(".")
+    ? filename.split(".").slice(1).join(".")
+    : ""
+
+  const processedFileName = `${v4()}.${fileExtension}`
+  const key = `${context.getProdAppId()}/${processedFileName}`
+  const bucket = objectStore.ObjectStoreBuckets.TEMP
+
+  const stream = response.body.pipe(bl((error, data) => data))
+
+  if (response.body) {
+    const contentLength = response.headers.get("content-length")
+    if (contentLength) {
+      size = parseInt(contentLength, 10)
+    } else {
+      const chunks: Buffer[] = []
+      for await (const chunk of response.body) {
+        chunks.push(chunk)
+        size += chunk.length
+      }
+    }
+
+    await objectStore.streamUpload({
+      bucket,
+      filename: key,
+      stream,
+      ttl: 1,
+      type: response.headers["content-type"],
+    })
+  }
+  presignedUrl = await objectStore.getPresignedUrl(bucket, key)
+  return {
+    data: {
+      size,
+      name: processedFileName,
+      url: presignedUrl,
+      extension: fileExtension,
+      key: key,
+    },
+    info: {
+      code: response.status,
+      size: formatBytes(size.toString()),
+      time: `${Math.round(performance.now() - startTime)}ms`,
+    },
+  }
+}

packages/server/src/sdk/app/rows/search/tests/utils.spec.ts

@@ -2,7 +2,7 @@ import { searchInputMapping } from "../utils"
 import { db as dbCore } from "@budibase/backend-core"
 import {
   FieldType,
-  FieldTypeSubtypes,
+  BBReferenceFieldSubType,
   INTERNAL_TABLE_SOURCE_ID,
   RowSearchParams,
   Table,
@@ -20,7 +20,7 @@ const tableWithUserCol: Table = {
     user: {
       name: "user",
       type: FieldType.BB_REFERENCE,
-      subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+      subtype: BBReferenceFieldSubType.USER,
     },
   },
 }
@@ -35,7 +35,7 @@ const tableWithUsersCol: Table = {
     user: {
       name: "user",
       type: FieldType.BB_REFERENCE,
-      subtype: FieldTypeSubtypes.BB_REFERENCE.USERS,
+      subtype: BBReferenceFieldSubType.USERS,
     },
   },
 }

packages/server/src/sdk/app/rows/search/utils.ts

@@ -3,7 +3,7 @@ import {
   Table,
   DocumentType,
   SEPARATOR,
-  FieldSubtype,
+  BBReferenceFieldSubType,
   SearchFilters,
   SearchIndex,
   SearchResponse,
@@ -89,8 +89,8 @@ export function searchInputMapping(table: Table, options: RowSearchParams) {
       case FieldType.BB_REFERENCE: {
         const subtype = column.subtype
         switch (subtype) {
-          case FieldSubtype.USER:
+          case BBReferenceFieldSubType.USER:
-          case FieldSubtype.USERS:
+          case BBReferenceFieldSubType.USERS:
             userColumnMapping(key, options)
             break
           default:

packages/server/src/sdk/app/tables/migration.ts

@@ -2,7 +2,7 @@ import { BadRequestError, context, db as dbCore } from "@budibase/backend-core"
 import {
   BBReferenceFieldMetadata,
   FieldSchema,
-  FieldSubtype,
+  BBReferenceFieldSubType,
   InternalTable,
   isBBReferenceField,
   isRelationshipField,
@@ -96,7 +96,7 @@ function getColumnMigrator(
   }
 
   if (oldColumn.relationshipType === RelationshipType.ONE_TO_MANY) {
-    if (newColumn.subtype !== FieldSubtype.USER) {
+    if (newColumn.subtype !== BBReferenceFieldSubType.USER) {
       throw new BadRequestError(
         `Column "${oldColumn.name}" is a one-to-many column but "${newColumn.name}" is not a single user column`
       )
@@ -107,7 +107,7 @@ function getColumnMigrator(
     oldColumn.relationshipType === RelationshipType.MANY_TO_MANY ||
     oldColumn.relationshipType === RelationshipType.MANY_TO_ONE
   ) {
-    if (newColumn.subtype !== FieldSubtype.USERS) {
+    if (newColumn.subtype !== BBReferenceFieldSubType.USERS) {
       throw new BadRequestError(
         `Column "${oldColumn.name}" is a ${oldColumn.relationshipType} column but "${newColumn.name}" is not a multi user column`
       )

packages/server/src/sdk/app/tables/tests/validation.spec.ts

@@ -125,7 +125,7 @@ describe("validation and update of external table schemas", () => {
   }
 
   it("should correctly set utilised foreign keys to autocolumns", () => {
-    const response = populateExternalTableSchemas(cloneDeep(SCHEMA) as any)
+    const response = populateExternalTableSchemas(cloneDeep(SCHEMA))
     const foreignKey = getForeignKeyColumn(response)
     expect(foreignKey.autocolumn).toBe(true)
     expect(foreignKey.autoReason).toBe(AutoReason.FOREIGN_KEY)
@@ -133,7 +133,7 @@ describe("validation and update of external table schemas", () => {
   })
 
   it("should correctly unset foreign keys when no longer used", () => {
-    const setResponse = populateExternalTableSchemas(cloneDeep(SCHEMA) as any)
+    const setResponse = populateExternalTableSchemas(cloneDeep(SCHEMA))
     const beforeFk = getForeignKeyColumn(setResponse)
     delete setResponse.entities!.client.schema.project
     delete setResponse.entities!.project.schema.client

packages/server/src/sdk/app/tables/validation.ts

@@ -44,7 +44,10 @@ function checkForeignKeysAreAutoColumns(datasource: Datasource) {
       if (shouldBeForeign && !column.autocolumn) {
         column.autocolumn = true
         column.autoReason = AutoReason.FOREIGN_KEY
-      } else if (column.autoReason === AutoReason.FOREIGN_KEY) {
+      } else if (
+        !shouldBeForeign &&
+        column.autoReason === AutoReason.FOREIGN_KEY
+      ) {
         delete column.autocolumn
         delete column.autoReason
       }

packages/server/src/tests/utilities/api/datasource.ts

@@ -5,6 +5,7 @@ import {
   UpdateDatasourceResponse,
   UpdateDatasourceRequest,
   QueryJson,
+  BuildSchemaFromSourceResponse,
 } from "@budibase/types"
 import { Expectations, TestAPI } from "./base"
 
@@ -69,4 +70,13 @@ export class DatasourceAPI extends TestAPI {
       expectations,
     })
   }
+
+  fetchSchema = async (id: string, expectations?: Expectations) => {
+    return await this._post<BuildSchemaFromSourceResponse>(
+      `/api/datasources/${id}/schema`,
+      {
+        expectations,
+      }
+    )
+  }
 }

packages/server/src/utilities/fileSystem/clientLibrary.ts

@@ -106,22 +106,22 @@ export async function updateClientLibrary(appId: string) {
   }
 
   // Upload latest manifest and client library
-  const manifestUpload = objectStore.streamUpload(
+  const manifestUpload = objectStore.streamUpload({
-    ObjectStoreBuckets.APPS,
+    bucket: ObjectStoreBuckets.APPS,
-    join(appId, "manifest.json"),
+    filename: join(appId, "manifest.json"),
-    fs.createReadStream(manifest),
+    stream: fs.createReadStream(manifest),
-    {
+    extra: {
       ContentType: "application/json",
-    }
+    },
-  )
+  })
-  const clientUpload = objectStore.streamUpload(
+  const clientUpload = objectStore.streamUpload({
-    ObjectStoreBuckets.APPS,
+    bucket: ObjectStoreBuckets.APPS,
-    join(appId, "budibase-client.js"),
+    filename: join(appId, "budibase-client.js"),
-    fs.createReadStream(client),
+    stream: fs.createReadStream(client),
-    {
+    extra: {
       ContentType: "application/javascript",
-    }
+    },
-  )
+  })
 
   const manifestSrc = fs.promises.readFile(manifest, "utf8")
 

packages/server/src/utilities/fileSystem/filesystem.ts

@@ -43,7 +43,7 @@ export const checkDevelopmentEnvironment = () => {
     error = "Must run via yarn once to generate environment."
   }
   if (error) {
-    console.error(error)
+    console.error("Error during development environment check", error)
     process.exit(-1)
   }
 }

packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts

@@ -1,13 +1,17 @@
 import { cache, db as dbCore } from "@budibase/backend-core"
 import { utils } from "@budibase/shared-core"
-import { FieldSubtype, DocumentType, SEPARATOR } from "@budibase/types"
+import {
+  BBReferenceFieldSubType,
+  DocumentType,
+  SEPARATOR,
+} from "@budibase/types"
 import { InvalidBBRefError } from "./errors"
 
 const ROW_PREFIX = DocumentType.ROW + SEPARATOR
 
 export async function processInputBBReferences(
   value: string | string[] | { _id: string } | { _id: string }[],
-  subtype: FieldSubtype.USER | FieldSubtype.USERS
+  subtype: BBReferenceFieldSubType.USER | BBReferenceFieldSubType.USERS
 ): Promise<string | string[] | null> {
   let referenceIds: string[] = []
 
@@ -40,15 +44,18 @@ export async function processInputBBReferences(
   })
 
   switch (subtype) {
-    case FieldSubtype.USER:
+    case BBReferenceFieldSubType.USER:
-    case FieldSubtype.USERS: {
+    case BBReferenceFieldSubType.USERS: {
       const { notFoundIds } = await cache.user.getUsers(referenceIds)
 
       if (notFoundIds?.length) {
-        throw new InvalidBBRefError(notFoundIds[0], FieldSubtype.USER)
+        throw new InvalidBBRefError(
+          notFoundIds[0],
+          BBReferenceFieldSubType.USER
+        )
       }
 
-      if (subtype === FieldSubtype.USERS) {
+      if (subtype === BBReferenceFieldSubType.USERS) {
         return referenceIds
       }
 
@@ -61,7 +68,7 @@ export async function processInputBBReferences(
 
 export async function processOutputBBReferences(
   value: string | string[],
-  subtype: FieldSubtype.USER | FieldSubtype.USERS
+  subtype: BBReferenceFieldSubType.USER | BBReferenceFieldSubType.USERS
 ) {
   if (value === null || value === undefined) {
     // Already processed or nothing to process
@@ -72,8 +79,8 @@ export async function processOutputBBReferences(
     typeof value === "string" ? value.split(",").filter(id => !!id) : value
 
   switch (subtype) {
-    case FieldSubtype.USER:
+    case BBReferenceFieldSubType.USER:
-    case FieldSubtype.USERS: {
+    case BBReferenceFieldSubType.USERS: {
       const { users } = await cache.user.getUsers(ids)
       if (!users.length) {
         return undefined

packages/server/src/utilities/rowProcessor/errors.ts

@@ -1,7 +1,7 @@
-import { FieldSubtype } from "@budibase/types"
+import { BBReferenceFieldSubType } from "@budibase/types"
 
 export class InvalidBBRefError extends Error {
-  constructor(id: string, subtype: FieldSubtype) {
+  constructor(id: string, subtype: BBReferenceFieldSubType) {
     super(`Id "${id}" is not valid for the subtype "${subtype}"`)
   }
 }

packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts

@@ -1,6 +1,6 @@
 import _ from "lodash"
 import * as backendCore from "@budibase/backend-core"
-import { FieldSubtype, User } from "@budibase/types"
+import { BBReferenceFieldSubType, User } from "@budibase/types"
 import {
   processInputBBReferences,
   processOutputBBReferences,
@@ -63,7 +63,7 @@ describe("bbReferenceProcessor", () => {
         const userId = user!._id!
 
         const result = await config.doInTenant(() =>
-          processInputBBReferences(userId, FieldSubtype.USER)
+          processInputBBReferences(userId, BBReferenceFieldSubType.USER)
         )
 
         expect(result).toEqual(userId)
@@ -76,9 +76,11 @@ describe("bbReferenceProcessor", () => {
 
         await expect(
           config.doInTenant(() =>
-            processInputBBReferences(userId, FieldSubtype.USER)
+            processInputBBReferences(userId, BBReferenceFieldSubType.USER)
+          )
+        ).rejects.toThrow(
+          new InvalidBBRefError(userId, BBReferenceFieldSubType.USER)
         )
-        ).rejects.toThrow(new InvalidBBRefError(userId, FieldSubtype.USER))
         expect(cacheGetUsersSpy).toHaveBeenCalledTimes(1)
         expect(cacheGetUsersSpy).toHaveBeenCalledWith([userId])
       })
@@ -88,7 +90,7 @@ describe("bbReferenceProcessor", () => {
 
         const userIdCsv = userIds.join(" ,  ")
         const result = await config.doInTenant(() =>
-          processInputBBReferences(userIdCsv, FieldSubtype.USER)
+          processInputBBReferences(userIdCsv, BBReferenceFieldSubType.USER)
         )
 
         expect(result).toEqual(userIds.join(","))
@@ -108,16 +110,21 @@ describe("bbReferenceProcessor", () => {
 
         await expect(
           config.doInTenant(() =>
-            processInputBBReferences(userIdCsv, FieldSubtype.USER)
+            processInputBBReferences(userIdCsv, BBReferenceFieldSubType.USER)
+          )
+        ).rejects.toThrow(
+          new InvalidBBRefError(wrongId, BBReferenceFieldSubType.USER)
         )
-        ).rejects.toThrow(new InvalidBBRefError(wrongId, FieldSubtype.USER))
       })
 
       it("validate valid user object", async () => {
         const userId = _.sample(users)!._id!
 
         const result = await config.doInTenant(() =>
-          processInputBBReferences({ _id: userId }, FieldSubtype.USER)
+          processInputBBReferences(
+            { _id: userId },
+            BBReferenceFieldSubType.USER
+          )
         )
 
         expect(result).toEqual(userId)
@@ -129,7 +136,7 @@ describe("bbReferenceProcessor", () => {
         const userIds = _.sampleSize(users, 3).map(x => x._id!)
 
         const result = await config.doInTenant(() =>
-          processInputBBReferences(userIds, FieldSubtype.USER)
+          processInputBBReferences(userIds, BBReferenceFieldSubType.USER)
         )
 
         expect(result).toEqual(userIds.join(","))
@@ -139,7 +146,7 @@ describe("bbReferenceProcessor", () => {
 
       it("empty strings will return null", async () => {
         const result = await config.doInTenant(() =>
-          processInputBBReferences("", FieldSubtype.USER)
+          processInputBBReferences("", BBReferenceFieldSubType.USER)
         )
 
         expect(result).toEqual(null)
@@ -147,7 +154,7 @@ describe("bbReferenceProcessor", () => {
 
       it("empty arrays will return null", async () => {
         const result = await config.doInTenant(() =>
-          processInputBBReferences([], FieldSubtype.USER)
+          processInputBBReferences([], BBReferenceFieldSubType.USER)
         )
 
         expect(result).toEqual(null)
@@ -157,7 +164,7 @@ describe("bbReferenceProcessor", () => {
         const userId = _.sample(users)!._id!
         const userMetadataId = backendCore.db.generateUserMetadataID(userId)
         const result = await config.doInTenant(() =>
-          processInputBBReferences(userMetadataId, FieldSubtype.USER)
+          processInputBBReferences(userMetadataId, BBReferenceFieldSubType.USER)
         )
         expect(result).toBe(userId)
       })
@@ -171,7 +178,7 @@ describe("bbReferenceProcessor", () => {
         const userId = user._id!
 
         const result = await config.doInTenant(() =>
-          processOutputBBReferences(userId, FieldSubtype.USER)
+          processOutputBBReferences(userId, BBReferenceFieldSubType.USER)
         )
 
         expect(result).toEqual([
@@ -195,7 +202,7 @@ describe("bbReferenceProcessor", () => {
         const result = await config.doInTenant(() =>
           processOutputBBReferences(
             [userId1, userId2].join(","),
-            FieldSubtype.USER
+            BBReferenceFieldSubType.USER
           )
         )
 

packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts

@@ -2,7 +2,7 @@ import { inputProcessing } from ".."
 import { generator, structures } from "@budibase/backend-core/tests"
 import {
   FieldType,
-  FieldTypeSubtypes,
+  BBReferenceFieldSubType,
   INTERNAL_TABLE_SOURCE_ID,
   Table,
   TableSourceType,
@@ -39,7 +39,7 @@ describe("rowProcessor - inputProcessing", () => {
         },
         user: {
           type: FieldType.BB_REFERENCE,
-          subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+          subtype: BBReferenceFieldSubType.USER,
           name: "user",
           constraints: {
             presence: true,
@@ -93,7 +93,7 @@ describe("rowProcessor - inputProcessing", () => {
         },
         user: {
           type: FieldType.BB_REFERENCE,
-          subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+          subtype: BBReferenceFieldSubType.USER,
           name: "user",
           constraints: {
             presence: false,
@@ -135,7 +135,7 @@ describe("rowProcessor - inputProcessing", () => {
           },
           user: {
             type: FieldType.BB_REFERENCE,
-            subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+            subtype: BBReferenceFieldSubType.USER,
             name: "user",
             constraints: {
               presence: false,

packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts

@@ -1,7 +1,6 @@
 import {
-  FieldSubtype,
   FieldType,
-  FieldTypeSubtypes,
+  BBReferenceFieldSubType,
   INTERNAL_TABLE_SOURCE_ID,
   RowAttachment,
   Table,
@@ -42,7 +41,7 @@ describe("rowProcessor - outputProcessing", () => {
         },
         user: {
           type: FieldType.BB_REFERENCE,
-          subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+          subtype: BBReferenceFieldSubType.USER,
           name: "user",
           constraints: {
             presence: false,
@@ -69,7 +68,7 @@ describe("rowProcessor - outputProcessing", () => {
     ).toHaveBeenCalledTimes(1)
     expect(bbReferenceProcessor.processOutputBBReferences).toHaveBeenCalledWith(
       "123",
-      FieldSubtype.USER
+      BBReferenceFieldSubType.USER
     )
   })
 
@@ -175,7 +174,7 @@ describe("rowProcessor - outputProcessing", () => {
         },
         user: {
           type: FieldType.BB_REFERENCE,
-          subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+          subtype: BBReferenceFieldSubType.USER,
           name: "user",
           constraints: {
             presence: false,

packages/server/src/utilities/schema.ts

@@ -1,6 +1,6 @@
 import {
   FieldType,
-  FieldSubtype,
+  BBReferenceFieldSubType,
   TableSchema,
   FieldSchema,
   Row,
@@ -137,10 +137,10 @@ export function parse(rows: Rows, schema: TableSchema): Rows {
           parsedRow[columnName] = undefined
         } else {
           switch (columnSubtype) {
-            case FieldSubtype.USER:
+            case BBReferenceFieldSubType.USER:
               parsedRow[columnName] = parsedValues[0]?._id
               break
-            case FieldSubtype.USERS:
+            case BBReferenceFieldSubType.USERS:
               parsedRow[columnName] = parsedValues.map(u => u._id)
               break
             default:
@@ -164,11 +164,11 @@ export function parse(rows: Rows, schema: TableSchema): Rows {
 
 function isValidBBReference(
   columnData: any,
-  columnSubtype: FieldSubtype.USER | FieldSubtype.USERS
+  columnSubtype: BBReferenceFieldSubType.USER | BBReferenceFieldSubType.USERS
 ): boolean {
   switch (columnSubtype) {
-    case FieldSubtype.USER:
+    case BBReferenceFieldSubType.USER:
-    case FieldSubtype.USERS: {
+    case BBReferenceFieldSubType.USERS: {
       if (typeof columnData !== "string") {
         return false
       }
@@ -177,7 +177,10 @@ function isValidBBReference(
         return false
       }
 
-      if (columnSubtype === FieldSubtype.USER && userArray.length > 1) {
+      if (
+        columnSubtype === BBReferenceFieldSubType.USER &&
+        userArray.length > 1
+      ) {
         return false
       }
 

packages/server/src/utilities/workerRequests.ts

@@ -8,7 +8,7 @@ import {
   logging,
   env as coreEnv,
 } from "@budibase/backend-core"
-import { Ctx, User, EmailInvite } from "@budibase/types"
+import { Ctx, User, EmailInvite, EmailAttachment } from "@budibase/types"
 
 interface Request {
   ctx?: Ctx
@@ -97,6 +97,7 @@ export async function sendSmtpEmail({
   bcc,
   automation,
   invite,
+  attachments,
 }: {
   to: string
   from: string
@@ -105,6 +106,7 @@ export async function sendSmtpEmail({
   cc: string
   bcc: string
   automation: boolean
+  attachments?: EmailAttachment[]
   invite?: EmailInvite
 }) {
   // tenant ID will be set in header
@@ -122,6 +124,7 @@ export async function sendSmtpEmail({
         purpose: "custom",
         automation,
         invite,
+        attachments,
       },
     })
   )

packages/shared-core/src/filters.ts

@@ -1,6 +1,6 @@
 import {
   Datasource,
-  FieldSubtype,
+  BBReferenceFieldSubType,
   FieldType,
   FormulaType,
   SearchFilter,
@@ -22,7 +22,7 @@ const HBS_REGEX = /{{([^{].*?)}}/g
 export const getValidOperatorsForType = (
   fieldType: {
     type: FieldType
-    subtype?: FieldSubtype
+    subtype?: BBReferenceFieldSubType
     formulaType?: FormulaType
   },
   field: string,
@@ -68,9 +68,15 @@ export const getValidOperatorsForType = (
     ops = numOps
   } else if (type === FieldType.FORMULA && formulaType === FormulaType.STATIC) {
     ops = stringOps.concat([Op.MoreThan, Op.LessThan])
-  } else if (type === FieldType.BB_REFERENCE && subtype == FieldSubtype.USER) {
+  } else if (
+    type === FieldType.BB_REFERENCE &&
+    subtype == BBReferenceFieldSubType.USER
+  ) {
     ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In]
-  } else if (type === FieldType.BB_REFERENCE && subtype == FieldSubtype.USERS) {
+  } else if (
+    type === FieldType.BB_REFERENCE &&
+    subtype == BBReferenceFieldSubType.USERS
+  ) {
     ops = [Op.Contains, Op.NotContains, Op.ContainsAny, Op.Empty, Op.NotEmpty]
   }
 

packages/string-templates/package.json

@@ -38,7 +38,7 @@
     "doctrine": "^3.0.0",
     "jest": "29.7.0",
     "marked": "^4.0.10",
-    "rollup": "^2.36.2",
+    "rollup": "^4.9.6",
     "rollup-plugin-inject-process-env": "^1.3.1",
     "rollup-plugin-node-builtins": "^2.1.2",
     "rollup-plugin-node-globals": "^1.4.0",

packages/string-templates/src/index.ts

@@ -16,7 +16,7 @@ import { setJSRunner, removeJSRunner } from "./helpers/javascript"
 import manifest from "./manifest.json"
 import { ProcessOptions } from "./types"
 
-export { helpersToRemoveForJs } from "./helpers/list"
+export { helpersToRemoveForJs, getJsHelperList } from "./helpers/list"
 export { FIND_ANY_HBS_REGEX } from "./utilities"
 export { setJSRunner, setOnErrorLog } from "./helpers/javascript"
 export { iifeWrapper } from "./iife"

packages/types/src/api/web/user.ts

@@ -66,6 +66,8 @@ export interface CreateAdminUserRequest {
   password?: string
   tenantId: string
   ssoId?: string
+  familyName?: string
+  givenName?: string
 }
 
 export interface AddSSoUserRequest {

packages/types/src/documents/app/automation.ts

@@ -10,6 +10,7 @@ export enum AutomationIOType {
   ARRAY = "array",
   JSON = "json",
   DATE = "date",
+  ATTACHMENT = "attachment",
 }
 
 export enum AutomationCustomIOType {
@@ -30,6 +31,7 @@ export enum AutomationCustomIOType {
   WEBHOOK_URL = "webhookUrl",
   AUTOMATION = "automation",
   AUTOMATION_FIELDS = "automationFields",
+  MULTI_ATTACHMENTS = "multi_attachments",
 }
 
 export enum AutomationTriggerStepId {
@@ -80,6 +82,11 @@ export interface EmailInvite {
   url?: string
 }
 
+export interface EmailAttachment {
+  url: string
+  filename: string
+}
+
 export interface SendEmailOpts {
   // workspaceId If finer grain controls being used then this will lookup config for workspace.
   workspaceId?: string
@@ -97,6 +104,7 @@ export interface SendEmailOpts {
   bcc?: boolean
   automation?: boolean
   invite?: EmailInvite
+  attachments?: EmailAttachment[]
 }
 
 export const AutomationStepIdArray = [

packages/types/src/documents/app/datasource.ts

@@ -13,9 +13,7 @@ export interface Datasource extends Document {
   config?: Record<string, any>
   plus?: boolean
   isSQL?: boolean
-  entities?: {
+  entities?: Record<string, Table>
-    [key: string]: Table
-  }
 }
 
 export enum RestAuthType {

packages/types/src/documents/app/row.ts

@@ -124,16 +124,3 @@ export interface Row extends Document {
   _viewId?: string
   [key: string]: any
 }
-
-export enum FieldSubtype {
-  USER = "user",
-  USERS = "users",
-}
-
-// The 'as' are required for typescript not to type the outputs as generic FieldSubtype
-export const FieldTypeSubtypes = {
-  BB_REFERENCE: {
-    USER: FieldSubtype.USER as FieldSubtype.USER,
-    USERS: FieldSubtype.USERS as FieldSubtype.USERS,
-  },
-}

packages/types/src/documents/app/table/constants.ts

@@ -24,3 +24,8 @@ export enum FormulaType {
   STATIC = "static",
   DYNAMIC = "dynamic",
 }
+
+export enum BBReferenceFieldSubType {
+  USER = "user",
+  USERS = "users",
+}

packages/types/src/documents/app/table/schema.ts

@@ -1,9 +1,10 @@
 // all added by grid/table when defining the
 // column size, position and whether it can be viewed
-import { FieldSubtype, FieldType } from "../row"
+import { FieldType } from "../row"
 import {
   AutoFieldSubType,
   AutoReason,
+  BBReferenceFieldSubType,
   FormulaType,
   JsonFieldSubType,
   RelationshipType,
@@ -91,6 +92,7 @@ export interface DateFieldMetadata extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.DATETIME
   ignoreTimezones?: boolean
   timeOnly?: boolean
+  dateOnly?: boolean
   subtype?: AutoFieldSubType.CREATED_AT | AutoFieldSubType.UPDATED_AT
 }
 
@@ -108,7 +110,7 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
 export interface BBReferenceFieldMetadata
   extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.BB_REFERENCE
-  subtype: FieldSubtype.USER | FieldSubtype.USERS
+  subtype: BBReferenceFieldSubType.USER | BBReferenceFieldSubType.USERS
   relationshipType?: RelationshipType
 }
 

packages/types/src/documents/global/user.ts

@@ -22,6 +22,13 @@ export interface UserSSO {
   providerType: SSOProviderType
   oauth2?: OAuth2
   thirdPartyProfile?: SSOProfileJson
+  profile?: {
+    displayName?: string
+    name?: {
+      givenName?: string
+      familyName?: string
+    }
+  }
 }
 
 export type SSOUser = User & UserSSO

packages/types/src/sdk/licensing/plan.ts

@@ -7,9 +7,11 @@ export enum PlanType {
   /** @deprecated */
   PREMIUM = "premium",
   PREMIUM_PLUS = "premium_plus",
+  PREMIUM_PLUS_TRIAL = "premium_plus_trial",
   /** @deprecated */
   BUSINESS = "business",
   ENTERPRISE_BASIC = "enterprise_basic",
+  ENTERPRISE_BASIC_TRIAL = "enterprise_basic_trial",
   ENTERPRISE = "enterprise",
 }
 

packages/worker/src/api/controllers/global/auth.ts

@@ -35,8 +35,7 @@ async function passportCallback(
   info: { message: string } | null = null
 ) {
   if (err) {
-    console.error("Authentication error")
+    console.error("Authentication error", err)
-    console.error(err)
     console.trace(err)
     return ctx.throw(403, info ? info : "Unauthorized")
   }

packages/worker/src/api/controllers/global/email.ts

@@ -15,6 +15,7 @@ export async function sendEmail(ctx: BBContext) {
     bcc,
     automation,
     invite,
+    attachments,
   } = ctx.request.body
   let user: any
   if (userId) {
@@ -31,6 +32,7 @@ export async function sendEmail(ctx: BBContext) {
     bcc,
     automation,
     invite,
+    attachments,
   })
   ctx.body = {
     ...response,

packages/worker/src/api/controllers/global/users.ts

@@ -116,7 +116,8 @@ const parseBooleanParam = (param: any) => {
 export const adminUser = async (
   ctx: Ctx<CreateAdminUserRequest, CreateAdminUserResponse>
 ) => {
-  const { email, password, tenantId, ssoId } = ctx.request.body
+  const { email, password, tenantId, ssoId, givenName, familyName } =
+    ctx.request.body
 
   if (await platform.tenants.exists(tenantId)) {
     ctx.throw(403, "Organisation already exists.")
@@ -151,6 +152,8 @@ export const adminUser = async (
         ssoId,
         hashPassword,
         requirePassword,
+        firstName: givenName,
+        lastName: familyName,
       })
 
       // events