Merge master.

.github/workflows/budibase_ci.yml

@@ -202,6 +202,9 @@ jobs:
 
       - run: yarn --frozen-lockfile
 
+      - name: Build client library - necessary for component tests
+        run: yarn build:client
+
       - name: Set up PostgreSQL 16
         if: matrix.datasource == 'postgres'
         run: |

hosting/letsencrypt/nginx-ssl.conf

@@ -41,6 +41,11 @@ server {
     }
 
     location ~ ^/api/(system|admin|global)/ {
+        # Enable buffering for potentially large OIDC configs
+        proxy_buffering on;
+        proxy_buffer_size 16k;
+        proxy_buffers 4 32k;
+        
         proxy_pass      http://127.0.0.1:4002;
     }
 

hosting/nginx.dev.conf

@@ -63,6 +63,11 @@ http {
       proxy_send_timeout 120s;
       proxy_http_version 1.1;
 
+      # Enable buffering for potentially large OIDC configs
+      proxy_buffering on;
+      proxy_buffer_size 16k;
+      proxy_buffers 4 32k;
+
       proxy_set_header Host $host;
       proxy_set_header Connection "";
 

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.4.4",
+  "version": "3.4.13",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

package.json

@@ -18,6 +18,7 @@
     "eslint-plugin-jest": "28.9.0",
     "eslint-plugin-local-rules": "3.0.2",
     "eslint-plugin-svelte": "2.46.1",
+    "svelte-preprocess": "^6.0.3",
     "husky": "^8.0.3",
     "kill-port": "^1.6.1",
     "lerna": "7.4.2",
@@ -67,6 +68,7 @@
     "lint:fix:eslint": "eslint --fix --max-warnings=0 packages",
     "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"",
     "lint:fix": "yarn run lint:fix:eslint && yarn run lint:fix:prettier",
+    "build:client": "lerna run --stream build --scope @budibase/client",
     "build:specs": "lerna run --stream specs",
     "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild",

packages/backend-core/__mocks__/@aws-sdk/client-s3.ts

@@ -0,0 +1,28 @@
+export class S3 {
+  headBucket() {
+    return jest.fn().mockReturnThis()
+  }
+  deleteObject() {
+    return jest.fn().mockReturnThis()
+  }
+  deleteObjects() {
+    return jest.fn().mockReturnThis()
+  }
+  createBucket() {
+    return jest.fn().mockReturnThis()
+  }
+  getObject() {
+    return jest.fn().mockReturnThis()
+  }
+  listObject() {
+    return jest.fn().mockReturnThis()
+  }
+  promise() {
+    return jest.fn().mockReturnThis()
+  }
+  catch() {
+    return jest.fn()
+  }
+}
+
+export const GetObjectCommand = jest.fn(inputs => ({ inputs }))

packages/backend-core/__mocks__/@aws-sdk/s3-request-presigner.ts

@@ -0,0 +1,4 @@
+export const getSignedUrl = jest.fn((_, cmd) => {
+  const { inputs } = cmd
+  return `http://s3.example.com/${inputs?.Bucket}/${inputs?.Key}`
+})

packages/backend-core/__mocks__/aws-sdk.ts

@@ -1,19 +0,0 @@
-const mockS3 = {
-  headBucket: jest.fn().mockReturnThis(),
-  deleteObject: jest.fn().mockReturnThis(),
-  deleteObjects: jest.fn().mockReturnThis(),
-  createBucket: jest.fn().mockReturnThis(),
-  getObject: jest.fn().mockReturnThis(),
-  listObject: jest.fn().mockReturnThis(),
-  getSignedUrl: jest.fn((operation: string, params: any) => {
-    return `http://s3.example.com/${params.Bucket}/${params.Key}`
-  }),
-  promise: jest.fn().mockReturnThis(),
-  catch: jest.fn(),
-}
-
-const AWS = {
-  S3: jest.fn(() => mockS3),
-}
-
-export default AWS

packages/backend-core/package.json

@@ -30,6 +30,9 @@
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "3.709.0",
+    "@aws-sdk/lib-storage": "3.709.0",
+    "@aws-sdk/s3-request-presigner": "3.709.0",
     "@budibase/nano": "10.1.5",
     "@budibase/pouchdb-replication-stream": "1.2.11",
     "@budibase/shared-core": "*",
@@ -71,11 +74,13 @@
   "devDependencies": {
     "@jest/types": "^29.6.3",
     "@shopify/jest-koa-mocks": "5.1.1",
+    "@smithy/types": "4.0.0",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
     "@types/chance": "1.1.3",
     "@types/cookies": "0.7.8",
     "@types/jest": "29.5.5",
+    "@types/koa": "2.13.4",
     "@types/lodash": "4.14.200",
     "@types/node-fetch": "2.6.4",
     "@types/pouchdb": "6.4.2",
@@ -83,7 +88,6 @@
     "@types/semver": "7.3.7",
     "@types/tar-fs": "2.0.1",
     "@types/uuid": "8.3.4",
-    "@types/koa": "2.13.4",
     "chance": "1.1.8",
     "ioredis-mock": "8.9.0",
     "jest": "29.7.0",

packages/backend-core/src/context/mainContext.ts

@@ -123,7 +123,7 @@ export async function doInAutomationContext<T>(params: {
   task: () => T
 }): Promise<T> {
   await ensureSnippetContext()
-  return newContext(
+  return await newContext(
     {
       tenantId: getTenantIDFromAppID(params.appId),
       appId: params.appId,
@@ -266,9 +266,9 @@ export const getProdAppId = () => {
   return conversions.getProdAppID(appId)
 }
 
-export function doInEnvironmentContext(
+export function doInEnvironmentContext<T>(
   values: Record<string, string>,
-  task: any
+  task: () => T
 ) {
   if (!values) {
     throw new Error("Must supply environment variables.")

packages/backend-core/src/docIds/params.ts

@@ -8,6 +8,10 @@ import {
 import { getProdAppID } from "./conversions"
 import { DatabaseQueryOpts, VirtualDocumentType } from "@budibase/types"
 
+const EXTERNAL_TABLE_ID_REGEX = new RegExp(
+  `^${DocumentType.DATASOURCE_PLUS}_(.+)__(.+)$`
+)
+
 /**
  * If creating DB allDocs/query params with only a single top level ID this can be used, this
  * is usually the case as most of our docs are top level e.g. tables, automations, users and so on.
@@ -64,6 +68,11 @@ export function getQueryIndex(viewName: ViewName) {
   return `database/${viewName}`
 }
 
+export const isExternalTableId = (id: string): boolean => {
+  const matches = id.match(EXTERNAL_TABLE_ID_REGEX)
+  return !!id && matches !== null
+}
+
 /**
  * Check if a given ID is that of a table.
  */
@@ -72,7 +81,7 @@ export const isTableId = (id: string): boolean => {
   return (
     !!id &&
     (id.startsWith(`${DocumentType.TABLE}${SEPARATOR}`) ||
-      id.startsWith(`${DocumentType.DATASOURCE_PLUS}${SEPARATOR}`))
+      isExternalTableId(id))
   )
 }
 

packages/backend-core/src/environment.ts

@@ -154,7 +154,7 @@ const environment = {
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   AWS_SESSION_TOKEN: process.env.AWS_SESSION_TOKEN,
-  AWS_REGION: process.env.AWS_REGION,
+  AWS_REGION: process.env.AWS_REGION || "eu-west-1",
   MINIO_URL: process.env.MINIO_URL,
   MINIO_ENABLED: process.env.MINIO_ENABLED || 1,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,

packages/backend-core/src/objectStore/buckets/app.ts

@@ -13,7 +13,7 @@ export function clientLibraryPath(appId: string) {
  * due to issues with the domain we were unable to continue doing this - keeping
  * incase we are able to switch back to CDN path again in future.
  */
-export function clientLibraryCDNUrl(appId: string, version: string) {
+export async function clientLibraryCDNUrl(appId: string, version: string) {
   let file = clientLibraryPath(appId)
   if (env.CLOUDFRONT_CDN) {
     // append app version to bust the cache
@@ -24,7 +24,7 @@ export function clientLibraryCDNUrl(appId: string, version: string) {
     // file is public
     return cloudfront.getUrl(file)
   } else {
-    return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
+    return await objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
   }
 }
 
@@ -44,10 +44,10 @@ export function clientLibraryUrl(appId: string, version: string) {
   return `/api/assets/client?${qs.encode(qsParams)}`
 }
 
-export function getAppFileUrl(s3Key: string) {
+export async function getAppFileUrl(s3Key: string) {
   if (env.CLOUDFRONT_CDN) {
     return cloudfront.getPresignedUrl(s3Key)
   } else {
-    return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, s3Key)
+    return await objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, s3Key)
   }
 }

packages/backend-core/src/objectStore/buckets/global.ts

@@ -5,7 +5,11 @@ import * as cloudfront from "../cloudfront"
 
 // URLs
 
-export const getGlobalFileUrl = (type: string, name: string, etag?: string) => {
+export const getGlobalFileUrl = async (
+  type: string,
+  name: string,
+  etag?: string
+) => {
   let file = getGlobalFileS3Key(type, name)
   if (env.CLOUDFRONT_CDN) {
     if (etag) {
@@ -13,7 +17,7 @@ export const getGlobalFileUrl = (type: string, name: string, etag?: string) => {
     }
     return cloudfront.getPresignedUrl(file)
   } else {
-    return objectStore.getPresignedUrl(env.GLOBAL_BUCKET_NAME, file)
+    return await objectStore.getPresignedUrl(env.GLOBAL_BUCKET_NAME, file)
   }
 }
 

packages/backend-core/src/objectStore/buckets/plugins.ts

@@ -6,23 +6,25 @@ import { Plugin } from "@budibase/types"
 
 // URLS
 
-export function enrichPluginURLs(plugins?: Plugin[]): Plugin[] {
+export async function enrichPluginURLs(plugins?: Plugin[]): Promise<Plugin[]> {
   if (!plugins || !plugins.length) {
     return []
   }
-  return plugins.map(plugin => {
+  return await Promise.all(
-    const jsUrl = getPluginJSUrl(plugin)
+    plugins.map(async plugin => {
-    const iconUrl = getPluginIconUrl(plugin)
+      const jsUrl = await getPluginJSUrl(plugin)
+      const iconUrl = await getPluginIconUrl(plugin)
       return { ...plugin, jsUrl, iconUrl }
     })
+  )
 }
 
-function getPluginJSUrl(plugin: Plugin) {
+async function getPluginJSUrl(plugin: Plugin) {
   const s3Key = getPluginJSKey(plugin)
   return getPluginUrl(s3Key)
 }
 
-function getPluginIconUrl(plugin: Plugin): string | undefined {
+async function getPluginIconUrl(plugin: Plugin) {
   const s3Key = getPluginIconKey(plugin)
   if (!s3Key) {
     return
@@ -30,11 +32,11 @@ function getPluginIconUrl(plugin: Plugin): string | undefined {
   return getPluginUrl(s3Key)
 }
 
-function getPluginUrl(s3Key: string) {
+async function getPluginUrl(s3Key: string) {
   if (env.CLOUDFRONT_CDN) {
     return cloudfront.getPresignedUrl(s3Key)
   } else {
-    return objectStore.getPresignedUrl(env.PLUGIN_BUCKET_NAME, s3Key)
+    return await objectStore.getPresignedUrl(env.PLUGIN_BUCKET_NAME, s3Key)
   }
 }
 

packages/backend-core/src/objectStore/buckets/tests/app.spec.ts

@@ -93,25 +93,25 @@ describe("app", () => {
         testEnv.multiTenant()
       })
 
-      it("gets url with embedded minio", () => {
+      it("gets url with embedded minio", async () => {
         testEnv.withMinio()
-        const url = getAppFileUrl()
+        const url = await getAppFileUrl()
         expect(url).toBe(
           "/files/signed/prod-budi-app-assets/app_123/attachments/image.jpeg"
         )
       })
 
-      it("gets url with custom S3", () => {
+      it("gets url with custom S3", async () => {
         testEnv.withS3()
-        const url = getAppFileUrl()
+        const url = await getAppFileUrl()
         expect(url).toBe(
           "http://s3.example.com/prod-budi-app-assets/app_123/attachments/image.jpeg"
         )
       })
 
-      it("gets url with cloudfront + s3", () => {
+      it("gets url with cloudfront + s3", async () => {
         testEnv.withCloudfront()
-        const url = getAppFileUrl()
+        const url = await getAppFileUrl()
         // omit rest of signed params
         expect(
           url.includes("http://cf.example.com/app_123/attachments/image.jpeg?")
@@ -126,8 +126,8 @@ describe("app", () => {
 
       it("gets url with embedded minio", async () => {
         testEnv.withMinio()
-        await testEnv.withTenant(() => {
+        await testEnv.withTenant(async () => {
-          const url = getAppFileUrl()
+          const url = await getAppFileUrl()
           expect(url).toBe(
             "/files/signed/prod-budi-app-assets/app_123/attachments/image.jpeg"
           )
@@ -136,8 +136,8 @@ describe("app", () => {
 
       it("gets url with custom S3", async () => {
         testEnv.withS3()
-        await testEnv.withTenant(() => {
+        await testEnv.withTenant(async () => {
-          const url = getAppFileUrl()
+          const url = await getAppFileUrl()
           expect(url).toBe(
             "http://s3.example.com/prod-budi-app-assets/app_123/attachments/image.jpeg"
           )
@@ -146,8 +146,8 @@ describe("app", () => {
 
       it("gets url with cloudfront + s3", async () => {
         testEnv.withCloudfront()
-        await testEnv.withTenant(() => {
+        await testEnv.withTenant(async () => {
-          const url = getAppFileUrl()
+          const url = await getAppFileUrl()
           // omit rest of signed params
           expect(
             url.includes(

packages/backend-core/src/objectStore/buckets/tests/global.spec.ts

@@ -3,7 +3,7 @@ import { testEnv } from "../../../../tests/extra"
 
 describe("global", () => {
   describe("getGlobalFileUrl", () => {
-    function getGlobalFileUrl() {
+    async function getGlobalFileUrl() {
       return global.getGlobalFileUrl("settings", "logoUrl", "etag")
     }
 
@@ -12,21 +12,21 @@ describe("global", () => {
         testEnv.singleTenant()
       })
 
-      it("gets url with embedded minio", () => {
+      it("gets url with embedded minio", async () => {
         testEnv.withMinio()
-        const url = getGlobalFileUrl()
+        const url = await getGlobalFileUrl()
         expect(url).toBe("/files/signed/global/settings/logoUrl")
       })
 
-      it("gets url with custom S3", () => {
+      it("gets url with custom S3", async () => {
         testEnv.withS3()
-        const url = getGlobalFileUrl()
+        const url = await getGlobalFileUrl()
         expect(url).toBe("http://s3.example.com/global/settings/logoUrl")
       })
 
-      it("gets url with cloudfront + s3", () => {
+      it("gets url with cloudfront + s3", async () => {
         testEnv.withCloudfront()
-        const url = getGlobalFileUrl()
+        const url = await getGlobalFileUrl()
         // omit rest of signed params
         expect(
           url.includes("http://cf.example.com/settings/logoUrl?etag=etag&")
@@ -41,16 +41,16 @@ describe("global", () => {
 
       it("gets url with embedded minio", async () => {
         testEnv.withMinio()
-        await testEnv.withTenant(tenantId => {
+        await testEnv.withTenant(async tenantId => {
-          const url = getGlobalFileUrl()
+          const url = await getGlobalFileUrl()
           expect(url).toBe(`/files/signed/global/${tenantId}/settings/logoUrl`)
         })
       })
 
       it("gets url with custom S3", async () => {
         testEnv.withS3()
-        await testEnv.withTenant(tenantId => {
+        await testEnv.withTenant(async tenantId => {
-          const url = getGlobalFileUrl()
+          const url = await getGlobalFileUrl()
           expect(url).toBe(
             `http://s3.example.com/global/${tenantId}/settings/logoUrl`
           )
@@ -59,8 +59,8 @@ describe("global", () => {
 
       it("gets url with cloudfront + s3", async () => {
         testEnv.withCloudfront()
-        await testEnv.withTenant(tenantId => {
+        await testEnv.withTenant(async tenantId => {
-          const url = getGlobalFileUrl()
+          const url = await getGlobalFileUrl()
           // omit rest of signed params
           expect(
             url.includes(

packages/backend-core/src/objectStore/buckets/tests/plugins.spec.ts

@@ -6,8 +6,8 @@ describe("plugins", () => {
   describe("enrichPluginURLs", () => {
     const plugin = structures.plugins.plugin()
 
-    function getEnrichedPluginUrls() {
+    async function getEnrichedPluginUrls() {
-      const enriched = plugins.enrichPluginURLs([plugin])[0]
+      const enriched = (await plugins.enrichPluginURLs([plugin]))[0]
       return {
         jsUrl: enriched.jsUrl!,
         iconUrl: enriched.iconUrl!,
@@ -19,9 +19,9 @@ describe("plugins", () => {
         testEnv.singleTenant()
       })
 
-      it("gets url with embedded minio", () => {
+      it("gets url with embedded minio", async () => {
         testEnv.withMinio()
-        const urls = getEnrichedPluginUrls()
+        const urls = await getEnrichedPluginUrls()
         expect(urls.jsUrl).toBe(
           `/files/signed/plugins/${plugin.name}/plugin.min.js`
         )
@@ -30,9 +30,9 @@ describe("plugins", () => {
         )
       })
 
-      it("gets url with custom S3", () => {
+      it("gets url with custom S3", async () => {
         testEnv.withS3()
-        const urls = getEnrichedPluginUrls()
+        const urls = await getEnrichedPluginUrls()
         expect(urls.jsUrl).toBe(
           `http://s3.example.com/plugins/${plugin.name}/plugin.min.js`
         )
@@ -41,9 +41,9 @@ describe("plugins", () => {
         )
       })
 
-      it("gets url with cloudfront + s3", () => {
+      it("gets url with cloudfront + s3", async () => {
         testEnv.withCloudfront()
-        const urls = getEnrichedPluginUrls()
+        const urls = await getEnrichedPluginUrls()
         // omit rest of signed params
         expect(
           urls.jsUrl.includes(
@@ -65,8 +65,8 @@ describe("plugins", () => {
 
       it("gets url with embedded minio", async () => {
         testEnv.withMinio()
-        await testEnv.withTenant(tenantId => {
+        await testEnv.withTenant(async tenantId => {
-          const urls = getEnrichedPluginUrls()
+          const urls = await getEnrichedPluginUrls()
           expect(urls.jsUrl).toBe(
             `/files/signed/plugins/${tenantId}/${plugin.name}/plugin.min.js`
           )
@@ -78,8 +78,8 @@ describe("plugins", () => {
 
       it("gets url with custom S3", async () => {
         testEnv.withS3()
-        await testEnv.withTenant(tenantId => {
+        await testEnv.withTenant(async tenantId => {
-          const urls = getEnrichedPluginUrls()
+          const urls = await getEnrichedPluginUrls()
           expect(urls.jsUrl).toBe(
             `http://s3.example.com/plugins/${tenantId}/${plugin.name}/plugin.min.js`
           )
@@ -91,8 +91,8 @@ describe("plugins", () => {
 
       it("gets url with cloudfront + s3", async () => {
         testEnv.withCloudfront()
-        await testEnv.withTenant(tenantId => {
+        await testEnv.withTenant(async tenantId => {
-          const urls = getEnrichedPluginUrls()
+          const urls = await getEnrichedPluginUrls()
           // omit rest of signed params
           expect(
             urls.jsUrl.includes(

packages/backend-core/src/objectStore/objectStore.ts

@@ -1,6 +1,15 @@
 const sanitize = require("sanitize-s3-objectkey")
 
-import AWS from "aws-sdk"
+import {
+  HeadObjectCommandOutput,
+  PutObjectCommandInput,
+  S3,
+  S3ClientConfig,
+  GetObjectCommand,
+  _Object as S3Object,
+} from "@aws-sdk/client-s3"
+import { Upload } from "@aws-sdk/lib-storage"
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner"
 import stream, { Readable } from "stream"
 import fetch from "node-fetch"
 import tar from "tar-fs"
@@ -13,8 +22,8 @@ import { bucketTTLConfig, budibaseTempDir } from "./utils"
 import { v4 } from "uuid"
 import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
 import fsp from "fs/promises"
-import { HeadObjectOutput } from "aws-sdk/clients/s3"
 import { ReadableStream } from "stream/web"
+import { NodeJsClient } from "@smithy/types"
 
 const streamPipeline = promisify(stream.pipeline)
 // use this as a temporary store of buckets that are being created
@@ -84,26 +93,24 @@ export function sanitizeBucket(input: string) {
  * @constructor
  */
 export function ObjectStore(
-  bucket: string,
   opts: { presigning: boolean } = { presigning: false }
 ) {
-  const config: AWS.S3.ClientConfiguration = {
+  const config: S3ClientConfig = {
-    s3ForcePathStyle: true,
+    forcePathStyle: true,
-    signatureVersion: "v4",
+    credentials: {
-    apiVersion: "2006-03-01",
+      accessKeyId: env.MINIO_ACCESS_KEY!,
-    accessKeyId: env.MINIO_ACCESS_KEY,
+      secretAccessKey: env.MINIO_SECRET_KEY!,
-    secretAccessKey: env.MINIO_SECRET_KEY,
+    },
     region: env.AWS_REGION,
   }
-  if (bucket) {
-    config.params = {
-      Bucket: sanitizeBucket(bucket),
-    }
-  }
 
   // for AWS Credentials using temporary session token
   if (!env.MINIO_ENABLED && env.AWS_SESSION_TOKEN) {
-    config.sessionToken = env.AWS_SESSION_TOKEN
+    config.credentials = {
+      accessKeyId: env.MINIO_ACCESS_KEY!,
+      secretAccessKey: env.MINIO_SECRET_KEY!,
+      sessionToken: env.AWS_SESSION_TOKEN,
+    }
   }
 
   // custom S3 is in use i.e. minio
@@ -113,13 +120,13 @@ export function ObjectStore(
       // Normally a signed url will need to be generated with a specified host in mind.
       // To support dynamic hosts, e.g. some unknown self-hosted installation url,
       // use a predefined host. The host 'minio-service' is also forwarded to minio requests via nginx
-      config.endpoint = "minio-service"
+      config.endpoint = "http://minio-service"
     } else {
       config.endpoint = env.MINIO_URL
     }
   }
 
-  return new AWS.S3(config)
+  return new S3(config) as NodeJsClient<S3>
 }
 
 /**
@@ -132,26 +139,25 @@ export async function createBucketIfNotExists(
 ): Promise<{ created: boolean; exists: boolean }> {
   bucketName = sanitizeBucket(bucketName)
   try {
-    await client
+    await client.headBucket({
-      .headBucket({
       Bucket: bucketName,
     })
-      .promise()
     return { created: false, exists: true }
   } catch (err: any) {
-    const promises: any = STATE.bucketCreationPromises
+    const statusCode = err.statusCode || err.$response?.statusCode
-    const doesntExist = err.statusCode === 404,
+    const promises: Record<string, Promise<any> | undefined> =
-      noAccess = err.statusCode === 403
+      STATE.bucketCreationPromises
+    const doesntExist = statusCode === 404,
+      noAccess = statusCode === 403
     if (promises[bucketName]) {
       await promises[bucketName]
       return { created: false, exists: true }
     } else if (doesntExist || noAccess) {
       if (doesntExist) {
-        promises[bucketName] = client
+        promises[bucketName] = client.createBucket({
-          .createBucket({
           Bucket: bucketName,
         })
-          .promise()
+
         await promises[bucketName]
         delete promises[bucketName]
         return { created: true, exists: false }
@@ -180,25 +186,26 @@ export async function upload({
 
   const fileBytes = path ? (await fsp.open(path)).createReadStream() : body
 
-  const objectStore = ObjectStore(bucketName)
+  const objectStore = ObjectStore()
   const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
 
   if (ttl && bucketCreated.created) {
     let ttlConfig = bucketTTLConfig(bucketName, ttl)
-    await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
+    await objectStore.putBucketLifecycleConfiguration(ttlConfig)
   }
 
   let contentType = type
-  if (!contentType) {
+  const finalContentType = contentType
-    contentType = extension
+    ? contentType
+    : extension
     ? CONTENT_TYPE_MAP[extension.toLowerCase()]
     : CONTENT_TYPE_MAP.txt
-  }
+  const config: PutObjectCommandInput = {
-  const config: any = {
     // windows file paths need to be converted to forward slashes for s3
+    Bucket: sanitizeBucket(bucketName),
     Key: sanitizeKey(filename),
-    Body: fileBytes,
+    Body: fileBytes as stream.Readable | Buffer,
-    ContentType: contentType,
+    ContentType: finalContentType,
   }
   if (metadata && typeof metadata === "object") {
     // remove any nullish keys from the metadata object, as these may be considered invalid
@@ -207,10 +214,15 @@ export async function upload({
         delete metadata[key]
       }
     }
-    config.Metadata = metadata
+    config.Metadata = metadata as Record<string, string>
   }
 
-  return objectStore.upload(config).promise()
+  const upload = new Upload({
+    client: objectStore,
+    params: config,
+  })
+
+  return upload.done()
 }
 
 /**
@@ -229,12 +241,12 @@ export async function streamUpload({
     throw new Error("Stream to upload is invalid/undefined")
   }
   const extension = filename.split(".").pop()
-  const objectStore = ObjectStore(bucketName)
+  const objectStore = ObjectStore()
   const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
 
   if (ttl && bucketCreated.created) {
     let ttlConfig = bucketTTLConfig(bucketName, ttl)
-    await objectStore.putBucketLifecycleConfiguration(ttlConfig).promise()
+    await objectStore.putBucketLifecycleConfiguration(ttlConfig)
   }
 
   // Set content type for certain known extensions
@@ -267,13 +279,15 @@ export async function streamUpload({
     ...extra,
   }
 
-  const details = await objectStore.upload(params).promise()
+  const upload = new Upload({
-  const headDetails = await objectStore
+    client: objectStore,
-    .headObject({
+    params,
+  })
+  const details = await upload.done()
+  const headDetails = await objectStore.headObject({
     Bucket: bucket,
     Key: objKey,
   })
-    .promise()
   return {
     ...details,
     ContentLength: headDetails.ContentLength,
@@ -284,35 +298,46 @@ export async function streamUpload({
  * retrieves the contents of a file from the object store, if it is a known content type it
  * will be converted, otherwise it will be returned as a buffer stream.
  */
-export async function retrieve(bucketName: string, filepath: string) {
+export async function retrieve(
-  const objectStore = ObjectStore(bucketName)
+  bucketName: string,
+  filepath: string
+): Promise<string | stream.Readable> {
+  const objectStore = ObjectStore()
   const params = {
     Bucket: sanitizeBucket(bucketName),
     Key: sanitizeKey(filepath),
   }
-  const response: any = await objectStore.getObject(params).promise()
+  const response = await objectStore.getObject(params)
-  // currently these are all strings
+  if (!response.Body) {
+    throw new Error("Unable to retrieve object")
+  }
   if (STRING_CONTENT_TYPES.includes(response.ContentType)) {
-    return response.Body.toString("utf8")
+    return response.Body.transformToString()
   } else {
-    return response.Body
+    // this typecast is required - for some reason the AWS SDK V3 defines its own "ReadableStream"
+    // found in the @aws-sdk/types package which is meant to be the Node type, but due to the SDK
+    // supporting both the browser and Nodejs it is a polyfill which causes a type clash with Node.
+    const readableStream =
+      response.Body.transformToWebStream() as ReadableStream
+    return stream.Readable.fromWeb(readableStream)
   }
 }
 
-export async function listAllObjects(bucketName: string, path: string) {
+export async function listAllObjects(
-  const objectStore = ObjectStore(bucketName)
+  bucketName: string,
+  path: string
+): Promise<S3Object[]> {
+  const objectStore = ObjectStore()
   const list = (params: ListParams = {}) => {
-    return objectStore
+    return objectStore.listObjectsV2({
-      .listObjectsV2({
       ...params,
       Bucket: sanitizeBucket(bucketName),
       Prefix: sanitizeKey(path),
     })
-      .promise()
   }
   let isTruncated = false,
     token,
-    objects: AWS.S3.Types.Object[] = []
+    objects: Object[] = []
   do {
     let params: ListParams = {}
     if (token) {
@@ -331,18 +356,19 @@ export async function listAllObjects(bucketName: string, path: string) {
 /**
  * Generate a presigned url with a default TTL of 1 hour
  */
-export function getPresignedUrl(
+export async function getPresignedUrl(
   bucketName: string,
   key: string,
   durationSeconds = 3600
 ) {
-  const objectStore = ObjectStore(bucketName, { presigning: true })
+  const objectStore = ObjectStore({ presigning: true })
   const params = {
     Bucket: sanitizeBucket(bucketName),
     Key: sanitizeKey(key),
-    Expires: durationSeconds,
   }
-  const url = objectStore.getSignedUrl("getObject", params)
+  const url = await getSignedUrl(objectStore, new GetObjectCommand(params), {
+    expiresIn: durationSeconds,
+  })
 
   if (!env.MINIO_ENABLED) {
     // return the full URL to the client
@@ -366,7 +392,11 @@ export async function retrieveToTmp(bucketName: string, filepath: string) {
   filepath = sanitizeKey(filepath)
   const data = await retrieve(bucketName, filepath)
   const outputPath = join(budibaseTempDir(), v4())
+  if (data instanceof stream.Readable) {
+    data.pipe(fs.createWriteStream(outputPath))
+  } else {
     fs.writeFileSync(outputPath, data)
+  }
   return outputPath
 }
 
@@ -394,7 +424,7 @@ export async function retrieveDirectory(bucketName: string, path: string) {
     stream.pipe(writeStream)
     writePromises.push(
       new Promise((resolve, reject) => {
-        stream.on("finish", resolve)
+        writeStream.on("finish", resolve)
         stream.on("error", reject)
         writeStream.on("error", reject)
       })
@@ -408,17 +438,17 @@ export async function retrieveDirectory(bucketName: string, path: string) {
  * Delete a single file.
  */
 export async function deleteFile(bucketName: string, filepath: string) {
-  const objectStore = ObjectStore(bucketName)
+  const objectStore = ObjectStore()
   await createBucketIfNotExists(objectStore, bucketName)
   const params = {
     Bucket: bucketName,
     Key: sanitizeKey(filepath),
   }
-  return objectStore.deleteObject(params).promise()
+  return objectStore.deleteObject(params)
 }
 
 export async function deleteFiles(bucketName: string, filepaths: string[]) {
-  const objectStore = ObjectStore(bucketName)
+  const objectStore = ObjectStore()
   await createBucketIfNotExists(objectStore, bucketName)
   const params = {
     Bucket: bucketName,
@@ -426,7 +456,7 @@ export async function deleteFiles(bucketName: string, filepaths: string[]) {
       Objects: filepaths.map((path: any) => ({ Key: sanitizeKey(path) })),
     },
   }
-  return objectStore.deleteObjects(params).promise()
+  return objectStore.deleteObjects(params)
 }
 
 /**
@@ -438,13 +468,13 @@ export async function deleteFolder(
 ): Promise<any> {
   bucketName = sanitizeBucket(bucketName)
   folder = sanitizeKey(folder)
-  const client = ObjectStore(bucketName)
+  const client = ObjectStore()
   const listParams = {
     Bucket: bucketName,
     Prefix: folder,
   }
 
-  const existingObjectsResponse = await client.listObjects(listParams).promise()
+  const existingObjectsResponse = await client.listObjects(listParams)
   if (existingObjectsResponse.Contents?.length === 0) {
     return
   }
@@ -459,7 +489,7 @@ export async function deleteFolder(
     deleteParams.Delete.Objects.push({ Key: content.Key })
   })
 
-  const deleteResponse = await client.deleteObjects(deleteParams).promise()
+  const deleteResponse = await client.deleteObjects(deleteParams)
   // can only empty 1000 items at once
   if (deleteResponse.Deleted?.length === 1000) {
     return deleteFolder(bucketName, folder)
@@ -534,29 +564,33 @@ export async function getReadStream(
 ): Promise<Readable> {
   bucketName = sanitizeBucket(bucketName)
   path = sanitizeKey(path)
-  const client = ObjectStore(bucketName)
+  const client = ObjectStore()
   const params = {
     Bucket: bucketName,
     Key: path,
   }
-  return client.getObject(params).createReadStream()
+  const response = await client.getObject(params)
+  if (!response.Body || !(response.Body instanceof stream.Readable)) {
+    throw new Error("Unable to retrieve stream - invalid response")
+  }
+  return response.Body
 }
 
 export async function getObjectMetadata(
   bucket: string,
   path: string
-): Promise<HeadObjectOutput> {
+): Promise<HeadObjectCommandOutput> {
   bucket = sanitizeBucket(bucket)
   path = sanitizeKey(path)
 
-  const client = ObjectStore(bucket)
+  const client = ObjectStore()
   const params = {
     Bucket: bucket,
     Key: path,
   }
 
   try {
-    return await client.headObject(params).promise()
+    return await client.headObject(params)
   } catch (err: any) {
     throw new Error("Unable to retrieve metadata from object")
   }

packages/backend-core/src/objectStore/utils.ts

@@ -2,7 +2,10 @@ import path, { join } from "path"
 import { tmpdir } from "os"
 import fs from "fs"
 import env from "../environment"
-import { PutBucketLifecycleConfigurationRequest } from "aws-sdk/clients/s3"
+import {
+  LifecycleRule,
+  PutBucketLifecycleConfigurationCommandInput,
+} from "@aws-sdk/client-s3"
 import * as objectStore from "./objectStore"
 import {
   AutomationAttachment,
@@ -43,8 +46,8 @@ export function budibaseTempDir() {
 export const bucketTTLConfig = (
   bucketName: string,
   days: number
-): PutBucketLifecycleConfigurationRequest => {
+): PutBucketLifecycleConfigurationCommandInput => {
-  const lifecycleRule = {
+  const lifecycleRule: LifecycleRule = {
     ID: `${bucketName}-ExpireAfter${days}days`,
     Prefix: "",
     Status: "Enabled",

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -3,6 +3,7 @@ import { newid } from "../utils"
 import { Queue, QueueOptions, JobOptions } from "./queue"
 import { helpers } from "@budibase/shared-core"
 import { Job, JobId, JobInformation } from "bull"
+import { cloneDeep } from "lodash"
 
 function jobToJobInformation(job: Job): JobInformation {
   let cron = ""
@@ -33,12 +34,13 @@ function jobToJobInformation(job: Job): JobInformation {
   }
 }
 
-interface JobMessage<T = any> extends Partial<Job<T>> {
+export interface TestQueueMessage<T = any> extends Partial<Job<T>> {
   id: string
   timestamp: number
   queue: Queue<T>
   data: any
   opts?: JobOptions
+  manualTrigger?: boolean
 }
 
 /**
@@ -47,12 +49,16 @@ interface JobMessage<T = any> extends Partial<Job<T>> {
  * internally to register when messages are available to the consumers - in can
  * support many inputs and many consumers.
  */
-class InMemoryQueue implements Partial<Queue> {
+export class InMemoryQueue<T = any> implements Partial<Queue<T>> {
   _name: string
   _opts?: QueueOptions
-  _messages: JobMessage[]
+  _messages: TestQueueMessage<T>[]
   _queuedJobIds: Set<string>
-  _emitter: NodeJS.EventEmitter<{ message: [JobMessage]; completed: [Job] }>
+  _emitter: NodeJS.EventEmitter<{
+    message: [TestQueueMessage<T>]
+    completed: [Job<T>]
+    removed: [TestQueueMessage<T>]
+  }>
   _runCount: number
   _addCount: number
 
@@ -82,7 +88,15 @@ class InMemoryQueue implements Partial<Queue> {
    */
   async process(concurrencyOrFunc: number | any, func?: any) {
     func = typeof concurrencyOrFunc === "number" ? func : concurrencyOrFunc
-    this._emitter.on("message", async message => {
+    this._emitter.on("message", async msg => {
+      const message = cloneDeep(msg)
+
+      // For the purpose of testing, don't trigger cron jobs immediately.
+      // Require the test to trigger them manually with timestamps.
+      if (!message.manualTrigger && message.opts?.repeat != null) {
+        return
+      }
+
       let resp = func(message)
 
       async function retryFunc(fnc: any) {
@@ -97,7 +111,7 @@ class InMemoryQueue implements Partial<Queue> {
       if (resp.then != null) {
         try {
           await retryFunc(resp)
-          this._emitter.emit("completed", message as Job)
+          this._emitter.emit("completed", message as Job<T>)
         } catch (e: any) {
           console.error(e)
         }
@@ -114,7 +128,6 @@ class InMemoryQueue implements Partial<Queue> {
     return this as any
   }
 
-  // simply puts a message to the queue and emits to the queue for processing
   /**
    * Simple function to replicate the add message functionality of Bull, putting
    * a new message on the queue. This then emits an event which will be used to
@@ -123,7 +136,13 @@ class InMemoryQueue implements Partial<Queue> {
    * a JSON message as this is required by Bull.
    * @param repeat serves no purpose for the import queue.
    */
-  async add(data: any, opts?: JobOptions) {
+  async add(data: T | string, optsOrT?: JobOptions | T) {
+    if (typeof data === "string") {
+      throw new Error("doesn't support named jobs")
+    }
+
+    const opts = optsOrT as JobOptions
+
     const jobId = opts?.jobId?.toString()
     if (jobId && this._queuedJobIds.has(jobId)) {
       console.log(`Ignoring already queued job ${jobId}`)
@@ -138,7 +157,7 @@ class InMemoryQueue implements Partial<Queue> {
     }
 
     const pushMessage = () => {
-      const message: JobMessage = {
+      const message: TestQueueMessage = {
         id: newid(),
         timestamp: Date.now(),
         queue: this as unknown as Queue,
@@ -164,13 +183,14 @@ class InMemoryQueue implements Partial<Queue> {
    */
   async close() {}
 
-  /**
+  async removeRepeatableByKey(id: string) {
-   * This removes a cron which has been implemented, this is part of Bull API.
+    for (const [idx, message] of this._messages.entries()) {
-   * @param cronJobId The cron which is to be removed.
+      if (message.id === id) {
-   */
+        this._messages.splice(idx, 1)
-  async removeRepeatableByKey(cronJobId: string) {
+        this._emitter.emit("removed", message)
-    // TODO: implement for testing
+        return
-    console.log(cronJobId)
+      }
+    }
   }
 
   async removeJobs(_pattern: string) {
@@ -193,12 +213,28 @@ class InMemoryQueue implements Partial<Queue> {
     return null
   }
 
+  manualTrigger(id: JobId) {
+    for (const message of this._messages) {
+      if (message.id === id) {
+        this._emitter.emit("message", { ...message, manualTrigger: true })
+        return
+      }
+    }
+    throw new Error(`Job with id ${id} not found`)
+  }
+
   on(event: string, callback: (...args: any[]) => void): Queue {
     // @ts-expect-error - this callback can be one of many types
     this._emitter.on(event, callback)
     return this as unknown as Queue
   }
 
+  off(event: string, callback: (...args: any[]) => void): Queue {
+    // @ts-expect-error - this callback can be one of many types
+    this._emitter.off(event, callback)
+    return this as unknown as Queue
+  }
+
   async count() {
     return this._messages.length
   }
@@ -208,7 +244,9 @@ class InMemoryQueue implements Partial<Queue> {
   }
 
   async getRepeatableJobs() {
-    return this._messages.map(job => jobToJobInformation(job as Job))
+    return this._messages
+      .filter(job => job.opts?.repeat != null)
+      .map(job => jobToJobInformation(job as Job))
   }
 }
 

packages/backend-core/src/queue/index.ts

@@ -1,2 +1,3 @@
 export * from "./queue"
 export * from "./constants"
+export * from "./inMemoryQueue"

packages/backend-core/src/sql/utils.ts

@@ -5,10 +5,10 @@ import {
   SqlQuery,
   Table,
   TableSourceType,
+  SEPARATOR,
 } from "@budibase/types"
 import { DEFAULT_BB_DATASOURCE_ID } from "../constants"
 import { Knex } from "knex"
-import { SEPARATOR } from "../db"
 import environment from "../environment"
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`

packages/backend-core/src/users/db.ts

@@ -264,7 +264,9 @@ export class UserDB {
     const creatorsChange =
       (await isCreator(dbUser)) !== (await isCreator(user)) ? 1 : 0
     return UserDB.quotas.addUsers(change, creatorsChange, async () => {
+      if (!opts.isAccountHolder) {
         await validateUniqueUser(email, tenantId)
+      }
 
       let builtUser = await UserDB.buildUser(user, opts, tenantId, dbUser)
       // don't allow a user to update its own roles/perms
@@ -569,6 +571,7 @@ export class UserDB {
       hashPassword: opts?.hashPassword,
       requirePassword: opts?.requirePassword,
       skipPasswordValidation: opts?.skipPasswordValidation,
+      isAccountHolder: true,
     })
   }
 

packages/backend-core/src/utils/Duration.ts

@@ -15,23 +15,27 @@ const conversion: Record<DurationType, number> = {
 }
 
 export class Duration {
+  constructor(public ms: number) {}
+
+  to(type: DurationType) {
+    return this.ms / conversion[type]
+  }
+
+  toMs() {
+    return this.ms
+  }
+
+  toSeconds() {
+    return this.to(DurationType.SECONDS)
+  }
+
   static convert(from: DurationType, to: DurationType, duration: number) {
     const milliseconds = duration * conversion[from]
     return milliseconds / conversion[to]
   }
 
   static from(from: DurationType, duration: number) {
-    return {
+    return new Duration(duration * conversion[from])
-      to: (to: DurationType) => {
-        return Duration.convert(from, to, duration)
-      },
-      toMs: () => {
-        return Duration.convert(from, DurationType.MILLISECONDS, duration)
-      },
-      toSeconds: () => {
-        return Duration.convert(from, DurationType.SECONDS, duration)
-      },
-    }
   }
 
   static fromSeconds(duration: number) {

packages/backend-core/src/utils/index.ts

@@ -2,3 +2,4 @@ export * from "./hashing"
 export * from "./utils"
 export * from "./stringUtils"
 export * from "./Duration"
+export * from "./time"

packages/backend-core/src/utils/tests/utils.spec.ts

@@ -67,6 +67,15 @@ describe("utils", () => {
       })
     })
 
+    it("gets appId from query params", async () => {
+      const ctx = structures.koa.newContext()
+      const expected = db.generateAppID()
+      ctx.query = { appId: expected }
+
+      const actual = await utils.getAppIdFromCtx(ctx)
+      expect(actual).toBe(expected)
+    })
+
     it("doesn't get appId from url when previewing", async () => {
       const ctx = structures.koa.newContext()
       const appId = db.generateAppID()

packages/backend-core/src/utils/time.ts

@@ -0,0 +1,7 @@
+import { Duration } from "./Duration"
+
+export async function time<T>(f: () => Promise<T>): Promise<[T, Duration]> {
+  const start = performance.now()
+  const result = await f()
+  return [result, Duration.fromMilliseconds(performance.now() - start)]
+}

packages/backend-core/src/utils/utils.ts

@@ -101,6 +101,11 @@ export async function getAppIdFromCtx(ctx: Ctx) {
     appId = confirmAppId(pathId)
   }
 
+  // look in queryParams
+  if (!appId && ctx.query?.appId) {
+    appId = confirmAppId(ctx.query?.appId as string)
+  }
+
   // lookup using custom url - prod apps only
   // filter out the builder preview path which collides with the prod app path
   // to ensure we don't load all apps excessively
@@ -247,3 +252,7 @@ export function hasCircularStructure(json: any) {
   }
   return false
 }
+
+export function urlHasProtocol(url: string): boolean {
+  return !!url.match(/^.+:\/\/.+$/)
+}

packages/bbui/src/Actions/click_outside.ts

@@ -93,7 +93,10 @@ const handleMouseDown = (e: MouseEvent) => {
 
 // Handle iframe clicks by detecting a loss of focus on the main window
 const handleBlur = () => {
-  if (document.activeElement?.tagName === "IFRAME") {
+  if (
+    document.activeElement &&
+    ["IFRAME", "BODY"].includes(document.activeElement.tagName)
+  ) {
     handleClick(
       new MouseEvent("click", { relatedTarget: document.activeElement })
     )

packages/bbui/src/Form/Core/Multiselect.svelte

@@ -14,7 +14,7 @@
   export let sort = false
   export let autoWidth = false
   export let searchTerm = null
-  export let customPopoverHeight
+  export let customPopoverHeight = undefined
   export let open = false
   export let loading
   export let onOptionMouseenter = () => {}

packages/bbui/src/Typography/Body.svelte

@@ -1,11 +1,11 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/typography/dist/index-vars.css"
 
-  export let size = "M"
+  export let size: "XS" | "S" | "M" | "L" | "XL" = "M"
-  export let serif = false
+  export let serif: boolean = false
-  export let weight = null
+  export let weight: string | null = null
-  export let textAlign = null
+  export let textAlign: string | null = null
-  export let color = null
+  export let color: string | null = null
 </script>
 
 <p

packages/builder/package.json

@@ -53,7 +53,7 @@
     "@budibase/shared-core": "*",
     "@budibase/string-templates": "*",
     "@budibase/types": "*",
-    "@codemirror/autocomplete": "^6.7.1",
+    "@codemirror/autocomplete": "6.9.0",
     "@codemirror/commands": "^6.2.4",
     "@codemirror/lang-javascript": "^6.1.8",
     "@codemirror/language": "^6.6.0",

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -386,7 +386,7 @@
       editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
     } else if (editableColumn.type === FieldType.FORMULA) {
       editableColumn.formulaType = "dynamic"
-      editableColumn.responseType = field.responseType || FIELDS.STRING.type
+      editableColumn.responseType = field?.responseType || FIELDS.STRING.type
     }
   }
 

packages/builder/src/components/backend/modals/DeleteDataConfirmationModal.svelte

@@ -151,6 +151,8 @@
     const screenCount = affectedScreens.length
     let message = `Removing ${source?.name} `
     let initialLength = message.length
+    const hasChanged = () => message.length !== initialLength
+
     if (sourceType === SourceType.TABLE) {
       const views = "views" in source ? Object.values(source?.views ?? []) : []
       message += `will delete its data${
@@ -169,10 +171,10 @@
         initialLength !== message.length
           ? ", and break connected screens:"
           : "will break connected screens:"
-    } else {
+    } else if (hasChanged()) {
       message += "."
     }
-    return message.length !== initialLength ? message : ""
+    return hasChanged() ? message : ""
   }
 </script>
 

packages/builder/src/components/common/CodeEditor/CodeEditor.svelte

@@ -40,15 +40,19 @@
     indentMore,
     indentLess,
   } from "@codemirror/commands"
+  import { setDiagnostics } from "@codemirror/lint"
   import { Compartment, EditorState } from "@codemirror/state"
+  import type { Extension } from "@codemirror/state"
   import { javascript } from "@codemirror/lang-javascript"
   import { EditorModes } from "./"
   import { themeStore } from "@/stores/portal"
   import type { EditorMode } from "@budibase/types"
+  import type { BindingCompletion, CodeValidator } from "@/types"
+  import { validateHbsTemplate } from "./validator/hbs"
 
   export let label: string | undefined = undefined
-  // TODO: work out what best type fits this
+  export let completions: BindingCompletion[] = []
-  export let completions: any[] = []
+  export let validations: CodeValidator | null = null
   export let mode: EditorMode = EditorModes.Handlebars
   export let value: string | null = ""
   export let placeholder: string | null = null
@@ -247,7 +251,7 @@
   // None of this is reactive, but it never has been, so we just assume most
   // config flags aren't changed at runtime
   // TODO: work out type for base
-  const buildExtensions = (base: any[]) => {
+  const buildExtensions = (base: Extension[]) => {
     let complete = [...base]
 
     if (autocompleteEnabled) {
@@ -339,6 +343,24 @@
     return complete
   }
 
+  function validate(
+    value: string | null,
+    editor: EditorView | undefined,
+    mode: EditorMode,
+    validations: CodeValidator | null
+  ) {
+    if (!value || !validations || !editor) {
+      return
+    }
+
+    if (mode === EditorModes.Handlebars) {
+      const diagnostics = validateHbsTemplate(value, validations)
+      editor.dispatch(setDiagnostics(editor.state, diagnostics))
+    }
+  }
+
+  $: validate(value, editor, mode, validations)
+
   const initEditor = () => {
     const baseExtensions = buildBaseExtensions()
 
@@ -365,7 +387,6 @@
     <Label size="S">{label}</Label>
   </div>
 {/if}
-
 <div class={`code-editor ${mode?.name || ""}`}>
   <div tabindex="-1" bind:this={textarea} />
 </div>

packages/builder/src/components/common/CodeEditor/index.ts

@@ -1,13 +1,10 @@
 import { getManifest } from "@budibase/string-templates"
 import sanitizeHtml from "sanitize-html"
 import { groupBy } from "lodash"
-import {
+import { EditorModesMap, Helper, Snippet } from "@budibase/types"
-  BindingCompletion,
-  EditorModesMap,
-  Helper,
-  Snippet,
-} from "@budibase/types"
 import { CompletionContext } from "@codemirror/autocomplete"
+import { EditorView } from "@codemirror/view"
+import { BindingCompletion, BindingCompletionOption } from "@/types"
 
 export const EditorModes: EditorModesMap = {
   JS: {
@@ -25,15 +22,7 @@ export const EditorModes: EditorModesMap = {
   },
 }
 
-export const SECTIONS = {
+const buildHelperInfoNode = (helper: Helper) => {
-  HB_HELPER: {
-    name: "Helper",
-    type: "helper",
-    icon: "Code",
-  },
-}
-
-export const buildHelperInfoNode = (completion: any, helper: Helper) => {
   const ele = document.createElement("div")
   ele.classList.add("info-bubble")
 
@@ -65,7 +54,7 @@ const toSpectrumIcon = (name: string) => {
   </svg>`
 }
 
-export const buildSectionHeader = (
+const buildSectionHeader = (
   type: string,
   sectionName: string,
   icon: string,
@@ -84,30 +73,29 @@ export const buildSectionHeader = (
   }
 }
 
-export const helpersToCompletion = (
+const helpersToCompletion = (
   helpers: Record<string, Helper>,
   mode: { name: "javascript" | "handlebars" }
-) => {
+): BindingCompletionOption[] => {
-  const { type, name: sectionName, icon } = SECTIONS.HB_HELPER
+  const helperSection = buildSectionHeader("helper", "Helpers", "Code", 99)
-  const helperSection = buildSectionHeader(type, sectionName, icon, 99)
 
   return Object.keys(helpers).flatMap(helperName => {
-    let helper = helpers[helperName]
+    const helper = helpers[helperName]
     return {
       label: helperName,
-      info: (completion: BindingCompletion) => {
+      args: helper.args,
-        return buildHelperInfoNode(completion, helper)
+      requiresBlock: helper.requiresBlock,
-      },
+      info: () => buildHelperInfoNode(helper),
       type: "helper",
       section: helperSection,
       detail: "Function",
       apply: (
-        view: any,
+        view: EditorView,
-        completion: BindingCompletion,
+        _completion: BindingCompletionOption,
         from: number,
         to: number
       ) => {
-        insertBinding(view, from, to, helperName, mode)
+        insertBinding(view, from, to, helperName, mode, AutocompleteType.HELPER)
       },
     }
   })
@@ -115,7 +103,7 @@ export const helpersToCompletion = (
 
 export const getHelperCompletions = (mode: {
   name: "javascript" | "handlebars"
-}) => {
+}): BindingCompletionOption[] => {
   // TODO: manifest needs to be properly typed
   const manifest: any = getManifest()
   return Object.keys(manifest).flatMap(key => {
@@ -123,52 +111,40 @@ export const getHelperCompletions = (mode: {
   })
 }
 
-export const snippetAutoComplete = (snippets: Snippet[]) => {
+export const snippetAutoComplete = (snippets: Snippet[]): BindingCompletion => {
-  return function myCompletions(context: CompletionContext) {
+  return setAutocomplete(
-    if (!snippets?.length) {
+    snippets.map(snippet => ({
-      return null
+      section: buildSectionHeader("snippets", "Snippets", "Code", 100),
-    }
-    const word = context.matchBefore(/\w*/)
-    if (!word || (word.from == word.to && !context.explicit)) {
-      return null
-    }
-    return {
-      from: word.from,
-      options: snippets.map(snippet => ({
       label: `snippets.${snippet.name}`,
-        type: "text",
+      displayLabel: snippet.name,
-        simple: true,
+    }))
-        apply: (
+  )
-          view: any,
-          completion: BindingCompletion,
-          from: number,
-          to: number
-        ) => {
-          insertSnippet(view, from, to, completion.label)
-        },
-      })),
-    }
-  }
 }
 
-const bindingFilter = (options: BindingCompletion[], query: string) => {
+const bindingFilter = (options: BindingCompletionOption[], query: string) => {
   return options.filter(completion => {
-    const section_parsed = completion.section.name.toLowerCase()
+    const section_parsed = completion.section?.toString().toLowerCase()
     const label_parsed = completion.label.toLowerCase()
     const query_parsed = query.toLowerCase()
 
     return (
-      section_parsed.includes(query_parsed) ||
+      section_parsed?.includes(query_parsed) ||
       label_parsed.includes(query_parsed)
     )
   })
 }
 
-export const hbAutocomplete = (baseCompletions: BindingCompletion[]) => {
+export const hbAutocomplete = (
-  async function coreCompletion(context: CompletionContext) {
+  baseCompletions: BindingCompletionOption[]
-    let bindingStart = context.matchBefore(EditorModes.Handlebars.match)
+): BindingCompletion => {
+  function coreCompletion(context: CompletionContext) {
+    if (!baseCompletions.length) {
+      return null
+    }
 
-    let options = baseCompletions || []
+    const bindingStart = context.matchBefore(EditorModes.Handlebars.match)
+
+    const options = baseCompletions
 
     if (!bindingStart) {
       return null
@@ -179,7 +155,7 @@ export const hbAutocomplete = (baseCompletions: BindingCompletion[]) => {
       return null
     }
     const query = bindingStart.text.replace(match[0], "")
-    let filtered = bindingFilter(options, query)
+    const filtered = bindingFilter(options, query)
 
     return {
       from: bindingStart.from + match[0].length,
@@ -191,10 +167,20 @@ export const hbAutocomplete = (baseCompletions: BindingCompletion[]) => {
   return coreCompletion
 }
 
-export const jsAutocomplete = (baseCompletions: BindingCompletion[]) => {
+function wrappedAutocompleteMatch(context: CompletionContext) {
-  async function coreCompletion(context: CompletionContext) {
+  return context.matchBefore(/\$\("[\s\w]*/)
-    let jsBinding = context.matchBefore(/\$\("[\s\w]*/)
+}
-    let options = baseCompletions || []
+
+export const jsAutocomplete = (
+  baseCompletions: BindingCompletionOption[]
+): BindingCompletion => {
+  function coreCompletion(context: CompletionContext) {
+    if (!baseCompletions.length) {
+      return null
+    }
+
+    const jsBinding = wrappedAutocompleteMatch(context)
+    const options = baseCompletions
 
     if (jsBinding) {
       // Accommodate spaces
@@ -217,10 +203,46 @@ export const jsAutocomplete = (baseCompletions: BindingCompletion[]) => {
   return coreCompletion
 }
 
-export const buildBindingInfoNode = (
+export const jsHelperAutocomplete = (
-  completion: BindingCompletion,
+  baseCompletions: BindingCompletionOption[]
-  binding: any
+): BindingCompletion => {
-) => {
+  return setAutocomplete(
+    baseCompletions.map(helper => ({
+      ...helper,
+      displayLabel: helper.label,
+      label: `helpers.${helper.label}()`,
+    }))
+  )
+}
+
+function setAutocomplete(
+  options: BindingCompletionOption[]
+): BindingCompletion {
+  return function (context: CompletionContext) {
+    if (!options.length) {
+      return null
+    }
+
+    if (wrappedAutocompleteMatch(context)) {
+      return null
+    }
+
+    const word = context.matchBefore(/\b\w*(\.\w*)?/)
+    if (!word || (word.from == word.to && !context.explicit)) {
+      return null
+    }
+
+    return {
+      from: word.from,
+      options,
+    }
+  }
+}
+
+const buildBindingInfoNode = (binding: {
+  valueHTML: string
+  value: string | null
+}) => {
   if (!binding.valueHTML || binding.value == null) {
     return null
   }
@@ -278,18 +300,28 @@ export function jsInsert(
   return parsedInsert
 }
 
+const enum AutocompleteType {
+  BINDING,
+  HELPER,
+  TEXT,
+}
+
 // Autocomplete apply behaviour
-export const insertBinding = (
+const insertBinding = (
-  view: any,
+  view: EditorView,
   from: number,
   to: number,
   text: string,
-  mode: { name: "javascript" | "handlebars" }
+  mode: { name: "javascript" | "handlebars" },
+  type: AutocompleteType
 ) => {
   let parsedInsert
 
   if (mode.name == "javascript") {
-    parsedInsert = jsInsert(view.state.doc?.toString(), from, to, text)
+    parsedInsert = jsInsert(view.state.doc?.toString(), from, to, text, {
+      helper: type === AutocompleteType.HELPER,
+      disableWrapping: type === AutocompleteType.TEXT,
+    })
   } else if (mode.name == "handlebars") {
     parsedInsert = hbInsert(view.state.doc?.toString(), from, to, text)
   } else {
@@ -319,30 +351,11 @@ export const insertBinding = (
   })
 }
 
-export const insertSnippet = (
-  view: any,
-  from: number,
-  to: number,
-  text: string
-) => {
-  let cursorPos = from + text.length
-  view.dispatch({
-    changes: {
-      from,
-      to,
-      insert: text,
-    },
-    selection: {
-      anchor: cursorPos,
-    },
-  })
-}
-
 // TODO: typing in this function isn't great
 export const bindingsToCompletions = (
   bindings: any,
   mode: { name: "javascript" | "handlebars" }
-) => {
+): BindingCompletionOption[] => {
   const bindingByCategory = groupBy(bindings, "category")
   const categoryMeta = bindings?.reduce((acc: any, ele: any) => {
     acc[ele.category] = acc[ele.category] || {}
@@ -356,8 +369,9 @@ export const bindingsToCompletions = (
     return acc
   }, {})
 
-  const completions = Object.keys(bindingByCategory).reduce(
+  const completions = Object.keys(bindingByCategory).reduce<
-    (comps: any, catKey: string) => {
+    BindingCompletionOption[]
+  >((comps, catKey) => {
     const { icon, rank } = categoryMeta[catKey] || {}
 
     const bindingSectionHeader = buildSectionHeader(
@@ -368,34 +382,41 @@ export const bindingsToCompletions = (
       typeof rank == "number" ? rank : 1
     )
 
-      return [
+    comps.push(
-        ...comps,
+      ...bindingByCategory[catKey].reduce<BindingCompletionOption[]>(
-        ...bindingByCategory[catKey].reduce((acc, binding) => {
+        (acc, binding) => {
           let displayType = binding.fieldSchema?.type || binding.display?.type
           acc.push({
             label:
               binding.display?.name || binding.readableBinding || "NO NAME",
-            info: (completion: BindingCompletion) => {
+            info: () => buildBindingInfoNode(binding),
-              return buildBindingInfoNode(completion, binding)
-            },
             type: "binding",
             detail: displayType,
             section: bindingSectionHeader,
             apply: (
-              view: any,
+              view: EditorView,
-              completion: BindingCompletion,
+              _completion: BindingCompletionOption,
               from: number,
               to: number
             ) => {
-              insertBinding(view, from, to, binding.readableBinding, mode)
+              insertBinding(
+                view,
+                from,
+                to,
+                binding.readableBinding,
+                mode,
+                AutocompleteType.BINDING
+              )
             },
           })
           return acc
-        }, []),
-      ]
         },
         []
       )
+    )
+
+    return comps
+  }, [])
 
   return completions
 }

packages/builder/src/components/common/CodeEditor/validator/hbs.ts

@@ -0,0 +1,113 @@
+/* global hbs */
+import Handlebars from "handlebars"
+import type { Diagnostic } from "@codemirror/lint"
+import { CodeValidator } from "@/types"
+
+function isMustacheStatement(
+  node: hbs.AST.Statement
+): node is hbs.AST.MustacheStatement {
+  return node.type === "MustacheStatement"
+}
+
+function isBlockStatement(
+  node: hbs.AST.Statement
+): node is hbs.AST.BlockStatement {
+  return node.type === "BlockStatement"
+}
+
+function isPathExpression(
+  node: hbs.AST.Statement
+): node is hbs.AST.PathExpression {
+  return node.type === "PathExpression"
+}
+
+export function validateHbsTemplate(
+  text: string,
+  validations: CodeValidator
+): Diagnostic[] {
+  const diagnostics: Diagnostic[] = []
+
+  try {
+    const ast = Handlebars.parse(text, {})
+
+    const lineOffsets: number[] = []
+    let offset = 0
+    for (const line of text.split("\n")) {
+      lineOffsets.push(offset)
+      offset += line.length + 1 // +1 for newline character
+    }
+
+    function traverseNodes(
+      nodes: hbs.AST.Statement[],
+      options?: {
+        ignoreMissing?: boolean
+      }
+    ) {
+      const ignoreMissing = options?.ignoreMissing || false
+      nodes.forEach(node => {
+        if (isMustacheStatement(node) && isPathExpression(node.path)) {
+          const helperName = node.path.original
+
+          const from =
+            lineOffsets[node.loc.start.line - 1] + node.loc.start.column
+          const to = lineOffsets[node.loc.end.line - 1] + node.loc.end.column
+
+          if (!(helperName in validations)) {
+            if (!ignoreMissing) {
+              diagnostics.push({
+                from,
+                to,
+                severity: "warning",
+                message: `"${helperName}" handler does not exist.`,
+              })
+            }
+            return
+          }
+
+          const { arguments: expectedArguments = [], requiresBlock } =
+            validations[helperName]
+
+          if (requiresBlock && !isBlockStatement(node)) {
+            diagnostics.push({
+              from,
+              to,
+              severity: "error",
+              message: `Helper "${helperName}" requires a body:\n{{#${helperName} ...}} [body] {{/${helperName}}}`,
+            })
+            return
+          }
+
+          const providedParams = node.params
+
+          if (providedParams.length !== expectedArguments.length) {
+            diagnostics.push({
+              from,
+              to,
+              severity: "error",
+              message: `Helper "${helperName}" expects ${
+                expectedArguments.length
+              } parameters (${expectedArguments.join(", ")}), but got ${
+                providedParams.length
+              }.`,
+            })
+          }
+        }
+
+        if (isBlockStatement(node)) {
+          traverseNodes(node.program.body, { ignoreMissing: true })
+        }
+      })
+    }
+
+    traverseNodes(ast.body, { ignoreMissing: true })
+  } catch (e: any) {
+    diagnostics.push({
+      from: 0,
+      to: text.length,
+      severity: "error",
+      message: `The handlebars code is not valid:\n${e.message}`,
+    })
+  }
+
+  return diagnostics
+}

packages/builder/src/components/common/CodeEditor/validator/tests/hbs.spec.ts

@@ -0,0 +1,142 @@
+import { validateHbsTemplate } from "../hbs"
+import { CodeValidator } from "@/types"
+
+describe("hbs validator", () => {
+  it("validate empty strings", () => {
+    const text = ""
+    const validators = {}
+
+    const result = validateHbsTemplate(text, validators)
+    expect(result).toHaveLength(0)
+  })
+
+  it("validate strings without hbs expressions", () => {
+    const text = "first line\nand another one"
+    const validators = {}
+
+    const result = validateHbsTemplate(text, validators)
+    expect(result).toHaveLength(0)
+  })
+
+  describe("basic expressions", () => {
+    const validators = {
+      fieldName: {},
+    }
+
+    it("validate valid expressions", () => {
+      const text = "{{ fieldName }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toHaveLength(0)
+    })
+
+    it("does not throw on missing validations", () => {
+      const text = "{{ anotherFieldName }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toHaveLength(0)
+    })
+
+    // Waiting for missing fields validation
+    it.skip("throws on untrimmed invalid expressions", () => {
+      const text = "    {{ anotherFieldName }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 4,
+          message: `"anotherFieldName" handler does not exist.`,
+          severity: "warning",
+          to: 26,
+        },
+      ])
+    })
+
+    // Waiting for missing fields validation
+    it.skip("throws on invalid expressions between valid lines", () => {
+      const text =
+        "literal expression\nthe value is {{ anotherFieldName }}\nanother expression"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 32,
+          message: `"anotherFieldName" handler does not exist.`,
+          severity: "warning",
+          to: 54,
+        },
+      ])
+    })
+
+    describe("expressions with whitespaces", () => {
+      const validators = {
+        [`field name`]: {},
+      }
+
+      it("validates expressions with whitespaces", () => {
+        const text = `{{ [field name] }}`
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toHaveLength(0)
+      })
+
+      // Waiting for missing fields validation
+      it.skip("throws if not wrapped between brackets", () => {
+        const text = `{{ field name }}`
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toEqual([
+          {
+            from: 0,
+            message: `"field" handler does not exist.`,
+            severity: "warning",
+            to: 16,
+          },
+        ])
+      })
+    })
+  })
+
+  describe("expressions with parameters", () => {
+    const validators: CodeValidator = {
+      helperFunction: {
+        arguments: ["a", "b", "c"],
+      },
+    }
+
+    it("validate valid params", () => {
+      const text = "{{ helperFunction 1 99 'a' }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toHaveLength(0)
+    })
+
+    it("throws on too few params", () => {
+      const text = "{{ helperFunction 100 }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 0,
+          message: `Helper "helperFunction" expects 3 parameters (a, b, c), but got 1.`,
+          severity: "error",
+          to: 24,
+        },
+      ])
+    })
+
+    it("throws on too many params", () => {
+      const text = "{{ helperFunction  1 99 'a' 100 }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 0,
+          message: `Helper "helperFunction" expects 3 parameters (a, b, c), but got 4.`,
+          severity: "error",
+          to: 34,
+        },
+      ])
+    })
+  })
+})

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -23,6 +23,7 @@
     snippetAutoComplete,
     EditorModes,
     bindingsToCompletions,
+    jsHelperAutocomplete,
   } from "../CodeEditor"
   import BindingSidePanel from "./BindingSidePanel.svelte"
   import EvaluationSidePanel from "./EvaluationSidePanel.svelte"
@@ -34,7 +35,6 @@
   import { BindingMode, SidePanel } from "@budibase/types"
   import type {
     EnrichedBinding,
-    BindingCompletion,
     Snippet,
     Helper,
     CaretPositionFn,
@@ -42,7 +42,7 @@
     JSONValue,
   } from "@budibase/types"
   import type { Log } from "@budibase/string-templates"
-  import type { CompletionContext } from "@codemirror/autocomplete"
+  import type { CodeValidator } from "@/types"
 
   const dispatch = createEventDispatcher()
 
@@ -58,7 +58,7 @@
   export let placeholder = null
   export let showTabBar = true
 
-  let mode: BindingMode | null
+  let mode: BindingMode
   let sidePanel: SidePanel | null
   let initialValueJS = value?.startsWith?.("{{ js ")
   let jsValue: string | null = initialValueJS ? value : null
@@ -88,10 +88,37 @@
     | null
   $: runtimeExpression = readableToRuntimeBinding(enrichedBindings, value)
   $: requestEval(runtimeExpression, context, snippets)
-  $: bindingCompletions = bindingsToCompletions(enrichedBindings, editorMode)
   $: bindingHelpers = new BindingHelpers(getCaretPosition, insertAtPos)
-  $: hbsCompletions = getHBSCompletions(bindingCompletions)
+
-  $: jsCompletions = getJSCompletions(bindingCompletions, snippets, useSnippets)
+  $: bindingOptions = bindingsToCompletions(bindings, editorMode)
+  $: helperOptions = allowHelpers ? getHelperCompletions(editorMode) : []
+  $: snippetsOptions =
+    usingJS && useSnippets && snippets?.length ? snippets : []
+
+  $: completions = !usingJS
+    ? [hbAutocomplete([...bindingOptions, ...helperOptions])]
+    : [
+        jsAutocomplete(bindingOptions),
+        jsHelperAutocomplete(helperOptions),
+        snippetAutoComplete(snippetsOptions),
+      ]
+
+  $: validations = {
+    ...bindingOptions.reduce<CodeValidator>((validations, option) => {
+      validations[option.label] = {
+        arguments: [],
+      }
+      return validations
+    }, {}),
+    ...helperOptions.reduce<CodeValidator>((validations, option) => {
+      validations[option.label] = {
+        arguments: option.args,
+        requiresBlock: option.requiresBlock,
+      }
+      return validations
+    }, {}),
+  }
+
   $: {
     // Ensure a valid side panel option is always selected
     if (sidePanel && !sidePanelOptions.includes(sidePanel)) {
@@ -99,32 +126,6 @@
     }
   }
 
-  const getHBSCompletions = (bindingCompletions: BindingCompletion[]) => {
-    return [
-      hbAutocomplete([
-        ...bindingCompletions,
-        ...getHelperCompletions(EditorModes.Handlebars),
-      ]),
-    ]
-  }
-
-  const getJSCompletions = (
-    bindingCompletions: BindingCompletion[],
-    snippets: Snippet[] | null,
-    useSnippets?: boolean
-  ) => {
-    const completions: ((_: CompletionContext) => any)[] = [
-      jsAutocomplete([
-        ...bindingCompletions,
-        ...getHelperCompletions(EditorModes.JS),
-      ]),
-    ]
-    if (useSnippets && snippets) {
-      completions.push(snippetAutoComplete(snippets))
-    }
-    return completions
-  }
-
   const getModeOptions = (allowHBS: boolean, allowJS: boolean) => {
     let options = []
     if (allowHBS) {
@@ -204,7 +205,7 @@
     bindings: EnrichedBinding[],
     context: any,
     snippets: Snippet[] | null
-  ) => {
+  ): EnrichedBinding[] => {
     // Create a single big array to enrich in one go
     const bindingStrings = bindings.map(binding => {
       if (binding.runtimeBinding.startsWith('trim "')) {
@@ -281,7 +282,7 @@
     jsValue = null
     hbsValue = null
     updateValue(null)
-    mode = targetMode
+    mode = targetMode!
     targetMode = null
   }
 
@@ -356,13 +357,14 @@
       {/if}
       <div class="editor">
         {#if mode === BindingMode.Text}
-          {#key hbsCompletions}
+          {#key completions}
             <CodeEditor
               value={hbsValue}
               on:change={onChangeHBSValue}
               bind:getCaretPosition
               bind:insertAtPos
-              completions={hbsCompletions}
+              {completions}
+              {validations}
               autofocus={autofocusEditor}
               placeholder={placeholder ||
                 "Add bindings by typing {{ or use the menu on the right"}
@@ -370,18 +372,18 @@
             />
           {/key}
         {:else if mode === BindingMode.JavaScript}
-          {#key jsCompletions}
+          {#key completions}
             <CodeEditor
               value={jsValue ? decodeJSBinding(jsValue) : jsValue}
               on:change={onChangeJSValue}
-              completions={jsCompletions}
+              {completions}
               mode={EditorModes.JS}
               bind:getCaretPosition
               bind:insertAtPos
               autofocus={autofocusEditor}
               placeholder={placeholder ||
                 "Add bindings by typing $ or use the menu on the right"}
-              jsBindingWrapping
+              jsBindingWrapping={completions.length > 0}
             />
           {/key}
         {/if}

packages/builder/src/components/common/bindings/SnippetDrawer.svelte

@@ -118,6 +118,7 @@
         allowHBS={false}
         allowJS
         allowSnippets={false}
+        allowHelpers={false}
         showTabBar={false}
         placeholder="return function(input) ❴ ... ❵"
         value={code}

packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte

@@ -43,7 +43,6 @@
     <EditComponentPopover
       {anchor}
       componentInstance={item}
-      {componentBindings}
       {bindings}
       on:change
       parseSettings={updatedNestedFlags}

packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte

@@ -1,13 +1,13 @@
 <script>
   import { Icon, Popover, Layout } from "@budibase/bbui"
-  import { componentStore } from "@/stores/builder"
+  import { componentStore, selectedScreen } from "@/stores/builder"
   import { cloneDeep } from "lodash/fp"
   import { createEventDispatcher, getContext } from "svelte"
   import ComponentSettingsSection from "@/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte"
+  import { getComponentBindableProperties } from "@/dataBinding"
 
   export let anchor
   export let componentInstance
-  export let componentBindings
   export let bindings
   export let parseSettings
 
@@ -28,6 +28,10 @@
   }
   $: componentDef = componentStore.getDefinition(componentInstance._component)
   $: parsedComponentDef = processComponentDefinitionSettings(componentDef)
+  $: componentBindings = getComponentBindableProperties(
+    $selectedScreen,
+    $componentStore.selectedComponentId
+  )
 
   const open = () => {
     isOpen = true

packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte

@@ -45,7 +45,6 @@
     <EditComponentPopover
       {anchor}
       componentInstance={item}
-      {componentBindings}
       {bindings}
       {parseSettings}
       on:change

packages/builder/src/components/design/settings/controls/PropertyControl.svelte

@@ -22,25 +22,59 @@
   export let propertyFocus = false
   export let info = null
   export let disableBindings = false
-  export let wide
+  export let wide = false
+  export let contextAccess = null
 
   let highlightType
   let domElement
 
   $: highlightedProp = $builderStore.highlightedSetting
-  $: allBindings = getAllBindings(bindings, componentBindings, nested)
+  $: allBindings = getAllBindings(
+    bindings,
+    componentBindings,
+    nested,
+    contextAccess
+  )
   $: safeValue = getSafeValue(value, defaultValue, allBindings)
   $: replaceBindings = val => readableToRuntimeBinding(allBindings, val)
-
   $: isHighlighted = highlightedProp?.key === key
-
   $: highlightType = isHighlighted ? `highlighted-${highlightedProp?.type}` : ""
+  $: highlightedProp && isHighlighted && scrollToElement(domElement)
 
-  const getAllBindings = (bindings, componentBindings, nested) => {
+  const getAllBindings = (
-    if (!nested) {
+    bindings,
+    componentBindings,
+    nested,
+    contextAccess
+  ) => {
+    // contextAccess is a bit of an escape hatch to get around how we render
+    // certain settings types by using a pseudo component definition, leading
+    // to problems with the nested flag
+    if (contextAccess != null) {
+      // Optionally include global bindings
+      let allBindings = contextAccess.global ? bindings : []
+
+      // Optionally include or exclude self (component) bindings.
+      // If this is a nested setting then we will already have our own context
+      // bindings mixed in, so if we don't want self context we need to filter
+      // them out.
+      if (contextAccess.self) {
+        return [...allBindings, ...componentBindings]
+      } else {
+        return allBindings.filter(binding => {
+          return !componentBindings.some(componentBinding => {
+            return componentBinding.runtimeBinding === binding.runtimeBinding
+          })
+        })
+      }
+    }
+
+    // Otherwise just honour the normal nested flag
+    if (nested) {
+      return [...bindings, ...componentBindings]
+    } else {
       return bindings
     }
-    return [...(componentBindings || []), ...(bindings || [])]
   }
 
   // Handle a value change of any type
@@ -81,8 +115,6 @@
       block: "center",
     })
   }
-
-  $: highlightedProp && isHighlighted && scrollToElement(domElement)
 </script>
 
 <div

packages/builder/src/components/design/settings/controls/URLVariableTestInput.svelte

@@ -0,0 +1,126 @@
+<script lang="ts">
+  import { onMount } from "svelte"
+  import { Input, Label } from "@budibase/bbui"
+  import { previewStore, selectedScreen } from "@/stores/builder"
+  import type { AppContext } from "@budibase/types"
+
+  export let baseRoute = ""
+
+  let testValue: string | undefined
+
+  $: routeParams = baseRoute.match(/:[a-zA-Z]+/g) || []
+  $: hasUrlParams = routeParams.length > 0
+  $: placeholder = getPlaceholder(baseRoute)
+  $: baseInput = createBaseInput(baseRoute)
+  $: updateTestValueFromContext($previewStore.selectedComponentContext)
+  $: if ($selectedScreen) {
+    testValue = ""
+  }
+
+  const getPlaceholder = (route: string) => {
+    const trimmed = route.replace(/\/$/, "")
+    if (trimmed.startsWith("/:")) {
+      return "1"
+    }
+    const segments = trimmed.split("/").slice(2)
+    let count = 1
+    return segments
+      .map(segment => (segment.startsWith(":") ? count++ : segment))
+      .join("/")
+  }
+
+  // This function is needed to repopulate the test value from componentContext
+  // when a user navigates to another component and then back again
+  const updateTestValueFromContext = (context: AppContext | null) => {
+    if (context?.url && !testValue) {
+      const { wild, ...urlParams } = context.url
+      const queryParams = context.query
+      if (Object.values(urlParams).some(v => Boolean(v))) {
+        let value = baseRoute
+          .split("/")
+          .slice(2)
+          .map(segment =>
+            segment.startsWith(":")
+              ? urlParams[segment.slice(1)] || ""
+              : segment
+          )
+          .join("/")
+        const qs = new URLSearchParams(queryParams).toString()
+        if (qs) {
+          value += `?${qs}`
+        }
+        testValue = value
+      }
+    }
+  }
+
+  const createBaseInput = (baseRoute: string) => {
+    return baseRoute === "/" || baseRoute.split("/")[1]?.startsWith(":")
+      ? "/"
+      : `/${baseRoute.split("/")[1]}/`
+  }
+
+  const onVariableChange = (e: CustomEvent) => {
+    previewStore.setUrlTestData({ route: baseRoute, testValue: e.detail })
+  }
+
+  onMount(() => {
+    previewStore.requestComponentContext()
+  })
+</script>
+
+{#if hasUrlParams}
+  <div class="url-test-section">
+    <div class="info">
+      <Label size="M">Set temporary URL variables for design preview</Label>
+    </div>
+    <div class="url-test-container">
+      <div class="base-input">
+        <Input disabled={true} value={baseInput} />
+      </div>
+      <div class="variable-input">
+        <Input value={testValue} on:change={onVariableChange} {placeholder} />
+      </div>
+    </div>
+  </div>
+{/if}
+
+<style>
+  .url-test-section {
+    width: 100%;
+    margin-top: var(--spacing-xl);
+  }
+
+  .info {
+    display: flex;
+    align-items: center;
+    gap: var(--spacing-s);
+    margin-bottom: var(--spacing-s);
+  }
+
+  .url-test-container {
+    display: flex;
+    width: 100%;
+  }
+
+  .base-input {
+    width: 98px;
+    margin-right: -1px;
+  }
+
+  .base-input :global(.spectrum-Textfield-input) {
+    border-top-right-radius: 0;
+    border-bottom-right-radius: 0;
+    background-color: var(--spectrum-global-color-gray-200);
+    color: var(--spectrum-global-color-gray-600);
+  }
+
+  .variable-input {
+    flex: 1;
+  }
+
+  .variable-input :global(.spectrum-Textfield-input) {
+    border-top-left-radius: 0;
+    border-bottom-left-radius: 0;
+  }
+</style>

packages/builder/src/helpers/components.ts

@@ -8,27 +8,32 @@ import {
 } from "@budibase/string-templates"
 import { capitalise } from "@/helpers"
 import { Constants } from "@budibase/frontend-core"
+import { Component, ComponentContext } from "@budibase/types"
 
 const { ContextScopes } = Constants
 
 /**
  * Recursively searches for a specific component ID
  */
-export const findComponent = (rootComponent, id) => {
+export const findComponent = (rootComponent: Component, id: string) => {
   return searchComponentTree(rootComponent, comp => comp._id === id)
 }
 
 /**
  * Recursively searches for a specific component type
  */
-export const findComponentType = (rootComponent, type) => {
+export const findComponentType = (rootComponent: Component, type: string) => {
   return searchComponentTree(rootComponent, comp => comp._component === type)
 }
 
 /**
  * Recursively searches for the parent component of a specific component ID
  */
-export const findComponentParent = (rootComponent, id, parentComponent) => {
+export const findComponentParent = (
+  rootComponent: Component | undefined,
+  id: string | undefined,
+  parentComponent: Component | null = null
+): Component | null => {
   if (!rootComponent || !id) {
     return null
   }
@@ -51,7 +56,11 @@ export const findComponentParent = (rootComponent, id, parentComponent) => {
  * Recursively searches for a specific component ID and records the component
  * path to this component
  */
-export const findComponentPath = (rootComponent, id, path = []) => {
+export const findComponentPath = (
+  rootComponent: Component,
+  id: string | undefined,
+  path: Component[] = []
+): Component[] => {
   if (!rootComponent || !id) {
     return []
   }
@@ -75,11 +84,14 @@ export const findComponentPath = (rootComponent, id, path = []) => {
  * Recurses through the component tree and finds all components which match
  * a certain selector
  */
-export const findAllMatchingComponents = (rootComponent, selector) => {
+export const findAllMatchingComponents = (
+  rootComponent: Component | null,
+  selector: (component: Component) => boolean
+) => {
   if (!rootComponent || !selector) {
     return []
   }
-  let components = []
+  let components: Component[] = []
   if (rootComponent._children) {
     rootComponent._children.forEach(child => {
       components = [
@@ -97,7 +109,7 @@ export const findAllMatchingComponents = (rootComponent, selector) => {
 /**
  * Recurses through the component tree and finds all components.
  */
-export const findAllComponents = rootComponent => {
+export const findAllComponents = (rootComponent: Component) => {
   return findAllMatchingComponents(rootComponent, () => true)
 }
 
@@ -105,9 +117,9 @@ export const findAllComponents = rootComponent => {
  * Finds the closest parent component which matches certain criteria
  */
 export const findClosestMatchingComponent = (
-  rootComponent,
+  rootComponent: Component,
-  componentId,
+  componentId: string | undefined,
-  selector
+  selector: (component: Component) => boolean
 ) => {
   if (!selector) {
     return null
@@ -125,7 +137,10 @@ export const findClosestMatchingComponent = (
  * Recurses through a component tree evaluating a matching function against
  * components until a match is found
  */
-const searchComponentTree = (rootComponent, matchComponent) => {
+const searchComponentTree = (
+  rootComponent: Component,
+  matchComponent: (component: Component) => boolean
+): Component | null => {
   if (!rootComponent || !matchComponent) {
     return null
   }
@@ -150,15 +165,18 @@ const searchComponentTree = (rootComponent, matchComponent) => {
  * This mutates the object in place.
  * @param component the component to randomise
  */
-export const makeComponentUnique = component => {
+export const makeComponentUnique = (component: Component) => {
   if (!component) {
     return
   }
 
   // Generate a full set of component ID replacements in this tree
-  const idReplacements = []
+  const idReplacements: [string, string][] = []
-  const generateIdReplacements = (component, replacements) => {
+  const generateIdReplacements = (
-    const oldId = component._id
+    component: Component,
+    replacements: [string, string][]
+  ) => {
+    const oldId = component._id!
     const newId = Helpers.uuid()
     replacements.push([oldId, newId])
     component._children?.forEach(x => generateIdReplacements(x, replacements))
@@ -182,9 +200,9 @@ export const makeComponentUnique = component => {
     let js = decodeJSBinding(sanitizedBinding)
     if (js != null) {
       // Replace ID inside JS binding
-      idReplacements.forEach(([oldId, newId]) => {
+      for (const [oldId, newId] of idReplacements) {
         js = js.replace(new RegExp(oldId, "g"), newId)
-      })
+      }
 
       // Create new valid JS binding
       let newBinding = encodeJSBinding(js)
@@ -204,7 +222,7 @@ export const makeComponentUnique = component => {
   return JSON.parse(definition)
 }
 
-export const getComponentText = component => {
+export const getComponentText = (component: Component) => {
   if (component == null) {
     return ""
   }
@@ -218,7 +236,7 @@ export const getComponentText = component => {
   return capitalise(type)
 }
 
-export const getComponentName = component => {
+export const getComponentName = (component: Component) => {
   if (component == null) {
     return ""
   }
@@ -229,9 +247,9 @@ export const getComponentName = component => {
 }
 
 // Gets all contexts exposed by a certain component type, including actions
-export const getComponentContexts = component => {
+export const getComponentContexts = (component: string) => {
   const def = componentStore.getDefinition(component)
-  let contexts = []
+  let contexts: ComponentContext[] = []
   if (def?.context) {
     contexts = Array.isArray(def.context) ? [...def.context] : [def.context]
   }
@@ -251,9 +269,9 @@ export const getComponentContexts = component => {
  * Recurses through the component tree and builds a tree of contexts provided
  * by components.
  */
-export const buildContextTree = (
+const buildContextTree = (
-  rootComponent,
+  rootComponent: Component,
-  tree = { root: [] },
+  tree: Record<string, string[]> = { root: [] },
   currentBranch = "root"
 ) => {
   // Sanity check
@@ -264,12 +282,12 @@ export const buildContextTree = (
   // Process this component's contexts
   const contexts = getComponentContexts(rootComponent._component)
   if (contexts.length) {
-    tree[currentBranch].push(rootComponent._id)
+    tree[currentBranch].push(rootComponent._id!)
 
     // If we provide local context, start a new branch for our children
     if (contexts.some(context => context.scope === ContextScopes.Local)) {
-      currentBranch = rootComponent._id
+      currentBranch = rootComponent._id!
-      tree[rootComponent._id] = []
+      tree[rootComponent._id!] = []
     }
   }
 
@@ -287,9 +305,9 @@ export const buildContextTree = (
  * Generates a lookup map of which context branch all components in a component
  * tree are inside.
  */
-export const buildContextTreeLookupMap = rootComponent => {
+export const buildContextTreeLookupMap = (rootComponent: Component) => {
   const tree = buildContextTree(rootComponent)
-  let map = {}
+  const map: Record<string, string> = {}
   Object.entries(tree).forEach(([branch, ids]) => {
     ids.forEach(id => {
       map[id] = branch
@@ -299,9 +317,9 @@ export const buildContextTreeLookupMap = rootComponent => {
 }
 
 // Get a flat list of ids for all descendants of a component
-export const getChildIdsForComponent = component => {
+export const getChildIdsForComponent = (component: Component): string[] => {
   return [
-    component._id,
+    component._id!,
     ...(component?._children ?? []).map(getChildIdsForComponent).flat(1),
   ]
 }

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte

@@ -147,6 +147,7 @@
           {componentInstance}
           {componentDefinition}
           {bindings}
+          {componentBindings}
         />
       {/if}
     </Panel>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte

@@ -151,6 +151,7 @@
               propertyFocus={$builderStore.propertyFocus === setting.key}
               info={setting.info}
               disableBindings={setting.disableBindings}
+              contextAccess={setting.contextAccess}
               props={{
                 // Generic settings
                 placeholder: setting.placeholder || null,

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUIDrawer.svelte

@@ -19,6 +19,7 @@
 
   export let conditions = []
   export let bindings = []
+  export let componentBindings = []
 
   const flipDurationMs = 150
   const actionOptions = [
@@ -55,6 +56,7 @@
   ]
 
   let dragDisabled = true
+
   $: settings = componentStore
     .getComponentSettings($selectedComponent?._component)
     ?.concat({
@@ -213,7 +215,10 @@
                       options: definition.options,
                       placeholder: definition.placeholder,
                     }}
+                    nested={definition.nested}
+                    contextAccess={definition.contextAccess}
                     {bindings}
+                    {componentBindings}
                   />
                 {:else}
                   <Select disabled placeholder=" " />

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUISection.svelte

@@ -64,7 +64,12 @@
     Show, hide and update components in response to conditions being met.
   </svelte:fragment>
   <Button cta slot="buttons" on:click={() => save()}>Save</Button>
-  <ConditionalUIDrawer slot="body" bind:conditions={tempValue} {bindings} />
+  <ConditionalUIDrawer
+    slot="body"
+    bind:conditions={tempValue}
+    {bindings}
+    {componentBindings}
+  />
 </Drawer>
 
 <style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Screen/GeneralPanel.svelte

@@ -15,6 +15,7 @@
   import ButtonActionEditor from "@/components/design/settings/controls/ButtonActionEditor/ButtonActionEditor.svelte"
   import { getBindableProperties } from "@/dataBinding"
   import BarButtonList from "@/components/design/settings/controls/BarButtonList.svelte"
+  import URLVariableTestInput from "@/components/design/settings/controls/URLVariableTestInput.svelte"
 
   $: bindings = getBindableProperties($selectedScreen, null)
   $: screenSettings = getScreenSettings($selectedScreen)
@@ -93,6 +94,13 @@
           ],
         },
       },
+      {
+        key: "urlTest",
+        control: URLVariableTestInput,
+        props: {
+          baseRoute: screen.routing?.route,
+        },
+      },
     ]
 
     return settings

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/NewComponentPanel.svelte

@@ -13,6 +13,11 @@
   import { fly } from "svelte/transition"
   import { findComponentPath } from "@/helpers/components"
 
+  // Smallest possible 1x1 transparent GIF
+  const ghost = new Image(1, 1)
+  ghost.src =
+    "data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw=="
+
   let searchString
   let searchRef
   let selectedIndex
@@ -217,7 +222,8 @@
     }
   })
 
-  const onDragStart = component => {
+  const onDragStart = (e, component) => {
+    e.dataTransfer.setDragImage(ghost, 0, 0)
     previewStore.startDrag(component)
   }
 
@@ -250,13 +256,12 @@
             {#each category.children as component}
               <div
                 draggable="true"
-                on:dragstart={() => onDragStart(component.component)}
+                on:dragstart={e => onDragStart(e, component.component)}
                 on:dragend={onDragEnd}
                 class="component"
                 class:selected={selectedIndex === orderMap[component.component]}
                 on:click={() => addComponent(component.component)}
-                on:mouseover={() => (selectedIndex = null)}
+                on:mouseenter={() => (selectedIndex = null)}
-                on:focus
               >
                 <Icon name={component.icon} />
                 <Body size="XS">{component.name}</Body>
@@ -308,7 +313,6 @@
   }
   .component:hover {
     background: var(--spectrum-global-color-gray-300);
-    cursor: pointer;
   }
   .component :global(.spectrum-Body) {
     line-height: 1.2 !important;

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte

@@ -189,8 +189,8 @@
     } else if (type === "reload-plugin") {
       await componentStore.refreshDefinitions()
     } else if (type === "drop-new-component") {
-      const { component, parent, index } = data
+      const { component, parent, index, props } = data
-      await componentStore.create(component, null, parent, index)
+      await componentStore.create(component, props, parent, index)
     } else if (type === "add-parent-component") {
       const { componentId, parentType } = data
       await componentStore.addParent(componentId, parentType)

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/dndStore.js

@@ -1,12 +1,8 @@
 import { writable, get } from "svelte/store"
 import { findComponentParent, findComponentPath } from "@/helpers/components"
 import { selectedScreen, componentStore } from "@/stores/builder"
-
+import { DropPosition } from "@budibase/types"
-export const DropPosition = {
+export { DropPosition } from "@budibase/types"
-  ABOVE: "above",
-  BELOW: "below",
-  INSIDE: "inside",
-}
 
 const initialState = {
   source: null,

packages/builder/src/stores/builder/app.ts

@@ -39,7 +39,7 @@ interface AppMetaState {
   appInstance: { _id: string } | null
   initialised: boolean
   hasAppPackage: boolean
-  usedPlugins: Plugin[] | null
+  usedPlugins: Plugin[]
   automations: AutomationSettings
   routes: { [key: string]: any }
   version?: string
@@ -76,7 +76,7 @@ export const INITIAL_APP_META_STATE: AppMetaState = {
   appInstance: null,
   initialised: false,
   hasAppPackage: false,
-  usedPlugins: null,
+  usedPlugins: [],
   automations: {},
   routes: {},
 }
@@ -109,7 +109,7 @@ export class AppMetaStore extends BudiStore<AppMetaState> {
       appInstance: app.instance,
       revertableVersion: app.revertableVersion,
       upgradableVersion: app.upgradableVersion,
-      usedPlugins: app.usedPlugins || null,
+      usedPlugins: app.usedPlugins || [],
       icon: app.icon,
       features: {
         ...INITIAL_APP_META_STATE.features,

packages/builder/src/stores/builder/automations.ts

@@ -15,7 +15,6 @@ import {
 import {
   AutomationTriggerStepId,
   AutomationEventType,
-  AutomationStepType,
   AutomationActionStepId,
   Automation,
   AutomationStep,
@@ -26,10 +25,14 @@ import {
   UILogicalOperator,
   EmptyFilterOption,
   AutomationIOType,
-  AutomationStepSchema,
-  AutomationTriggerSchema,
   BranchPath,
   BlockDefinitions,
+  isBranchStep,
+  isTrigger,
+  isRowUpdateTrigger,
+  isRowSaveTrigger,
+  isAppTrigger,
+  BranchStep,
 } from "@budibase/types"
 import { ActionStepID } from "@/constants/backend/automations"
 import { FIELDS } from "@/constants/backend"
@@ -291,16 +294,16 @@ const automationActions = (store: AutomationStore) => ({
     let result: (AutomationStep | AutomationTrigger)[] = []
     pathWay.forEach(path => {
       const { stepIdx, branchIdx } = path
-      let last = result.length ? result[result.length - 1] : []
       if (!result.length) {
         // Preceeding steps.
         result = steps.slice(0, stepIdx + 1)
         return
       }
-      if (last && "inputs" in last) {
+      let last = result[result.length - 1]
+      if (isBranchStep(last)) {
         if (Number.isInteger(branchIdx)) {
           const branchId = last.inputs.branches[branchIdx].id
-          const children = last.inputs.children[branchId]
+          const children = last.inputs.children?.[branchId] || []
           const stepChildren = children.slice(0, stepIdx + 1)
           // Preceeding steps.
           result = result.concat(stepChildren)
@@ -473,23 +476,28 @@ const automationActions = (store: AutomationStore) => ({
           id: block.id,
         },
       ]
-      const branches: Branch[] = block.inputs?.branches || []
+
+      if (isBranchStep(block)) {
+        const branches = block.inputs?.branches || []
+        const children = block.inputs?.children || {}
 
         branches.forEach((branch, bIdx) => {
-        block.inputs?.children[branch.id].forEach(
+          children[branch.id].forEach(
             (bBlock: AutomationStep, sIdx: number, array: AutomationStep[]) => {
-            const ended =
+              const ended = array.length - 1 === sIdx && !branches.length
-              array.length - 1 === sIdx && !bBlock.inputs?.branches?.length
               treeTraverse(bBlock, pathToCurrentNode, sIdx, bIdx, ended)
             }
           )
         })
 
+        terminating = terminating && !branches.length
+      }
+
       store.actions.registerBlock(
         blockRefs,
         block,
         pathToCurrentNode,
-        terminating && !branches.length
+        terminating
       )
     }
 
@@ -575,7 +583,6 @@ const automationActions = (store: AutomationStore) => ({
           pathBlock.stepId === ActionStepID.LOOP &&
           pathBlock.blockToLoop in blocks
       }
-      const isTrigger = pathBlock.type === AutomationStepType.TRIGGER
 
       if (isLoopBlock && loopBlockCount == 0) {
         schema = {
@@ -586,17 +593,14 @@ const automationActions = (store: AutomationStore) => ({
         }
       }
 
-      const icon = isTrigger
+      const icon = isTrigger(pathBlock)
         ? pathBlock.icon
         : isLoopBlock
         ? "Reuse"
         : pathBlock.icon
 
-      if (blockIdx === 0 && isTrigger) {
+      if (blockIdx === 0 && isTrigger(pathBlock)) {
-        if (
+        if (isRowUpdateTrigger(pathBlock) || isRowSaveTrigger(pathBlock)) {
-          pathBlock.event === AutomationEventType.ROW_UPDATE ||
-          pathBlock.event === AutomationEventType.ROW_SAVE
-        ) {
           let table: any = get(tables).list.find(
             (table: Table) => table._id === pathBlock.inputs.tableId
           )
@@ -608,7 +612,7 @@ const automationActions = (store: AutomationStore) => ({
             }
           }
           delete schema.row
-        } else if (pathBlock.event === AutomationEventType.APP_TRIGGER) {
+        } else if (isAppTrigger(pathBlock)) {
           schema = Object.fromEntries(
             Object.keys(pathBlock.inputs.fields || []).map(key => [
               key,
@@ -915,8 +919,10 @@ const automationActions = (store: AutomationStore) => ({
     ]
 
     let cache:
-      | AutomationStepSchema<AutomationActionStepId>
+      | AutomationStep
-      | AutomationTriggerSchema<AutomationTriggerStepId>
+      | AutomationTrigger
+      | AutomationStep[]
+      | undefined = undefined
 
     pathWay.forEach((path, pathIdx, array) => {
       const { stepIdx, branchIdx } = path
@@ -938,9 +944,13 @@ const automationActions = (store: AutomationStore) => ({
         }
         return
       }
-      if (Number.isInteger(branchIdx)) {
+      if (
+        Number.isInteger(branchIdx) &&
+        !Array.isArray(cache) &&
+        isBranchStep(cache)
+      ) {
         const branchId = cache.inputs.branches[branchIdx].id
-        const children = cache.inputs.children[branchId]
+        const children = cache.inputs.children?.[branchId] || []
 
         if (final) {
           insertBlock(children, stepIdx)
@@ -1090,7 +1100,7 @@ const automationActions = (store: AutomationStore) => ({
   branchLeft: async (
     pathTo: Array<any>,
     automation: Automation,
-    block: AutomationStep
+    block: BranchStep
   ) => {
     const update = store.actions.shiftBranch(pathTo, block)
     if (update) {
@@ -1113,7 +1123,7 @@ const automationActions = (store: AutomationStore) => ({
   branchRight: async (
     pathTo: Array<BranchPath>,
     automation: Automation,
-    block: AutomationStep
+    block: BranchStep
   ) => {
     const update = store.actions.shiftBranch(pathTo, block, 1)
     if (update) {
@@ -1133,7 +1143,7 @@ const automationActions = (store: AutomationStore) => ({
    * @param {Number} direction - the direction of the swap. Defaults to -1 for left, add 1 for right
    * @returns
    */
-  shiftBranch: (pathTo: Array<any>, block: AutomationStep, direction = -1) => {
+  shiftBranch: (pathTo: Array<any>, block: BranchStep, direction = -1) => {
     let newBlock = cloneDeep(block)
     const branchPath = pathTo.at(-1)
     const targetIdx = branchPath.branchIdx

packages/builder/src/stores/builder/componentTreeNodes.ts

@@ -58,7 +58,7 @@ export class ComponentTreeNodesStore extends BudiStore<OpenNodesState> {
 
     const path = findComponentPath(selectedScreen.props, componentId)
 
-    const componentIds = path.map((component: Component) => component._id)
+    const componentIds = path.map((component: Component) => component._id!)
 
     this.update((openNodes: OpenNodesState) => {
       const newNodes = Object.fromEntries(

packages/builder/src/stores/builder/components.ts

@@ -1,3 +1,5 @@
+// TODO: analise and fix all the undefined ! and ?
+
 import { get, derived } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
 import { API } from "@/api"
@@ -11,6 +13,7 @@ import {
   findComponentParent,
   findAllMatchingComponents,
   makeComponentUnique,
+  findComponentType,
 } from "@/helpers/components"
 import { getComponentFieldOptions } from "@/helpers/formFields"
 import { selectedScreen } from "./screens"
@@ -35,7 +38,7 @@ import { Utils } from "@budibase/frontend-core"
 import {
   ComponentDefinition,
   ComponentSetting,
-  Component as ComponentType,
+  Component,
   ComponentCondition,
   FieldType,
   Screen,
@@ -44,10 +47,6 @@ import {
 import { utils } from "@budibase/shared-core"
 import { getSequentialName } from "@/helpers/duplicate"
 
-interface Component extends ComponentType {
-  _id: string
-}
-
 export interface ComponentState {
   components: Record<string, ComponentDefinition>
   customComponents: string[]
@@ -139,10 +138,6 @@ export class ComponentStore extends BudiStore<ComponentState> {
 
   /**
    * Retrieve the component definition object
-   * @param {string} componentType
-   * @example
-   * '@budibase/standard-components/container'
-   * @returns {object}
    */
   getDefinition(componentType: string) {
     if (!componentType) {
@@ -151,10 +146,6 @@ export class ComponentStore extends BudiStore<ComponentState> {
     return get(this.store).components[componentType]
   }
 
-  /**
-   *
-   * @returns {object}
-   */
   getDefaultDatasource() {
     // Ignore users table
     const validTables = get(tables).list.filter(x => x._id !== "ta_users")
@@ -188,10 +179,8 @@ export class ComponentStore extends BudiStore<ComponentState> {
   /**
    * Takes an enriched component instance and applies any required migration
    * logic
-   * @param {object} enrichedComponent
-   * @returns {object} migrated Component
    */
-  migrateSettings(enrichedComponent: Component) {
+  migrateSettings(enrichedComponent: Component | null) {
     const componentPrefix = "@budibase/standard-components"
     let migrated = false
 
@@ -230,25 +219,18 @@ export class ComponentStore extends BudiStore<ComponentState> {
     for (let setting of filterableTypes || []) {
       const isLegacy = Array.isArray(enrichedComponent[setting.key])
       if (isLegacy) {
-        const processedSetting = utils.processSearchFilters(
+        enrichedComponent[setting.key] = utils.processSearchFilters(
           enrichedComponent[setting.key]
         )
-        enrichedComponent[setting.key] = processedSetting
         migrated = true
       }
     }
     return migrated
   }
 
-  /**
-   *
-   * @param {object} component
-   * @param {object} opts
-   * @returns
-   */
   enrichEmptySettings(
     component: Component,
-    opts: { screen?: Screen; parent?: Component; useDefaultValues?: boolean }
+    opts: { screen?: Screen; parent?: string; useDefaultValues?: boolean }
   ) {
     if (!component?._component) {
       return
@@ -256,7 +238,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
     const defaultDS = this.getDefaultDatasource()
     const settings = this.getComponentSettings(component._component)
     const { parent, screen, useDefaultValues } = opts || {}
-    const treeId = parent?._id || component._id
+    const treeId = parent || component._id
     if (!screen) {
       return
     }
@@ -280,14 +262,25 @@ export class ComponentStore extends BudiStore<ComponentState> {
             type: "table",
           }
         } else if (setting.type === "dataProvider") {
-          // Pick closest data provider where required
+          let providerId
+
+          // Pick closest parent data provider if one exists
           const path = findComponentPath(screen.props, treeId)
           const providers = path.filter((component: Component) =>
             component._component?.endsWith("/dataprovider")
           )
-          if (providers.length) {
+          providerId = providers[providers.length - 1]?._id
-            const id = providers[providers.length - 1]?._id
+
-            component[setting.key] = `{{ literal ${safe(id)} }}`
+          // If none in our direct path, select the first one the screen
+          if (!providerId) {
+            providerId = findComponentType(
+              screen.props,
+              "@budibase/standard-components/dataprovider"
+            )?._id
+          }
+
+          if (providerId) {
+            component[setting.key] = `{{ literal ${safe(providerId)} }}`
           }
         } else if (setting.type.startsWith("field/")) {
           // Autofill form field names
@@ -427,17 +420,10 @@ export class ComponentStore extends BudiStore<ComponentState> {
     }
   }
 
-  /**
-   *
-   * @param {string} componentName
-   * @param {object} presetProps
-   * @param {object} parent
-   * @returns
-   */
   createInstance(
     componentType: string,
-    presetProps: any,
+    presetProps?: Record<string, any>,
-    parent: any
+    parent?: string
   ): Component | null {
     const screen = get(selectedScreen)
     if (!screen) {
@@ -463,7 +449,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
       _id: Helpers.uuid(),
       _component: definition.component,
       _styles: {
-        normal: {},
+        normal: { ...presetProps?._styles?.normal },
         hover: {},
         active: {},
       },
@@ -512,19 +498,11 @@ export class ComponentStore extends BudiStore<ComponentState> {
     }
   }
 
-  /**
-   *
-   * @param {string} componentName
-   * @param {object} presetProps
-   * @param {object} parent
-   * @param {number} index
-   * @returns
-   */
   async create(
     componentType: string,
-    presetProps: any,
+    presetProps?: Record<string, any>,
-    parent: Component,
+    parent?: string,
-    index: number
+    index?: number
   ) {
     const state = get(this.store)
     const componentInstance = this.createInstance(
@@ -539,7 +517,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
     // Insert in position if specified
     if (parent && index != null) {
       await screenStore.patch((screen: Screen) => {
-        let parentComponent = findComponent(screen.props, parent)
+        let parentComponent = findComponent(screen.props, parent)!
         if (!parentComponent._children?.length) {
           parentComponent._children = [componentInstance]
         } else {
@@ -558,7 +536,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
         }
         const currentComponent = findComponent(
           screen.props,
-          selectedComponentId
+          selectedComponentId!
         )
         if (!currentComponent) {
           return false
@@ -601,7 +579,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
       return state
     })
 
-    componentTreeNodesStore.makeNodeVisible(componentInstance._id)
+    componentTreeNodesStore.makeNodeVisible(componentInstance._id!)
 
     // Log event
     analytics.captureEvent(Events.COMPONENT_CREATED, {
@@ -611,13 +589,6 @@ export class ComponentStore extends BudiStore<ComponentState> {
     return componentInstance
   }
 
-  /**
-   *
-   * @param {function} patchFn
-   * @param {string} componentId
-   * @param {string} screenId
-   * @returns
-   */
   async patch(
     patchFn: (component: Component, screen: Screen) => any,
     componentId?: string,
@@ -652,11 +623,6 @@ export class ComponentStore extends BudiStore<ComponentState> {
     await screenStore.patch(patchScreen, screenId)
   }
 
-  /**
-   *
-   * @param {object} component
-   * @returns
-   */
   async delete(component: Component) {
     if (!component) {
       return
@@ -665,7 +631,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
     // Determine the next component to select, and select it before deletion
     // to avoid an intermediate state of no component selection
     const state = get(this.store)
-    let nextId = ""
+    let nextId: string | null = ""
     if (state.selectedComponentId === component._id) {
       nextId = this.getNext()
       if (!nextId) {
@@ -678,7 +644,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
         nextId = nextId.replace("-navigation", "-screen")
       }
       this.update(state => {
-        state.selectedComponentId = nextId
+        state.selectedComponentId = nextId ?? undefined
         return state
       })
     }
@@ -686,18 +652,18 @@ export class ComponentStore extends BudiStore<ComponentState> {
     // Patch screen
     await screenStore.patch((screen: Screen) => {
       // Check component exists
-      component = findComponent(screen.props, component._id)
+      const updatedComponent = findComponent(screen.props, component._id!)
-      if (!component) {
+      if (!updatedComponent) {
         return false
       }
 
       // Check component has a valid parent
-      const parent = findComponentParent(screen.props, component._id)
+      const parent = findComponentParent(screen.props, updatedComponent._id)
       if (!parent) {
         return false
       }
-      parent._children = parent._children.filter(
+      parent._children = parent._children!.filter(
-        (child: Component) => child._id !== component._id
+        (child: Component) => child._id !== updatedComponent._id
       )
     }, null)
   }
@@ -737,13 +703,6 @@ export class ComponentStore extends BudiStore<ComponentState> {
     })
   }
 
-  /**
-   *
-   * @param {object} targetComponent
-   * @param {string} mode
-   * @param {object} targetScreen
-   * @returns
-   */
   async paste(
     targetComponent: Component,
     mode: string,
@@ -768,7 +727,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
     // Patch screen
     const patch = (screen: Screen) => {
       // Get up to date ref to target
-      targetComponent = findComponent(screen.props, targetComponent._id)
+      targetComponent = findComponent(screen.props, targetComponent!._id!)!
       if (!targetComponent) {
         return false
       }
@@ -782,7 +741,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
       if (!cut) {
         componentToPaste = makeComponentUnique(componentToPaste)
       }
-      newComponentId = componentToPaste._id
+      newComponentId = componentToPaste._id!
 
       // Strip grid position metadata if pasting into a new screen, but keep
       // alignment metadata
@@ -859,8 +818,8 @@ export class ComponentStore extends BudiStore<ComponentState> {
     if (!screen) {
       throw "A valid screen must be selected"
     }
-    const parent = findComponentParent(screen.props, componentId)
+    const parent = findComponentParent(screen.props, componentId)!
-    const index = parent?._children.findIndex(
+    const index = parent?._children?.findIndex(
       (x: Component) => x._id === componentId
     )
 
@@ -878,29 +837,29 @@ export class ComponentStore extends BudiStore<ComponentState> {
     }
 
     // If we have siblings above us, choose the sibling or a descendant
-    if (index > 0) {
+    if (index !== undefined && index > 0) {
       // If sibling before us accepts children, and is not collapsed, select a descendant
-      const previousSibling = parent._children[index - 1]
+      const previousSibling = parent._children![index - 1]
       if (
         previousSibling._children?.length &&
-        componentTreeNodesStore.isNodeExpanded(previousSibling._id)
+        componentTreeNodesStore.isNodeExpanded(previousSibling._id!)
       ) {
         let target = previousSibling
         while (
           target._children?.length &&
-          componentTreeNodesStore.isNodeExpanded(target._id)
+          componentTreeNodesStore.isNodeExpanded(target._id!)
         ) {
           target = target._children[target._children.length - 1]
         }
-        return target._id
+        return target._id!
       }
 
       // Otherwise just select sibling
-      return previousSibling._id
+      return previousSibling._id!
     }
 
     // If no siblings above us, select the parent
-    return parent._id
+    return parent._id!
   }
 
   getNext() {
@@ -912,9 +871,9 @@ export class ComponentStore extends BudiStore<ComponentState> {
       throw "A valid screen must be selected"
     }
     const parent = findComponentParent(screen.props, componentId)
-    const index = parent?._children.findIndex(
+    const index = parent?._children?.findIndex(
       (x: Component) => x._id === componentId
-    )
+    )!
 
     // Check for screen and navigation component edge cases
     const screenComponentId = `${screen._id}-screen`
@@ -927,37 +886,38 @@ export class ComponentStore extends BudiStore<ComponentState> {
     if (
       component?._children?.length &&
       (state.selectedComponentId === navComponentId ||
-        componentTreeNodesStore.isNodeExpanded(component._id))
+        componentTreeNodesStore.isNodeExpanded(component._id!))
     ) {
-      return component._children[0]._id
+      return component._children[0]._id!
     } else if (!parent) {
       return null
     }
 
     // Otherwise select the next sibling if we have one
-    if (index < parent._children.length - 1) {
+    if (index < parent._children!.length - 1) {
-      const nextSibling = parent._children[index + 1]
+      const nextSibling = parent._children![index + 1]
-      return nextSibling._id
+      return nextSibling._id!
     }
 
     // Last child, select our parents next sibling
     let target = parent
     let targetParent = findComponentParent(screen.props, target._id)
-    let targetIndex = targetParent?._children.findIndex(
+    let targetIndex = targetParent?._children?.findIndex(
       (child: Component) => child._id === target._id
-    )
+    )!
     while (
       targetParent != null &&
-      targetIndex === targetParent._children?.length - 1
+      targetParent._children &&
+      targetIndex === targetParent._children.length - 1
     ) {
       target = targetParent
       targetParent = findComponentParent(screen.props, target._id)
-      targetIndex = targetParent?._children.findIndex(
+      targetIndex = targetParent?._children!.findIndex(
         (child: Component) => child._id === target._id
-      )
+      )!
     }
     if (targetParent) {
-      return targetParent._children[targetIndex + 1]._id
+      return targetParent._children![targetIndex + 1]._id!
     } else {
       return null
     }
@@ -989,16 +949,16 @@ export class ComponentStore extends BudiStore<ComponentState> {
       const parent = findComponentParent(screen.props, componentId)
 
       // Check we aren't right at the top of the tree
-      const index = parent?._children.findIndex(
+      const index = parent?._children?.findIndex(
         (x: Component) => x._id === componentId
-      )
+      )!
       if (!parent || (index === 0 && parent._id === screen.props._id)) {
         return
       }
 
       // Copy original component and remove it from the parent
-      const originalComponent = cloneDeep(parent._children[index])
+      const originalComponent = cloneDeep(parent._children![index])
-      parent._children = parent._children.filter(
+      parent._children = parent._children!.filter(
         (component: Component) => component._id !== componentId
       )
 
@@ -1010,9 +970,9 @@ export class ComponentStore extends BudiStore<ComponentState> {
         const definition = this.getDefinition(previousSibling._component)
         if (
           definition?.hasChildren &&
-          componentTreeNodesStore.isNodeExpanded(previousSibling._id)
+          componentTreeNodesStore.isNodeExpanded(previousSibling._id!)
         ) {
-          previousSibling._children.push(originalComponent)
+          previousSibling._children!.push(originalComponent)
         }
 
         // Otherwise just move component above sibling
@@ -1024,11 +984,11 @@ export class ComponentStore extends BudiStore<ComponentState> {
       // If no siblings above us, go above the parent as long as it isn't
       // the screen
       else if (parent._id !== screen.props._id) {
-        const grandParent = findComponentParent(screen.props, parent._id)
+        const grandParent = findComponentParent(screen.props, parent._id)!
-        const parentIndex = grandParent._children.findIndex(
+        const parentIndex = grandParent._children!.findIndex(
           (child: Component) => child._id === parent._id
         )
-        grandParent._children.splice(parentIndex, 0, originalComponent)
+        grandParent._children!.splice(parentIndex, 0, originalComponent)
       }
     }, null)
   }
@@ -1067,9 +1027,9 @@ export class ComponentStore extends BudiStore<ComponentState> {
         const definition = this.getDefinition(nextSibling._component)
         if (
           definition?.hasChildren &&
-          componentTreeNodesStore.isNodeExpanded(nextSibling._id)
+          componentTreeNodesStore.isNodeExpanded(nextSibling._id!)
         ) {
-          nextSibling._children.splice(0, 0, originalComponent)
+          nextSibling._children!.splice(0, 0, originalComponent)
         }
 
         // Otherwise move below next sibling
@@ -1080,11 +1040,11 @@ export class ComponentStore extends BudiStore<ComponentState> {
 
       // Last child, so move below our parent
       else {
-        const grandParent = findComponentParent(screen.props, parent._id)
+        const grandParent = findComponentParent(screen.props, parent._id)!
-        const parentIndex = grandParent._children.findIndex(
+        const parentIndex = grandParent._children!.findIndex(
           (child: Component) => child._id === parent._id
         )
-        grandParent._children.splice(parentIndex + 1, 0, originalComponent)
+        grandParent._children!.splice(parentIndex + 1, 0, originalComponent)
       }
     }, null)
   }
@@ -1101,6 +1061,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
 
   async updateStyles(styles: Record<string, string>, id: string) {
     const patchFn = (component: Component) => {
+      delete component._placeholder
       component._styles.normal = {
         ...component._styles.normal,
         ...styles,
@@ -1231,7 +1192,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
     }
 
     // Create new parent instance
-    const newParentDefinition = this.createInstance(parentType, null, parent)
+    const newParentDefinition = this.createInstance(parentType)
     if (!newParentDefinition) {
       return
     }
@@ -1246,13 +1207,13 @@ export class ComponentStore extends BudiStore<ComponentState> {
       }
 
       // Replace component with parent
-      const index = oldParentDefinition._children.findIndex(
+      const index = oldParentDefinition._children!.findIndex(
         (component: Component) => component._id === componentId
       )
       if (index === -1) {
         return false
       }
-      oldParentDefinition._children[index] = {
+      oldParentDefinition._children![index] = {
         ...newParentDefinition,
         _children: [definition],
       }
@@ -1267,10 +1228,6 @@ export class ComponentStore extends BudiStore<ComponentState> {
 
   /**
    * Check if the components settings have been cached
-   * @param {string} componentType
-   * @example
-   * '@budibase/standard-components/container'
-   * @returns {boolean}
    */
   isCached(componentType: string) {
     const settings = get(this.store).settingsCache
@@ -1279,11 +1236,6 @@ export class ComponentStore extends BudiStore<ComponentState> {
 
   /**
    * Cache component settings
-   * @param {string} componentType
-   * @param {object} definition
-   * @example
-   * '@budibase/standard-components/container'
-   * @returns {array} the settings
    */
   cacheSettings(componentType: string, definition: ComponentDefinition | null) {
     let settings: ComponentSetting[] = []
@@ -1313,12 +1265,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
   /**
    * Retrieve an array of the component settings.
    * These settings are cached because they cannot change at run time.
-   *
    * Searches a component's definition for a setting matching a certain predicate.
-   * @param {string} componentType
-   * @example
-   * '@budibase/standard-components/container'
-   * @returns {Array<object>}
    */
   getComponentSettings(componentType: string) {
     if (!componentType) {

packages/builder/src/stores/builder/index.js

@@ -1,20 +1,20 @@
-import { layoutStore } from "./layouts.js"
+import { layoutStore } from "./layouts"
-import { appStore } from "./app.js"
+import { appStore } from "./app"
 import { componentStore, selectedComponent } from "./components"
-import { navigationStore } from "./navigation.js"
+import { navigationStore } from "./navigation"
-import { themeStore } from "./theme.js"
+import { themeStore } from "./theme"
 import { screenStore, selectedScreen, sortedScreens } from "./screens"
-import { builderStore } from "./builder.js"
+import { builderStore } from "./builder"
-import { hoverStore } from "./hover.js"
+import { hoverStore } from "./hover"
-import { previewStore } from "./preview.js"
+import { previewStore } from "./preview"
 import {
   automationStore,
   selectedAutomation,
   automationHistoryStore,
-} from "./automations.js"
+} from "./automations"
-import { userStore, userSelectedResourceMap, isOnlyUser } from "./users.js"
+import { userStore, userSelectedResourceMap, isOnlyUser } from "./users"
-import { deploymentStore } from "./deployments.js"
+import { deploymentStore } from "./deployments"
-import { contextMenuStore } from "./contextMenu.js"
+import { contextMenuStore } from "./contextMenu"
 import { snippets } from "./snippets"
 import {
   screenComponentsList,

packages/builder/src/stores/builder/navigation.ts

@@ -4,15 +4,13 @@ import { appStore } from "@/stores/builder"
 import { BudiStore } from "../BudiStore"
 import { AppNavigation, AppNavigationLink, UIObject } from "@budibase/types"
 
-interface BuilderNavigationStore extends AppNavigation {}
-
 export const INITIAL_NAVIGATION_STATE = {
   navigation: "Top",
   links: [],
   textAlign: "Left",
 }
 
-export class NavigationStore extends BudiStore<BuilderNavigationStore> {
+export class NavigationStore extends BudiStore<AppNavigation> {
   constructor() {
     super(INITIAL_NAVIGATION_STATE)
   }

packages/builder/src/stores/builder/preview.ts

@@ -1,15 +1,14 @@
 import { get } from "svelte/store"
 import { BudiStore } from "../BudiStore"
+import { PreviewDevice, ComponentContext, AppContext } from "@budibase/types"
 
-type PreviewDevice = "desktop" | "tablet" | "mobile"
 type PreviewEventHandler = (name: string, payload?: any) => void
-type ComponentContext = Record<string, any>
 
 interface PreviewState {
   previewDevice: PreviewDevice
   previewEventHandler: PreviewEventHandler | null
   showPreview: boolean
-  selectedComponentContext: ComponentContext | null
+  selectedComponentContext: AppContext | null
 }
 
 const INITIAL_PREVIEW_STATE: PreviewState = {
@@ -54,7 +53,7 @@ export class PreviewStore extends BudiStore<PreviewState> {
     }))
   }
 
-  startDrag(component: any) {
+  async startDrag(component: string) {
     this.sendEvent("dragging-new-component", {
       dragging: true,
       component,
@@ -86,6 +85,10 @@ export class PreviewStore extends BudiStore<PreviewState> {
     this.sendEvent("builder-state", data)
   }
 
+  setUrlTestData(data: Record<string, any>) {
+    this.sendEvent("builder-url-test-data", data)
+  }
+
   requestComponentContext() {
     this.sendEvent("request-context")
   }

packages/builder/src/stores/builder/screenComponent.ts

@@ -8,6 +8,7 @@ import {
   UIComponentError,
   ComponentDefinition,
   DependsOnComponentSetting,
+  Screen,
 } from "@budibase/types"
 import { queries } from "./queries"
 import { views } from "./views"
@@ -66,6 +67,7 @@ export const screenComponentErrorList = derived(
     if (!$selectedScreen) {
       return []
     }
+    const screen = $selectedScreen
 
     const datasources = {
       ...reduceBy("_id", $tables.list),
@@ -79,7 +81,9 @@ export const screenComponentErrorList = derived(
     const errors: UIComponentError[] = []
 
     function checkComponentErrors(component: Component, ancestors: string[]) {
-      errors.push(...getInvalidDatasources(component, datasources, definitions))
+      errors.push(
+        ...getInvalidDatasources(screen, component, datasources, definitions)
+      )
       errors.push(...getMissingRequiredSettings(component, definitions))
       errors.push(...getMissingAncestors(component, definitions, ancestors))
 
@@ -95,6 +99,7 @@ export const screenComponentErrorList = derived(
 )
 
 function getInvalidDatasources(
+  screen: Screen,
   component: Component,
   datasources: Record<string, any>,
   definitions: Record<string, ComponentDefinition>

packages/builder/src/stores/builder/screens.ts

@@ -490,7 +490,7 @@ export class ScreenStore extends BudiStore<ScreenState> {
     // Flatten the recursive component tree
     const components = findAllMatchingComponents(
       screen.props,
-      (x: Component) => x
+      (x: Component) => !!x
     )
 
     // Iterate over all components and run checks

packages/builder/src/types/bindings.ts

@@ -0,0 +1,19 @@
+import { CompletionContext, Completion } from "@codemirror/autocomplete"
+
+export type BindingCompletion = (context: CompletionContext) => {
+  from: number
+  options: Completion[]
+} | null
+
+export interface BindingCompletionOption extends Completion {
+  args?: any[]
+  requiresBlock?: boolean
+}
+
+export type CodeValidator = Record<
+  string,
+  {
+    arguments?: any[]
+    requiresBlock?: boolean
+  }
+>

packages/builder/src/types/index.ts

@@ -0,0 +1 @@
+export * from "./bindings"

packages/builder/tsconfig.json

@@ -14,5 +14,6 @@
       "assets/*": ["assets/*"],
       "@/*": ["src/*"]
     }
-  }
+  },
+  "exclude": []
 }

packages/cli/src/backups/objectStore.ts

@@ -3,6 +3,7 @@ import fs from "fs"
 import { join } from "path"
 import { TEMP_DIR, MINIO_DIR } from "./utils"
 import { progressBar } from "../utils"
+import * as stream from "node:stream"
 
 const {
   ObjectStoreBuckets,
@@ -20,15 +21,21 @@ export async function exportObjects() {
   let fullList: any[] = []
   let errorCount = 0
   for (let bucket of bucketList) {
-    const client = ObjectStore(bucket)
+    const client = ObjectStore()
     try {
-      await client.headBucket().promise()
+      await client.headBucket({
+        Bucket: bucket,
+      })
     } catch (err) {
       errorCount++
       continue
     }
-    const list = (await client.listObjectsV2().promise()) as { Contents: any[] }
+    const list = await client.listObjectsV2({
-    fullList = fullList.concat(list.Contents.map(el => ({ ...el, bucket })))
+      Bucket: bucket,
+    })
+    fullList = fullList.concat(
+      list.Contents?.map(el => ({ ...el, bucket })) || []
+    )
   }
   if (errorCount === bucketList.length) {
     throw new Error("Unable to access MinIO/S3 - check environment config.")
@@ -43,7 +50,13 @@ export async function exportObjects() {
       const dirs = possiblePath.slice(0, possiblePath.length - 1)
       fs.mkdirSync(join(path, object.bucket, ...dirs), { recursive: true })
     }
+    if (data instanceof stream.Readable) {
+      data.pipe(
+        fs.createWriteStream(join(path, object.bucket, ...possiblePath))
+      )
+    } else {
       fs.writeFileSync(join(path, object.bucket, ...possiblePath), data)
+    }
     bar.update(++count)
   }
   bar.stop()
@@ -60,7 +73,7 @@ export async function importObjects() {
   const bar = progressBar(total)
   let count = 0
   for (let bucket of buckets) {
-    const client = ObjectStore(bucket)
+    const client = ObjectStore()
     await createBucketIfNotExists(client, bucket)
     const files = await uploadDirectory(bucket, join(path, bucket), "/")
     count += files.length

packages/client/manifest.json

@@ -1455,7 +1455,8 @@
         "type": "icon",
         "label": "Icon",
         "key": "icon",
-        "required": true
+        "required": true,
+        "defaultValue": "ri-star-fill"
       },
       {
         "type": "select",
@@ -3088,7 +3089,21 @@
       {
         "type": "tableConditions",
         "label": "Conditions",
-        "key": "conditions"
+        "key": "conditions",
+        "contextAccess": {
+          "global": true,
+          "self": false
+        }
+      },
+      {
+        "type": "text",
+        "label": "Format",
+        "key": "format",
+        "info": "Changing format will display values as text",
+        "contextAccess": {
+          "global": false,
+          "self": true
+        }
       }
     ]
   },
@@ -7685,7 +7700,8 @@
           {
             "type": "columns/grid",
             "key": "columns",
-            "resetOn": "table"
+            "resetOn": "table",
+            "nested": true
           }
         ]
       },

packages/client/package.json

@@ -5,7 +5,7 @@
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
   "type": "module",
-  "svelte": "src/index.js",
+  "svelte": "src/index.ts",
   "exports": {
     ".": {
       "import": "./dist/budibase-client.js",

packages/client/src/api/api.ts

@@ -1,10 +1,6 @@
 import { createAPIClient } from "@budibase/frontend-core"
 import { authStore } from "../stores/auth"
-import {
+import { notificationStore, devToolsEnabled, devToolsStore } from "../stores"
-  notificationStore,
-  devToolsEnabled,
-  devToolsStore,
-} from "../stores/index"
 import { get } from "svelte/store"
 
 export const API = createAPIClient({

packages/client/src/components/Block.svelte

@@ -1,7 +1,7 @@
 <script>
   import { getContext, onDestroy, onMount, setContext } from "svelte"
-  import { builderStore } from "stores/builder.js"
+  import { builderStore } from "@/stores/builder"
-  import { blockStore } from "stores/blocks"
+  import { blockStore } from "@/stores/blocks"
 
   const component = getContext("component")
   const { styleable } = getContext("sdk")

packages/client/src/components/BlockComponent.svelte

@@ -1,8 +1,8 @@
 <script>
   import { getContext, onDestroy } from "svelte"
   import { generate } from "shortid"
-  import { builderStore } from "../stores/builder.js"
+  import { builderStore } from "../stores/builder"
-  import Component from "components/Component.svelte"
+  import Component from "@/components/Component.svelte"
 
   export let type
   export let props

packages/client/src/components/ClientApp.svelte

@@ -6,7 +6,7 @@
   import { Constants, CookieUtils } from "@budibase/frontend-core"
   import { getThemeClassNames } from "@budibase/shared-core"
   import Component from "./Component.svelte"
-  import SDK from "sdk"
+  import SDK from "@/sdk"
   import {
     featuresStore,
     createContextStore,
@@ -22,28 +22,30 @@
     environmentStore,
     sidePanelStore,
     modalStore,
-  } from "stores"
+  } from "@/stores"
-  import NotificationDisplay from "components/overlay/NotificationDisplay.svelte"
+  import NotificationDisplay from "./overlay/NotificationDisplay.svelte"
-  import ConfirmationDisplay from "components/overlay/ConfirmationDisplay.svelte"
+  import ConfirmationDisplay from "./overlay/ConfirmationDisplay.svelte"
-  import PeekScreenDisplay from "components/overlay/PeekScreenDisplay.svelte"
+  import PeekScreenDisplay from "./overlay/PeekScreenDisplay.svelte"
-  import UserBindingsProvider from "components/context/UserBindingsProvider.svelte"
+  import UserBindingsProvider from "./context/UserBindingsProvider.svelte"
-  import DeviceBindingsProvider from "components/context/DeviceBindingsProvider.svelte"
+  import DeviceBindingsProvider from "./context/DeviceBindingsProvider.svelte"
-  import StateBindingsProvider from "components/context/StateBindingsProvider.svelte"
+  import StateBindingsProvider from "./context/StateBindingsProvider.svelte"
-  import RowSelectionProvider from "components/context/RowSelectionProvider.svelte"
+  import TestUrlBindingsProvider from "./context/TestUrlBindingsProvider.svelte"
-  import QueryParamsProvider from "components/context/QueryParamsProvider.svelte"
+  import RowSelectionProvider from "./context/RowSelectionProvider.svelte"
-  import SettingsBar from "components/preview/SettingsBar.svelte"
+  import QueryParamsProvider from "./context/QueryParamsProvider.svelte"
-  import SelectionIndicator from "components/preview/SelectionIndicator.svelte"
+  import SettingsBar from "./preview/SettingsBar.svelte"
-  import HoverIndicator from "components/preview/HoverIndicator.svelte"
+  import SelectionIndicator from "./preview/SelectionIndicator.svelte"
+  import HoverIndicator from "./preview/HoverIndicator.svelte"
   import CustomThemeWrapper from "./CustomThemeWrapper.svelte"
-  import DNDHandler from "components/preview/DNDHandler.svelte"
+  import DNDHandler from "./preview/DNDHandler.svelte"
-  import GridDNDHandler from "components/preview/GridDNDHandler.svelte"
+  import GridDNDHandler from "./preview/GridDNDHandler.svelte"
-  import KeyboardManager from "components/preview/KeyboardManager.svelte"
+  import KeyboardManager from "./preview/KeyboardManager.svelte"
-  import DevToolsHeader from "components/devtools/DevToolsHeader.svelte"
+  import DevToolsHeader from "./devtools/DevToolsHeader.svelte"
-  import DevTools from "components/devtools/DevTools.svelte"
+  import DevTools from "./devtools/DevTools.svelte"
-  import FreeFooter from "components/FreeFooter.svelte"
+  import FreeFooter from "./FreeFooter.svelte"
-  import MaintenanceScreen from "components/MaintenanceScreen.svelte"
+  import MaintenanceScreen from "./MaintenanceScreen.svelte"
   import SnippetsProvider from "./context/SnippetsProvider.svelte"
   import EmbedProvider from "./context/EmbedProvider.svelte"
+  import DNDSelectionIndicators from "./preview/DNDSelectionIndicators.svelte"
 
   // Provide contexts
   setContext("sdk", SDK)
@@ -168,6 +170,7 @@
             <StateBindingsProvider>
               <RowSelectionProvider>
                 <QueryParamsProvider>
+                  <TestUrlBindingsProvider>
                     <SnippetsProvider>
                       <!-- Settings bar can be rendered outside of device preview -->
                       <!-- Key block needs to be outside the if statement or it breaks -->
@@ -266,9 +269,11 @@
                         {#if $builderStore.inBuilder}
                           <DNDHandler />
                           <GridDNDHandler />
+                          <DNDSelectionIndicators />
                         {/if}
                       </div>
                     </SnippetsProvider>
+                  </TestUrlBindingsProvider>
                 </QueryParamsProvider>
               </RowSelectionProvider>
             </StateBindingsProvider>

packages/client/src/components/Component.svelte

@@ -11,7 +11,7 @@
 <script>
   import { getContext, setContext, onMount } from "svelte"
   import { writable, get } from "svelte/store"
-  import { enrichProps, propsAreSame } from "utils/componentProps"
+  import { enrichProps, propsAreSame } from "@/utils/componentProps"
   import { getSettingsDefinition } from "@budibase/frontend-core"
   import {
     builderStore,
@@ -20,12 +20,15 @@
     appStore,
     dndComponentPath,
     dndIsDragging,
-  } from "stores"
+  } from "@/stores"
   import { Helpers } from "@budibase/bbui"
-  import { getActiveConditions, reduceConditionActions } from "utils/conditions"
+  import {
-  import EmptyPlaceholder from "components/app/EmptyPlaceholder.svelte"
+    getActiveConditions,
-  import ScreenPlaceholder from "components/app/ScreenPlaceholder.svelte"
+    reduceConditionActions,
-  import ComponentErrorState from "components/error-states/ComponentErrorState.svelte"
+  } from "@/utils/conditions"
+  import EmptyPlaceholder from "@/components/app/EmptyPlaceholder.svelte"
+  import ScreenPlaceholder from "@/components/app/ScreenPlaceholder.svelte"
+  import ComponentErrorState from "@/components/error-states/ComponentErrorState.svelte"
   import {
     decodeJSBinding,
     findHBSBlocks,
@@ -35,7 +38,7 @@
     getActionContextKey,
     getActionDependentContextKeys,
   } from "../utils/buttonActions.js"
-  import { gridLayout } from "utils/grid"
+  import { gridLayout } from "@/utils/grid"
 
   export let instance = {}
   export let parent = null
@@ -120,7 +123,7 @@
   $: children = instance._children || []
   $: id = instance._id
   $: name = isRoot ? "Screen" : instance._instanceName
-  $: icon = definition?.icon
+  $: icon = instance._icon || definition?.icon
 
   // Determine if the component is selected or is part of the critical path
   // leading to the selected component

packages/client/src/components/CustomThemeWrapper.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { themeStore } from "stores"
+  import { themeStore } from "@/stores"
   import { setContext } from "svelte"
   import { Context } from "@budibase/bbui"
 

packages/client/src/components/Router.svelte

@@ -1,7 +1,7 @@
 <script>
   import { setContext, getContext, onMount } from "svelte"
   import Router, { querystring } from "svelte-spa-router"
-  import { routeStore, stateStore } from "stores"
+  import { routeStore, stateStore } from "@/stores"
   import Screen from "./Screen.svelte"
   import { get } from "svelte/store"
 

packages/client/src/components/Screen.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { screenStore, routeStore, builderStore } from "stores"
+  import { screenStore, routeStore, builderStore } from "@/stores"
   import { get } from "svelte/store"
   import Component from "./Component.svelte"
   import Provider from "./context/Provider.svelte"

packages/client/src/components/app/GridBlock.svelte

@@ -3,9 +3,9 @@
   // because it functions similarly to one
   import { getContext, onMount } from "svelte"
   import { get, derived, readable } from "svelte/store"
-  import { featuresStore } from "stores"
+  import { featuresStore } from "@/stores"
   import { Grid } from "@budibase/frontend-core"
-  // import { processStringSync } from "@budibase/string-templates"
+  import { processStringSync } from "@budibase/string-templates"
 
   // table is actually any datasource, but called table for legacy compatibility
   export let table
@@ -47,8 +47,8 @@
   $: currentTheme = $context?.device?.theme
   $: darkMode = !currentTheme?.includes("light")
   $: parsedColumns = getParsedColumns(columns)
-  $: schemaOverrides = getSchemaOverrides(parsedColumns)
   $: enrichedButtons = enrichButtons(buttons)
+  $: schemaOverrides = getSchemaOverrides(parsedColumns, $context)
   $: selectedRows = deriveSelectedRows(gridContext)
   $: styles = patchStyles($component.styles, minHeight)
   $: data = { selectedRows: $selectedRows }
@@ -97,15 +97,19 @@
     }))
   }
 
-  const getSchemaOverrides = columns => {
+  const getSchemaOverrides = (columns, context) => {
     let overrides = {}
     columns.forEach((column, idx) => {
       overrides[column.field] = {
         displayName: column.label,
         order: idx,
-        conditions: column.conditions,
         visible: !!column.active,
-        // format: createFormatter(column),
+        conditions: enrichConditions(column.conditions, context),
+        format: createFormatter(column),
+
+        // Small hack to ensure we react to all changes, as our
+        // memoization cannot compare differences in functions
+        rand: column.conditions?.length ? Math.random() : null,
       }
       if (column.width) {
         overrides[column.field].width = column.width
@@ -114,12 +118,24 @@
     return overrides
   }
 
-  // const createFormatter = column => {
+  const enrichConditions = (conditions, context) => {
-  //   if (typeof column.format !== "string" || !column.format.trim().length) {
+    return conditions?.map(condition => {
-  //     return null
+      return {
-  //   }
+        ...condition,
-  //   return row => processStringSync(column.format, { [id]: row })
+        referenceValue: processStringSync(
-  // }
+          condition.referenceValue || "",
+          context
+        ),
+      }
+    })
+  }
+
+  const createFormatter = column => {
+    if (typeof column.format !== "string" || !column.format.trim().length) {
+      return null
+    }
+    return row => processStringSync(column.format, { [id]: row })
+  }
 
   const enrichButtons = buttons => {
     if (!buttons?.length) {

packages/client/src/components/app/Placeholder.svelte

@@ -5,7 +5,7 @@
   const component = getContext("component")
   const block = getContext("block")
 
-  export let text
+  export let text = undefined
 </script>
 
 {#if $builderStore.inBuilder}

packages/client/src/components/app/blocks/CardsBlock.svelte

@@ -1,9 +1,9 @@
 <script>
   import { getContext } from "svelte"
-  import Block from "components/Block.svelte"
+  import Block from "@/components/Block.svelte"
-  import BlockComponent from "components/BlockComponent.svelte"
+  import BlockComponent from "@/components/BlockComponent.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
-  import { enrichSearchColumns, enrichFilter } from "utils/blocks"
+  import { enrichSearchColumns, enrichFilter } from "@/utils/blocks"
   import { get } from "svelte/store"
 
   export let title

packages/client/src/components/app/blocks/ChartBlock.svelte

@@ -1,6 +1,6 @@
 <script>
-  import Block from "components/Block.svelte"
+  import Block from "@/components/Block.svelte"
-  import BlockComponent from "components/BlockComponent.svelte"
+  import BlockComponent from "@/components/BlockComponent.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
 
   // Datasource

packages/client/src/components/app/blocks/FormBlockComponent.svelte

@@ -1,5 +1,5 @@
 <script>
-  import BlockComponent from "components/BlockComponent.svelte"
+  import BlockComponent from "@/components/BlockComponent.svelte"
   import { FieldType } from "@budibase/types"
 
   export let field

packages/client/src/components/app/blocks/MultiStepFormblock.svelte

@@ -1,8 +1,8 @@
 <script>
-  import BlockComponent from "components/BlockComponent.svelte"
+  import BlockComponent from "@/components/BlockComponent.svelte"
   import { Helpers } from "@budibase/bbui"
   import { getContext, setContext } from "svelte"
-  import { builderStore } from "stores"
+  import { builderStore } from "@/stores"
   import { Utils } from "@budibase/frontend-core"
   import FormBlockWrapper from "./form/FormBlockWrapper.svelte"
   import { get, writable } from "svelte/store"

packages/client/src/components/app/blocks/RepeaterBlock.svelte

@@ -1,7 +1,7 @@
 <script>
-  import BlockComponent from "components/BlockComponent.svelte"
+  import BlockComponent from "@/components/BlockComponent.svelte"
-  import Block from "components/Block.svelte"
+  import Block from "@/components/Block.svelte"
-  import Placeholder from "components/app/Placeholder.svelte"
+  import Placeholder from "@/components/app/Placeholder.svelte"
   import { getContext } from "svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
   import { get } from "svelte/store"

packages/client/src/components/app/blocks/RowExplorer.svelte

@@ -1,6 +1,6 @@
 <script>
-  import Block from "components/Block.svelte"
+  import Block from "@/components/Block.svelte"
-  import BlockComponent from "components/BlockComponent.svelte"
+  import BlockComponent from "@/components/BlockComponent.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
   import { generate } from "shortid"
   import { get } from "svelte/store"

packages/client/src/components/app/blocks/form/FormBlockWrapper.svelte

@@ -1,6 +1,6 @@
 <script>
-  import BlockComponent from "components/BlockComponent.svelte"
+  import BlockComponent from "@/components/BlockComponent.svelte"
-  import Block from "components/Block.svelte"
+  import Block from "@/components/Block.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
   import { getContext } from "svelte"
 

packages/client/src/components/app/blocks/form/InnerFormBlock.svelte

@@ -1,6 +1,6 @@
 <script>
-  import BlockComponent from "components/BlockComponent.svelte"
+  import BlockComponent from "@/components/BlockComponent.svelte"
-  import Placeholder from "components/app/Placeholder.svelte"
+  import Placeholder from "@/components/app/Placeholder.svelte"
   import { getContext } from "svelte"
   import FormBlockComponent from "../FormBlockComponent.svelte"
 

packages/client/src/components/app/container/GridContainer.svelte

@@ -1,7 +1,7 @@
 <script>
   import { getContext, onMount } from "svelte"
   import { writable } from "svelte/store"
-  import { GridRowHeight, GridColumns } from "constants"
+  import { GridRowHeight, GridColumns } from "@/constants"
   import { memo } from "@budibase/frontend-core"
 
   export let onClick

packages/client/src/components/app/deprecated/TableBlock.svelte

@@ -2,10 +2,10 @@
   import { getContext } from "svelte"
   import { get } from "svelte/store"
   import { generate } from "shortid"
-  import Block from "components/Block.svelte"
+  import Block from "@/components/Block.svelte"
-  import BlockComponent from "components/BlockComponent.svelte"
+  import BlockComponent from "@/components/BlockComponent.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
-  import { enrichSearchColumns, enrichFilter } from "utils/blocks"
+  import { enrichSearchColumns, enrichFilter } from "@/utils/blocks"
   import { Utils } from "@budibase/frontend-core"
 
   export let title

packages/client/src/components/app/deprecated/table/Table.svelte

@@ -3,7 +3,7 @@
   import { Table } from "@budibase/bbui"
   import SlotRenderer from "./SlotRenderer.svelte"
   import { canBeSortColumn } from "@budibase/frontend-core"
-  import Provider from "components/context/Provider.svelte"
+  import Provider from "@/components/context/Provider.svelte"
 
   export let dataProvider
   export let columns

packages/client/src/components/app/forms/Field.svelte

@@ -1,4 +1,4 @@
-<script>
+<script lang="ts">
   import { getContext, onDestroy } from "svelte"
   import { writable } from "svelte/store"
   import { Icon } from "@budibase/bbui"
@@ -6,33 +6,33 @@
   import Placeholder from "../Placeholder.svelte"
   import InnerForm from "./InnerForm.svelte"
 
-  export let label
+  export let label: string | undefined = undefined
-  export let field
+  export let field: string | undefined = undefined
-  export let fieldState
+  export let fieldState: any
-  export let fieldApi
+  export let fieldApi: any
-  export let fieldSchema
+  export let fieldSchema: any
-  export let defaultValue
+  export let defaultValue: string | undefined = undefined
-  export let type
+  export let type: any
   export let disabled = false
   export let readonly = false
-  export let validation
+  export let validation: any
   export let span = 6
-  export let helpText = null
+  export let helpText: string | undefined = undefined
 
   // Get contexts
-  const formContext = getContext("form")
+  const formContext: any = getContext("form")
-  const formStepContext = getContext("form-step")
+  const formStepContext: any = getContext("form-step")
-  const fieldGroupContext = getContext("field-group")
+  const fieldGroupContext: any = getContext("field-group")
   const { styleable, builderStore, Provider } = getContext("sdk")
-  const component = getContext("component")
+  const component: any = getContext("component")
 
   // Register field with form
   const formApi = formContext?.formApi
   const labelPos = fieldGroupContext?.labelPosition || "above"
 
-  let formField
+  let formField: any
   let touched = false
-  let labelNode
+  let labelNode: any
 
   // Memoize values required to register the field to avoid loops
   const formStep = formStepContext || writable(1)
@@ -65,7 +65,7 @@
   $: $component.editing && labelNode?.focus()
 
   // Update form properties in parent component on every store change
-  $: unsubscribe = formField?.subscribe(value => {
+  $: unsubscribe = formField?.subscribe((value: any) => {
     fieldState = value?.fieldState
     fieldApi = value?.fieldApi
     fieldSchema = value?.fieldSchema
@@ -74,7 +74,7 @@
   // Determine label class from position
   $: labelClass = labelPos === "above" ? "" : `spectrum-FieldLabel--${labelPos}`
 
-  const registerField = info => {
+  const registerField = (info: any) => {
     formField = formApi?.registerField(
       info.field,
       info.type,
@@ -86,8 +86,9 @@
     )
   }
 
-  const updateLabel = e => {
+  const updateLabel = (e: any) => {
     if (touched) {
+      // @ts-expect-error and TODO updateProp isn't recognised - need builder TS conversion
       builderStore.actions.updateProp("label", e.target.textContent)
     }
     touched = false

packages/client/src/components/app/forms/InnerForm.svelte

@@ -4,13 +4,13 @@
   import { createValidatorFromConstraints } from "./validation"
   import { Helpers } from "@budibase/bbui"
 
-  export let dataSource
+  export let dataSource = undefined
   export let disabled = false
   export let readonly = false
-  export let initialValues
+  export let initialValues = undefined
-  export let size
+  export let size = undefined
-  export let schema
+  export let schema = undefined
-  export let definition
+  export let definition = undefined
   export let disableSchemaValidation = false
   export let editAutoColumns = false
 

packages/client/src/components/app/forms/RelationshipField.svelte

@@ -1,43 +1,61 @@
-<script>
+<script lang="ts">
   import { CoreSelect, CoreMultiselect } from "@budibase/bbui"
   import { FieldType } from "@budibase/types"
   import { fetchData, Utils } from "@budibase/frontend-core"
   import { getContext } from "svelte"
   import Field from "./Field.svelte"
+  import type {
+    SearchFilter,
+    RelationshipFieldMetadata,
+    Table,
+    Row,
+  } from "@budibase/types"
 
   const { API } = getContext("sdk")
 
-  export let field
+  export let field: string | undefined = undefined
-  export let label
+  export let label: string | undefined = undefined
-  export let placeholder
+  export let placeholder: any = undefined
-  export let disabled = false
+  export let disabled: boolean = false
-  export let readonly = false
+  export let readonly: boolean = false
-  export let validation
+  export let validation: any
-  export let autocomplete = true
+  export let autocomplete: boolean = true
-  export let defaultValue
+  export let defaultValue: string | undefined = undefined
-  export let onChange
+  export let onChange: any
-  export let filter
+  export let filter: SearchFilter[]
-  export let datasourceType = "table"
+  export let datasourceType: "table" | "user" | "groupUser" = "table"
-  export let primaryDisplay
+  export let primaryDisplay: string | undefined = undefined
-  export let span
+  export let span: number | undefined = undefined
-  export let helpText = null
+  export let helpText: string | undefined = undefined
-  export let type = FieldType.LINK
+  export let type:
+    | FieldType.LINK
+    | FieldType.BB_REFERENCE
+    | FieldType.BB_REFERENCE_SINGLE = FieldType.LINK
 
-  let fieldState
+  type RelationshipValue = { _id: string; [key: string]: any }
-  let fieldApi
+  type OptionObj = Record<string, RelationshipValue>
-  let fieldSchema
+  type OptionsObjType = Record<string, OptionObj>
-  let tableDefinition
-  let searchTerm
-  let open
 
+  let fieldState: any
+  let fieldApi: any
+  let fieldSchema: RelationshipFieldMetadata | undefined
+  let tableDefinition: Table | null | undefined
+  let searchTerm: any
+  let open: boolean
+  let selectedValue: string[] | string
+
+  // need a cast version of this for reactivity, components below aren't typed
+  $: castSelectedValue = selectedValue as any
   $: multiselect =
     [FieldType.LINK, FieldType.BB_REFERENCE].includes(type) &&
     fieldSchema?.relationshipType !== "one-to-many"
-  $: linkedTableId = fieldSchema?.tableId
+  $: linkedTableId = fieldSchema?.tableId!
   $: fetch = fetchData({
     API,
     datasource: {
-      type: datasourceType,
+      // typing here doesn't seem correct - we have the correct datasourceType options
+      // but when we configure the fetchData, it seems to think only "table" is valid
+      type: datasourceType as any,
       tableId: linkedTableId,
     },
     options: {
@@ -53,7 +71,8 @@
   $: component = multiselect ? CoreMultiselect : CoreSelect
   $: primaryDisplay = primaryDisplay || tableDefinition?.primaryDisplay
 
-  let optionsObj
+  let optionsObj: OptionsObjType = {}
+  const debouncedFetchRows = Utils.debounce(fetchRows, 250)
 
   $: {
     if (primaryDisplay && fieldState && !optionsObj) {
@@ -63,7 +82,11 @@
       if (!Array.isArray(valueAsSafeArray)) {
         valueAsSafeArray = [fieldState.value]
       }
-      optionsObj = valueAsSafeArray.reduce((accumulator, value) => {
+      optionsObj = valueAsSafeArray.reduce(
+        (
+          accumulator: OptionObj,
+          value: { _id: string; primaryDisplay: any }
+        ) => {
           // fieldState has to be an array of strings to be valid for an update
           // therefore we cannot guarantee value will be an object
           // https://linear.app/budibase/issue/BUDI-7577/refactor-the-relationshipfield-component-to-have-better-support-for
@@ -75,15 +98,17 @@
             [primaryDisplay]: value.primaryDisplay,
           }
           return accumulator
-      }, {})
+        },
+        {}
+      )
     }
   }
 
   $: enrichedOptions = enrichOptions(optionsObj, $fetch.rows)
-  const enrichOptions = (optionsObj, fetchResults) => {
+  const enrichOptions = (optionsObj: OptionsObjType, fetchResults: Row[]) => {
     const result = (fetchResults || [])?.reduce((accumulator, row) => {
-      if (!accumulator[row._id]) {
+      if (!accumulator[row._id!]) {
-        accumulator[row._id] = row
+        accumulator[row._id!] = row
       }
       return accumulator
     }, optionsObj || {})
@@ -92,24 +117,32 @@
   }
   $: {
     // We don't want to reorder while the dropdown is open, to avoid UX jumps
-    if (!open) {
+    if (!open && primaryDisplay) {
-      enrichedOptions = enrichedOptions.sort((a, b) => {
+      enrichedOptions = enrichedOptions.sort((a: OptionObj, b: OptionObj) => {
         const selectedValues = flatten(fieldState?.value) || []
 
-        const aIsSelected = selectedValues.find(v => v === a._id)
+        const aIsSelected = selectedValues.find(
-        const bIsSelected = selectedValues.find(v => v === b._id)
+          (v: RelationshipValue) => v === a._id
+        )
+        const bIsSelected = selectedValues.find(
+          (v: RelationshipValue) => v === b._id
+        )
         if (aIsSelected && !bIsSelected) {
           return -1
         } else if (!aIsSelected && bIsSelected) {
           return 1
         }
 
-        return a[primaryDisplay] > b[primaryDisplay]
+        return (a[primaryDisplay] > b[primaryDisplay]) as unknown as number
       })
     }
   }
 
-  $: forceFetchRows(filter)
+  $: {
+    if (filter || defaultValue) {
+      forceFetchRows()
+    }
+  }
   $: debouncedFetchRows(searchTerm, primaryDisplay, defaultValue)
 
   const forceFetchRows = async () => {
@@ -119,7 +152,11 @@
     selectedValue = []
     debouncedFetchRows(searchTerm, primaryDisplay, defaultValue)
   }
-  const fetchRows = async (searchTerm, primaryDisplay, defaultVal) => {
+  async function fetchRows(
+    searchTerm: any,
+    primaryDisplay: string,
+    defaultVal: string | string[]
+  ) {
     const allRowsFetched =
       $fetch.loaded &&
       !Object.keys($fetch.query?.string || {}).length &&
@@ -129,17 +166,39 @@
       return
     }
     // must be an array
-    if (defaultVal && !Array.isArray(defaultVal)) {
+    const defaultValArray: string[] = !defaultVal
-      defaultVal = defaultVal.split(",")
+      ? []
-    }
+      : !Array.isArray(defaultVal)
-    if (defaultVal && optionsObj && defaultVal.some(val => !optionsObj[val])) {
+      ? defaultVal.split(",")
+      : defaultVal
+
+    if (
+      defaultVal &&
+      optionsObj &&
+      defaultValArray.some(val => !optionsObj[val])
+    ) {
       await fetch.update({
-        query: { oneOf: { _id: defaultVal } },
+        query: { oneOf: { _id: defaultValArray } },
+      })
+    }
+
+    if (
+      (Array.isArray(selectedValue) &&
+        selectedValue.some(val => !optionsObj[val])) ||
+      (selectedValue && !optionsObj[selectedValue as string])
+    ) {
+      await fetch.update({
+        query: {
+          oneOf: {
+            _id: Array.isArray(selectedValue) ? selectedValue : [selectedValue],
+          },
+        },
       })
     }
 
     // Ensure we match all filters, rather than any
-    const baseFilter = (filter || []).filter(x => x.operator !== "allOr")
+    // @ts-expect-error this doesn't fit types, but don't want to change it yet
+    const baseFilter: any = (filter || []).filter(x => x.operator !== "allOr")
     await fetch.update({
       filter: [
         ...baseFilter,
@@ -152,9 +211,8 @@
       ],
     })
   }
-  const debouncedFetchRows = Utils.debounce(fetchRows, 250)
 
-  const flatten = values => {
+  const flatten = (values: any | any[]) => {
     if (!values) {
       return []
     }
@@ -162,17 +220,17 @@
     if (!Array.isArray(values)) {
       values = [values]
     }
-    values = values.map(value =>
+    values = values.map((value: any) =>
       typeof value === "object" ? value._id : value
     )
     return values
   }
 
-  const getDisplayName = row => {
+  const getDisplayName = (row: Row) => {
-    return row?.[primaryDisplay] || "-"
+    return row?.[primaryDisplay!] || "-"
   }
 
-  const handleChange = e => {
+  const handleChange = (e: any) => {
     let value = e.detail
     if (!multiselect) {
       value = value == null ? [] : [value]
@@ -220,13 +278,12 @@
       this={component}
       options={enrichedOptions}
       {autocomplete}
-      value={selectedValue}
+      value={castSelectedValue}
       on:change={handleChange}
       on:loadMore={loadMore}
       id={fieldState.fieldId}
       disabled={fieldState.disabled}
       readonly={fieldState.readonly}
-      error={fieldState.error}
       getOptionLabel={getDisplayName}
       getOptionValue={option => option._id}
       {placeholder}

packages/client/src/components/app/forms/S3Upload.svelte

@@ -2,7 +2,7 @@
   import Field from "./Field.svelte"
   import { CoreDropzone, ProgressCircle, Helpers } from "@budibase/bbui"
   import { getContext, onMount, onDestroy } from "svelte"
-  import { builderStore } from "stores/builder.js"
+  import { builderStore } from "@/stores/builder"
   import { processStringSync } from "@budibase/string-templates"
 
   export let datasourceId

packages/client/src/components/context/DeviceBindingsProvider.svelte

@@ -1,7 +1,7 @@
 <script>
   import Provider from "./Provider.svelte"
   import { onMount, onDestroy } from "svelte"
-  import { themeStore } from "stores"
+  import { themeStore } from "@/stores"
 
   let width = window.innerWidth
   let height = window.innerHeight

packages/client/src/components/context/Provider.svelte

@@ -1,7 +1,7 @@
 <script>
   import { getContext, setContext, onDestroy } from "svelte"
-  import { dataSourceStore, createContextStore } from "stores"
+  import { dataSourceStore, createContextStore } from "@/stores"
-  import { ActionTypes } from "constants"
+  import { ActionTypes } from "@/constants"
   import { generate } from "shortid"
 
   const { ContextScopes } = getContext("sdk")

packages/client/src/components/context/QueryParamsProvider.svelte

@@ -1,6 +1,6 @@
 <script>
   import Provider from "./Provider.svelte"
-  import { routeStore } from "stores"
+  import { routeStore } from "@/stores"
 </script>
 
 <Provider key="query" data={$routeStore.queryParams}>

packages/client/src/components/context/RowSelectionProvider.svelte

@@ -1,6 +1,6 @@
 <script>
   import Provider from "./Provider.svelte"
-  import { rowSelectionStore } from "stores"
+  import { rowSelectionStore } from "@/stores"
 </script>
 
 <Provider key="rowSelection" data={$rowSelectionStore}>

packages/client/src/components/context/SnippetsProvider.svelte

@@ -1,6 +1,6 @@
 <script>
   import Provider from "./Provider.svelte"
-  import { snippets } from "stores"
+  import { snippets } from "@/stores"
 </script>
 
 <Provider key="snippets" data={$snippets}>