Merge branch 'master' of https://github.com/budibase/budibase into BUDI-9011

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.4.7",
+  "version": "3.4.11",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -52,7 +52,11 @@ class InMemoryQueue implements Partial<Queue> {
   _opts?: QueueOptions
   _messages: JobMessage[]
   _queuedJobIds: Set<string>
-  _emitter: NodeJS.EventEmitter<{ message: [JobMessage]; completed: [Job] }>
+  _emitter: NodeJS.EventEmitter<{
+    message: [JobMessage]
+    completed: [Job]
+    removed: [JobMessage]
+  }>
   _runCount: number
   _addCount: number
 
@@ -83,6 +87,12 @@ class InMemoryQueue implements Partial<Queue> {
   async process(concurrencyOrFunc: number | any, func?: any) {
     func = typeof concurrencyOrFunc === "number" ? func : concurrencyOrFunc
     this._emitter.on("message", async message => {
+      // For the purpose of testing, don't trigger cron jobs immediately.
+      // Require the test to trigger them manually with timestamps.
+      if (message.opts?.repeat != null) {
+        return
+      }
+
       let resp = func(message)
 
       async function retryFunc(fnc: any) {
@@ -164,13 +174,14 @@ class InMemoryQueue implements Partial<Queue> {
    */
   async close() {}
 
-  /**
-   * This removes a cron which has been implemented, this is part of Bull API.
-   * @param cronJobId The cron which is to be removed.
-   */
-  async removeRepeatableByKey(cronJobId: string) {
-    // TODO: implement for testing
-    console.log(cronJobId)
+  async removeRepeatableByKey(id: string) {
+    for (const [idx, message] of this._messages.entries()) {
+      if (message.opts?.jobId?.toString() === id) {
+        this._messages.splice(idx, 1)
+        this._emitter.emit("removed", message)
+        return
+      }
+    }
   }
 
   async removeJobs(_pattern: string) {
@@ -214,7 +225,9 @@ class InMemoryQueue implements Partial<Queue> {
   }
 
   async getRepeatableJobs() {
-    return this._messages.map(job => jobToJobInformation(job as Job))
+    return this._messages
+      .filter(job => job.opts?.repeat != null)
+      .map(job => jobToJobInformation(job as Job))
   }
 }
 

packages/backend-core/src/utils/tests/utils.spec.ts

@@ -67,6 +67,15 @@ describe("utils", () => {
       })
     })
 
+    it("gets appId from query params", async () => {
+      const ctx = structures.koa.newContext()
+      const expected = db.generateAppID()
+      ctx.query = { appId: expected }
+
+      const actual = await utils.getAppIdFromCtx(ctx)
+      expect(actual).toBe(expected)
+    })
+
     it("doesn't get appId from url when previewing", async () => {
       const ctx = structures.koa.newContext()
       const appId = db.generateAppID()

packages/backend-core/src/utils/utils.ts

@@ -101,6 +101,11 @@ export async function getAppIdFromCtx(ctx: Ctx) {
     appId = confirmAppId(pathId)
   }
 
+  // look in queryParams
+  if (!appId && ctx.query?.appId) {
+    appId = confirmAppId(ctx.query?.appId as string)
+  }
+
   // lookup using custom url - prod apps only
   // filter out the builder preview path which collides with the prod app path
   // to ensure we don't load all apps excessively
@@ -247,3 +252,7 @@ export function hasCircularStructure(json: any) {
   }
   return false
 }
+
+export function urlHasProtocol(url: string): boolean {
+  return !!url.match(/^.+:\/\/.+$/)
+}

packages/bbui/src/Form/Core/Multiselect.svelte

@@ -14,7 +14,7 @@
   export let sort = false
   export let autoWidth = false
   export let searchTerm = null
-  export let customPopoverHeight
+  export let customPopoverHeight = undefined
   export let open = false
   export let loading
   export let onOptionMouseenter = () => {}

packages/bbui/src/Typography/Body.svelte

@@ -1,11 +1,11 @@
-<script>
+<script lang="ts">
   import "@spectrum-css/typography/dist/index-vars.css"
 
-  export let size = "M"
-  export let serif = false
-  export let weight = null
-  export let textAlign = null
-  export let color = null
+  export let size: "XS" | "S" | "M" | "L" | "XL" = "M"
+  export let serif: boolean = false
+  export let weight: string | null = null
+  export let textAlign: string | null = null
+  export let color: string | null = null
 </script>
 
 <p

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -386,7 +386,7 @@
       editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
     } else if (editableColumn.type === FieldType.FORMULA) {
       editableColumn.formulaType = "dynamic"
-      editableColumn.responseType = field.responseType || FIELDS.STRING.type
+      editableColumn.responseType = field?.responseType || FIELDS.STRING.type
     }
   }
 

packages/builder/src/components/common/CodeEditor/CodeEditor.svelte

@@ -40,16 +40,19 @@
     indentMore,
     indentLess,
   } from "@codemirror/commands"
+  import { setDiagnostics } from "@codemirror/lint"
   import { Compartment, EditorState } from "@codemirror/state"
+  import type { Extension } from "@codemirror/state"
   import { javascript } from "@codemirror/lang-javascript"
   import { EditorModes } from "./"
   import { themeStore } from "@/stores/portal"
   import type { EditorMode } from "@budibase/types"
-  import type { BindingCompletion } from "@/types"
+  import type { BindingCompletion, CodeValidator } from "@/types"
+  import { validateHbsTemplate } from "./validator/hbs"
 
   export let label: string | undefined = undefined
-  // TODO: work out what best type fits this
   export let completions: BindingCompletion[] = []
+  export let validations: CodeValidator | null = null
   export let mode: EditorMode = EditorModes.Handlebars
   export let value: string | null = ""
   export let placeholder: string | null = null
@@ -248,7 +251,7 @@
   // None of this is reactive, but it never has been, so we just assume most
   // config flags aren't changed at runtime
   // TODO: work out type for base
-  const buildExtensions = (base: any[]) => {
+  const buildExtensions = (base: Extension[]) => {
     let complete = [...base]
 
     if (autocompleteEnabled) {
@@ -340,6 +343,24 @@
     return complete
   }
 
+  function validate(
+    value: string | null,
+    editor: EditorView | undefined,
+    mode: EditorMode,
+    validations: CodeValidator | null
+  ) {
+    if (!value || !validations || !editor) {
+      return
+    }
+
+    if (mode === EditorModes.Handlebars) {
+      const diagnostics = validateHbsTemplate(value, validations)
+      editor.dispatch(setDiagnostics(editor.state, diagnostics))
+    }
+  }
+
+  $: validate(value, editor, mode, validations)
+
   const initEditor = () => {
     const baseExtensions = buildBaseExtensions()
 
@@ -366,7 +387,6 @@
     <Label size="S">{label}</Label>
   </div>
 {/if}
-
 <div class={`code-editor ${mode?.name || ""}`}>
   <div tabindex="-1" bind:this={textarea} />
 </div>

packages/builder/src/components/common/CodeEditor/index.ts

@@ -83,6 +83,8 @@ const helpersToCompletion = (
     const helper = helpers[helperName]
     return {
       label: helperName,
+      args: helper.args,
+      requiresBlock: helper.requiresBlock,
       info: () => buildHelperInfoNode(helper),
       type: "helper",
       section: helperSection,
@@ -136,9 +138,13 @@ export const hbAutocomplete = (
   baseCompletions: BindingCompletionOption[]
 ): BindingCompletion => {
   function coreCompletion(context: CompletionContext) {
-    let bindingStart = context.matchBefore(EditorModes.Handlebars.match)
+    if (!baseCompletions.length) {
+      return null
+    }
 
-    let options = baseCompletions || []
+    const bindingStart = context.matchBefore(EditorModes.Handlebars.match)
+
+    const options = baseCompletions
 
     if (!bindingStart) {
       return null
@@ -149,7 +155,7 @@ export const hbAutocomplete = (
       return null
     }
     const query = bindingStart.text.replace(match[0], "")
-    let filtered = bindingFilter(options, query)
+    const filtered = bindingFilter(options, query)
 
     return {
       from: bindingStart.from + match[0].length,
@@ -169,8 +175,12 @@ export const jsAutocomplete = (
   baseCompletions: BindingCompletionOption[]
 ): BindingCompletion => {
   function coreCompletion(context: CompletionContext) {
-    let jsBinding = wrappedAutocompleteMatch(context)
-    let options = baseCompletions || []
+    if (!baseCompletions.length) {
+      return null
+    }
+
+    const jsBinding = wrappedAutocompleteMatch(context)
+    const options = baseCompletions
 
     if (jsBinding) {
       // Accommodate spaces
@@ -209,6 +219,10 @@ function setAutocomplete(
   options: BindingCompletionOption[]
 ): BindingCompletion {
   return function (context: CompletionContext) {
+    if (!options.length) {
+      return null
+    }
+
     if (wrappedAutocompleteMatch(context)) {
       return null
     }

packages/builder/src/components/common/CodeEditor/validator/hbs.ts

@@ -0,0 +1,113 @@
+/* global hbs */
+import Handlebars from "handlebars"
+import type { Diagnostic } from "@codemirror/lint"
+import { CodeValidator } from "@/types"
+
+function isMustacheStatement(
+  node: hbs.AST.Statement
+): node is hbs.AST.MustacheStatement {
+  return node.type === "MustacheStatement"
+}
+
+function isBlockStatement(
+  node: hbs.AST.Statement
+): node is hbs.AST.BlockStatement {
+  return node.type === "BlockStatement"
+}
+
+function isPathExpression(
+  node: hbs.AST.Statement
+): node is hbs.AST.PathExpression {
+  return node.type === "PathExpression"
+}
+
+export function validateHbsTemplate(
+  text: string,
+  validations: CodeValidator
+): Diagnostic[] {
+  const diagnostics: Diagnostic[] = []
+
+  try {
+    const ast = Handlebars.parse(text, {})
+
+    const lineOffsets: number[] = []
+    let offset = 0
+    for (const line of text.split("\n")) {
+      lineOffsets.push(offset)
+      offset += line.length + 1 // +1 for newline character
+    }
+
+    function traverseNodes(
+      nodes: hbs.AST.Statement[],
+      options?: {
+        ignoreMissing?: boolean
+      }
+    ) {
+      const ignoreMissing = options?.ignoreMissing || false
+      nodes.forEach(node => {
+        if (isMustacheStatement(node) && isPathExpression(node.path)) {
+          const helperName = node.path.original
+
+          const from =
+            lineOffsets[node.loc.start.line - 1] + node.loc.start.column
+          const to = lineOffsets[node.loc.end.line - 1] + node.loc.end.column
+
+          if (!(helperName in validations)) {
+            if (!ignoreMissing) {
+              diagnostics.push({
+                from,
+                to,
+                severity: "warning",
+                message: `"${helperName}" handler does not exist.`,
+              })
+            }
+            return
+          }
+
+          const { arguments: expectedArguments = [], requiresBlock } =
+            validations[helperName]
+
+          if (requiresBlock && !isBlockStatement(node)) {
+            diagnostics.push({
+              from,
+              to,
+              severity: "error",
+              message: `Helper "${helperName}" requires a body:\n{{#${helperName} ...}} [body] {{/${helperName}}}`,
+            })
+            return
+          }
+
+          const providedParams = node.params
+
+          if (providedParams.length !== expectedArguments.length) {
+            diagnostics.push({
+              from,
+              to,
+              severity: "error",
+              message: `Helper "${helperName}" expects ${
+                expectedArguments.length
+              } parameters (${expectedArguments.join(", ")}), but got ${
+                providedParams.length
+              }.`,
+            })
+          }
+        }
+
+        if (isBlockStatement(node)) {
+          traverseNodes(node.program.body, { ignoreMissing: true })
+        }
+      })
+    }
+
+    traverseNodes(ast.body, { ignoreMissing: true })
+  } catch (e: any) {
+    diagnostics.push({
+      from: 0,
+      to: text.length,
+      severity: "error",
+      message: `The handlebars code is not valid:\n${e.message}`,
+    })
+  }
+
+  return diagnostics
+}

packages/builder/src/components/common/CodeEditor/validator/tests/hbs.spec.ts

@@ -0,0 +1,142 @@
+import { validateHbsTemplate } from "../hbs"
+import { CodeValidator } from "@/types"
+
+describe("hbs validator", () => {
+  it("validate empty strings", () => {
+    const text = ""
+    const validators = {}
+
+    const result = validateHbsTemplate(text, validators)
+    expect(result).toHaveLength(0)
+  })
+
+  it("validate strings without hbs expressions", () => {
+    const text = "first line\nand another one"
+    const validators = {}
+
+    const result = validateHbsTemplate(text, validators)
+    expect(result).toHaveLength(0)
+  })
+
+  describe("basic expressions", () => {
+    const validators = {
+      fieldName: {},
+    }
+
+    it("validate valid expressions", () => {
+      const text = "{{ fieldName }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toHaveLength(0)
+    })
+
+    it("does not throw on missing validations", () => {
+      const text = "{{ anotherFieldName }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toHaveLength(0)
+    })
+
+    // Waiting for missing fields validation
+    it.skip("throws on untrimmed invalid expressions", () => {
+      const text = "    {{ anotherFieldName }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 4,
+          message: `"anotherFieldName" handler does not exist.`,
+          severity: "warning",
+          to: 26,
+        },
+      ])
+    })
+
+    // Waiting for missing fields validation
+    it.skip("throws on invalid expressions between valid lines", () => {
+      const text =
+        "literal expression\nthe value is {{ anotherFieldName }}\nanother expression"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 32,
+          message: `"anotherFieldName" handler does not exist.`,
+          severity: "warning",
+          to: 54,
+        },
+      ])
+    })
+
+    describe("expressions with whitespaces", () => {
+      const validators = {
+        [`field name`]: {},
+      }
+
+      it("validates expressions with whitespaces", () => {
+        const text = `{{ [field name] }}`
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toHaveLength(0)
+      })
+
+      // Waiting for missing fields validation
+      it.skip("throws if not wrapped between brackets", () => {
+        const text = `{{ field name }}`
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toEqual([
+          {
+            from: 0,
+            message: `"field" handler does not exist.`,
+            severity: "warning",
+            to: 16,
+          },
+        ])
+      })
+    })
+  })
+
+  describe("expressions with parameters", () => {
+    const validators: CodeValidator = {
+      helperFunction: {
+        arguments: ["a", "b", "c"],
+      },
+    }
+
+    it("validate valid params", () => {
+      const text = "{{ helperFunction 1 99 'a' }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toHaveLength(0)
+    })
+
+    it("throws on too few params", () => {
+      const text = "{{ helperFunction 100 }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 0,
+          message: `Helper "helperFunction" expects 3 parameters (a, b, c), but got 1.`,
+          severity: "error",
+          to: 24,
+        },
+      ])
+    })
+
+    it("throws on too many params", () => {
+      const text = "{{ helperFunction  1 99 'a' 100 }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 0,
+          message: `Helper "helperFunction" expects 3 parameters (a, b, c), but got 4.`,
+          severity: "error",
+          to: 34,
+        },
+      ])
+    })
+  })
+})

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -42,7 +42,7 @@
     JSONValue,
   } from "@budibase/types"
   import type { Log } from "@budibase/string-templates"
-  import type { BindingCompletion, BindingCompletionOption } from "@/types"
+  import type { CodeValidator } from "@/types"
 
   const dispatch = createEventDispatcher()
 
@@ -58,7 +58,7 @@
   export let placeholder = null
   export let showTabBar = true
 
-  let mode: BindingMode | null
+  let mode: BindingMode
   let sidePanel: SidePanel | null
   let initialValueJS = value?.startsWith?.("{{ js ")
   let jsValue: string | null = initialValueJS ? value : null
@@ -88,13 +88,37 @@
     | null
   $: runtimeExpression = readableToRuntimeBinding(enrichedBindings, value)
   $: requestEval(runtimeExpression, context, snippets)
-  $: bindingCompletions = bindingsToCompletions(enrichedBindings, editorMode)
   $: bindingHelpers = new BindingHelpers(getCaretPosition, insertAtPos)
-  $: hbsCompletions = getHBSCompletions(bindingCompletions)
-  $: jsCompletions = getJSCompletions(bindingCompletions, snippets, {
-    useHelpers: allowHelpers,
-    useSnippets,
-  })
+
+  $: bindingOptions = bindingsToCompletions(bindings, editorMode)
+  $: helperOptions = allowHelpers ? getHelperCompletions(editorMode) : []
+  $: snippetsOptions =
+    usingJS && useSnippets && snippets?.length ? snippets : []
+
+  $: completions = !usingJS
+    ? [hbAutocomplete([...bindingOptions, ...helperOptions])]
+    : [
+        jsAutocomplete(bindingOptions),
+        jsHelperAutocomplete(helperOptions),
+        snippetAutoComplete(snippetsOptions),
+      ]
+
+  $: validations = {
+    ...bindingOptions.reduce<CodeValidator>((validations, option) => {
+      validations[option.label] = {
+        arguments: [],
+      }
+      return validations
+    }, {}),
+    ...helperOptions.reduce<CodeValidator>((validations, option) => {
+      validations[option.label] = {
+        arguments: option.args,
+        requiresBlock: option.requiresBlock,
+      }
+      return validations
+    }, {}),
+  }
+
   $: {
     // Ensure a valid side panel option is always selected
     if (sidePanel && !sidePanelOptions.includes(sidePanel)) {
@@ -102,38 +126,6 @@
     }
   }
 
-  const getHBSCompletions = (bindingCompletions: BindingCompletionOption[]) => {
-    return [
-      hbAutocomplete([
-        ...bindingCompletions,
-        ...getHelperCompletions(EditorModes.Handlebars),
-      ]),
-    ]
-  }
-
-  const getJSCompletions = (
-    bindingCompletions: BindingCompletionOption[],
-    snippets: Snippet[] | null,
-    config: {
-      useHelpers: boolean
-      useSnippets: boolean
-    }
-  ) => {
-    const completions: BindingCompletion[] = []
-    if (bindingCompletions.length) {
-      completions.push(jsAutocomplete([...bindingCompletions]))
-    }
-    if (config.useHelpers) {
-      completions.push(
-        jsHelperAutocomplete([...getHelperCompletions(EditorModes.JS)])
-      )
-    }
-    if (config.useSnippets && snippets) {
-      completions.push(snippetAutoComplete(snippets))
-    }
-    return completions
-  }
-
   const getModeOptions = (allowHBS: boolean, allowJS: boolean) => {
     let options = []
     if (allowHBS) {
@@ -213,7 +205,7 @@
     bindings: EnrichedBinding[],
     context: any,
     snippets: Snippet[] | null
-  ) => {
+  ): EnrichedBinding[] => {
     // Create a single big array to enrich in one go
     const bindingStrings = bindings.map(binding => {
       if (binding.runtimeBinding.startsWith('trim "')) {
@@ -290,7 +282,7 @@
     jsValue = null
     hbsValue = null
     updateValue(null)
-    mode = targetMode
+    mode = targetMode!
     targetMode = null
   }
 
@@ -365,13 +357,14 @@
       {/if}
       <div class="editor">
         {#if mode === BindingMode.Text}
-          {#key hbsCompletions}
+          {#key completions}
             <CodeEditor
               value={hbsValue}
               on:change={onChangeHBSValue}
               bind:getCaretPosition
               bind:insertAtPos
-              completions={hbsCompletions}
+              {completions}
+              {validations}
               autofocus={autofocusEditor}
               placeholder={placeholder ||
                 "Add bindings by typing {{ or use the menu on the right"}
@@ -379,18 +372,18 @@
             />
           {/key}
         {:else if mode === BindingMode.JavaScript}
-          {#key jsCompletions}
+          {#key completions}
             <CodeEditor
               value={jsValue ? decodeJSBinding(jsValue) : jsValue}
               on:change={onChangeJSValue}
-              completions={jsCompletions}
+              {completions}
               mode={EditorModes.JS}
               bind:getCaretPosition
               bind:insertAtPos
               autofocus={autofocusEditor}
               placeholder={placeholder ||
                 "Add bindings by typing $ or use the menu on the right"}
-              jsBindingWrapping={bindingCompletions.length > 0}
+              jsBindingWrapping={completions.length > 0}
             />
           {/key}
         {/if}

packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte

@@ -43,7 +43,6 @@
     <EditComponentPopover
       {anchor}
       componentInstance={item}
-      {componentBindings}
       {bindings}
       on:change
       parseSettings={updatedNestedFlags}

packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte

@@ -1,13 +1,13 @@
 <script>
   import { Icon, Popover, Layout } from "@budibase/bbui"
-  import { componentStore } from "@/stores/builder"
+  import { componentStore, selectedScreen } from "@/stores/builder"
   import { cloneDeep } from "lodash/fp"
   import { createEventDispatcher, getContext } from "svelte"
   import ComponentSettingsSection from "@/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte"
+  import { getComponentBindableProperties } from "@/dataBinding"
 
   export let anchor
   export let componentInstance
-  export let componentBindings
   export let bindings
   export let parseSettings
 
@@ -28,6 +28,10 @@
   }
   $: componentDef = componentStore.getDefinition(componentInstance._component)
   $: parsedComponentDef = processComponentDefinitionSettings(componentDef)
+  $: componentBindings = getComponentBindableProperties(
+    $selectedScreen,
+    $componentStore.selectedComponentId
+  )
 
   const open = () => {
     isOpen = true

packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte

@@ -45,7 +45,6 @@
     <EditComponentPopover
       {anchor}
       componentInstance={item}
-      {componentBindings}
       {bindings}
       {parseSettings}
       on:change

packages/builder/src/components/design/settings/controls/PropertyControl.svelte

@@ -22,25 +22,59 @@
   export let propertyFocus = false
   export let info = null
   export let disableBindings = false
-  export let wide
+  export let wide = false
+  export let contextAccess = null
 
   let highlightType
   let domElement
 
   $: highlightedProp = $builderStore.highlightedSetting
-  $: allBindings = getAllBindings(bindings, componentBindings, nested)
+  $: allBindings = getAllBindings(
+    bindings,
+    componentBindings,
+    nested,
+    contextAccess
+  )
   $: safeValue = getSafeValue(value, defaultValue, allBindings)
   $: replaceBindings = val => readableToRuntimeBinding(allBindings, val)
-
   $: isHighlighted = highlightedProp?.key === key
-
   $: highlightType = isHighlighted ? `highlighted-${highlightedProp?.type}` : ""
+  $: highlightedProp && isHighlighted && scrollToElement(domElement)
 
-  const getAllBindings = (bindings, componentBindings, nested) => {
-    if (!nested) {
+  const getAllBindings = (
+    bindings,
+    componentBindings,
+    nested,
+    contextAccess
+  ) => {
+    // contextAccess is a bit of an escape hatch to get around how we render
+    // certain settings types by using a pseudo component definition, leading
+    // to problems with the nested flag
+    if (contextAccess != null) {
+      // Optionally include global bindings
+      let allBindings = contextAccess.global ? bindings : []
+
+      // Optionally include or exclude self (component) bindings.
+      // If this is a nested setting then we will already have our own context
+      // bindings mixed in, so if we don't want self context we need to filter
+      // them out.
+      if (contextAccess.self) {
+        return [...allBindings, ...componentBindings]
+      } else {
+        return allBindings.filter(binding => {
+          return !componentBindings.some(componentBinding => {
+            return componentBinding.runtimeBinding === binding.runtimeBinding
+          })
+        })
+      }
+    }
+
+    // Otherwise just honour the normal nested flag
+    if (nested) {
+      return [...bindings, ...componentBindings]
+    } else {
       return bindings
     }
-    return [...(componentBindings || []), ...(bindings || [])]
   }
 
   // Handle a value change of any type
@@ -81,8 +115,6 @@
       block: "center",
     })
   }
-
-  $: highlightedProp && isHighlighted && scrollToElement(domElement)
 </script>
 
 <div

packages/builder/src/components/design/settings/controls/URLVariableTestInput.svelte

@@ -0,0 +1,126 @@
+<script lang="ts">
+  import { onMount } from "svelte"
+  import { Input, Label } from "@budibase/bbui"
+  import { previewStore, selectedScreen } from "@/stores/builder"
+  import type { ComponentContext } from "@budibase/types"
+
+  export let baseRoute = ""
+
+  let testValue: string | undefined
+
+  $: routeParams = baseRoute.match(/:[a-zA-Z]+/g) || []
+  $: hasUrlParams = routeParams.length > 0
+  $: placeholder = getPlaceholder(baseRoute)
+  $: baseInput = createBaseInput(baseRoute)
+  $: updateTestValueFromContext($previewStore.selectedComponentContext)
+  $: if ($selectedScreen) {
+    testValue = ""
+  }
+
+  const getPlaceholder = (route: string) => {
+    const trimmed = route.replace(/\/$/, "")
+    if (trimmed.startsWith("/:")) {
+      return "1"
+    }
+    const segments = trimmed.split("/").slice(2)
+    let count = 1
+    return segments
+      .map(segment => (segment.startsWith(":") ? count++ : segment))
+      .join("/")
+  }
+
+  // This function is needed to repopulate the test value from componentContext
+  // when a user navigates to another component and then back again
+  const updateTestValueFromContext = (context: ComponentContext | null) => {
+    if (context?.url && !testValue) {
+      const { wild, ...urlParams } = context.url
+      const queryParams = context.query
+      if (Object.values(urlParams).some(v => Boolean(v))) {
+        let value = baseRoute
+          .split("/")
+          .slice(2)
+          .map(segment =>
+            segment.startsWith(":")
+              ? urlParams[segment.slice(1)] || ""
+              : segment
+          )
+          .join("/")
+        const qs = new URLSearchParams(queryParams).toString()
+        if (qs) {
+          value += `?${qs}`
+        }
+        testValue = value
+      }
+    }
+  }
+
+  const createBaseInput = (baseRoute: string) => {
+    return baseRoute === "/" || baseRoute.split("/")[1]?.startsWith(":")
+      ? "/"
+      : `/${baseRoute.split("/")[1]}/`
+  }
+
+  const onVariableChange = (e: CustomEvent) => {
+    previewStore.setUrlTestData({ route: baseRoute, testValue: e.detail })
+  }
+
+  onMount(() => {
+    previewStore.requestComponentContext()
+  })
+</script>
+
+{#if hasUrlParams}
+  <div class="url-test-section">
+    <div class="info">
+      <Label size="M">Set temporary URL variables for design preview</Label>
+    </div>
+    <div class="url-test-container">
+      <div class="base-input">
+        <Input disabled={true} value={baseInput} />
+      </div>
+      <div class="variable-input">
+        <Input value={testValue} on:change={onVariableChange} {placeholder} />
+      </div>
+    </div>
+  </div>
+{/if}
+
+<style>
+  .url-test-section {
+    width: 100%;
+    margin-top: var(--spacing-xl);
+  }
+
+  .info {
+    display: flex;
+    align-items: center;
+    gap: var(--spacing-s);
+    margin-bottom: var(--spacing-s);
+  }
+
+  .url-test-container {
+    display: flex;
+    width: 100%;
+  }
+
+  .base-input {
+    width: 98px;
+    margin-right: -1px;
+  }
+
+  .base-input :global(.spectrum-Textfield-input) {
+    border-top-right-radius: 0;
+    border-bottom-right-radius: 0;
+    background-color: var(--spectrum-global-color-gray-200);
+    color: var(--spectrum-global-color-gray-600);
+  }
+
+  .variable-input {
+    flex: 1;
+  }
+
+  .variable-input :global(.spectrum-Textfield-input) {
+    border-top-left-radius: 0;
+    border-bottom-left-radius: 0;
+  }
+</style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte

@@ -147,6 +147,7 @@
           {componentInstance}
           {componentDefinition}
           {bindings}
+          {componentBindings}
         />
       {/if}
     </Panel>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte

@@ -151,6 +151,7 @@
               propertyFocus={$builderStore.propertyFocus === setting.key}
               info={setting.info}
               disableBindings={setting.disableBindings}
+              contextAccess={setting.contextAccess}
               props={{
                 // Generic settings
                 placeholder: setting.placeholder || null,

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUIDrawer.svelte

@@ -19,6 +19,7 @@
 
   export let conditions = []
   export let bindings = []
+  export let componentBindings = []
 
   const flipDurationMs = 150
   const actionOptions = [
@@ -55,6 +56,7 @@
   ]
 
   let dragDisabled = true
+
   $: settings = componentStore
     .getComponentSettings($selectedComponent?._component)
     ?.concat({
@@ -213,7 +215,10 @@
                       options: definition.options,
                       placeholder: definition.placeholder,
                     }}
+                    nested={definition.nested}
+                    contextAccess={definition.contextAccess}
                     {bindings}
+                    {componentBindings}
                   />
                 {:else}
                   <Select disabled placeholder=" " />

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUISection.svelte

@@ -64,7 +64,12 @@
     Show, hide and update components in response to conditions being met.
   </svelte:fragment>
   <Button cta slot="buttons" on:click={() => save()}>Save</Button>
-  <ConditionalUIDrawer slot="body" bind:conditions={tempValue} {bindings} />
+  <ConditionalUIDrawer
+    slot="body"
+    bind:conditions={tempValue}
+    {bindings}
+    {componentBindings}
+  />
 </Drawer>
 
 <style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Screen/GeneralPanel.svelte

@@ -15,6 +15,7 @@
   import ButtonActionEditor from "@/components/design/settings/controls/ButtonActionEditor/ButtonActionEditor.svelte"
   import { getBindableProperties } from "@/dataBinding"
   import BarButtonList from "@/components/design/settings/controls/BarButtonList.svelte"
+  import URLVariableTestInput from "@/components/design/settings/controls/URLVariableTestInput.svelte"
 
   $: bindings = getBindableProperties($selectedScreen, null)
   $: screenSettings = getScreenSettings($selectedScreen)
@@ -93,6 +94,13 @@
           ],
         },
       },
+      {
+        key: "urlTest",
+        control: URLVariableTestInput,
+        props: {
+          baseRoute: screen.routing?.route,
+        },
+      },
     ]
 
     return settings

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/dndStore.js

@@ -1,12 +1,8 @@
 import { writable, get } from "svelte/store"
 import { findComponentParent, findComponentPath } from "@/helpers/components"
 import { selectedScreen, componentStore } from "@/stores/builder"
-
-export const DropPosition = {
-  ABOVE: "above",
-  BELOW: "below",
-  INSIDE: "inside",
-}
+import { DropPosition } from "@budibase/types"
+export { DropPosition } from "@budibase/types"
 
 const initialState = {
   source: null,

packages/builder/src/stores/builder/index.js

@@ -1,20 +1,20 @@
-import { layoutStore } from "./layouts.js"
-import { appStore } from "./app.js"
+import { layoutStore } from "./layouts"
+import { appStore } from "./app"
 import { componentStore, selectedComponent } from "./components"
-import { navigationStore } from "./navigation.js"
-import { themeStore } from "./theme.js"
+import { navigationStore } from "./navigation"
+import { themeStore } from "./theme"
 import { screenStore, selectedScreen, sortedScreens } from "./screens"
-import { builderStore } from "./builder.js"
-import { hoverStore } from "./hover.js"
-import { previewStore } from "./preview.js"
+import { builderStore } from "./builder"
+import { hoverStore } from "./hover"
+import { previewStore } from "./preview"
 import {
   automationStore,
   selectedAutomation,
   automationHistoryStore,
-} from "./automations.js"
-import { userStore, userSelectedResourceMap, isOnlyUser } from "./users.js"
-import { deploymentStore } from "./deployments.js"
-import { contextMenuStore } from "./contextMenu.js"
+} from "./automations"
+import { userStore, userSelectedResourceMap, isOnlyUser } from "./users"
+import { deploymentStore } from "./deployments"
+import { contextMenuStore } from "./contextMenu"
 import { snippets } from "./snippets"
 import {
   screenComponentsList,

packages/builder/src/stores/builder/preview.ts

@@ -1,9 +1,8 @@
 import { get } from "svelte/store"
 import { BudiStore } from "../BudiStore"
+import { PreviewDevice, ComponentContext } from "@budibase/types"
 
-type PreviewDevice = "desktop" | "tablet" | "mobile"
 type PreviewEventHandler = (name: string, payload?: any) => void
-type ComponentContext = Record<string, any>
 
 interface PreviewState {
   previewDevice: PreviewDevice
@@ -86,6 +85,10 @@ export class PreviewStore extends BudiStore<PreviewState> {
     this.sendEvent("builder-state", data)
   }
 
+  setUrlTestData(data: Record<string, any>) {
+    this.sendEvent("builder-url-test-data", data)
+  }
+
   requestComponentContext() {
     this.sendEvent("request-context")
   }

packages/builder/src/types/bindings.ts

@@ -5,4 +5,15 @@ export type BindingCompletion = (context: CompletionContext) => {
   options: Completion[]
 } | null
 
-export type BindingCompletionOption = Completion
+export interface BindingCompletionOption extends Completion {
+  args?: any[]
+  requiresBlock?: boolean
+}
+
+export type CodeValidator = Record<
+  string,
+  {
+    arguments?: any[]
+    requiresBlock?: boolean
+  }
+>

packages/builder/tsconfig.json

@@ -14,5 +14,6 @@
       "assets/*": ["assets/*"],
       "@/*": ["src/*"]
     }
-  }
+  },
+  "exclude": []
 }

packages/client/manifest.json

@@ -3089,7 +3089,21 @@
       {
         "type": "tableConditions",
         "label": "Conditions",
-        "key": "conditions"
+        "key": "conditions",
+        "contextAccess": {
+          "global": true,
+          "self": false
+        }
+      },
+      {
+        "type": "text",
+        "label": "Format",
+        "key": "format",
+        "info": "Changing format will display values as text",
+        "contextAccess": {
+          "global": false,
+          "self": true
+        }
       }
     ]
   },
@@ -7686,7 +7700,8 @@
           {
             "type": "columns/grid",
             "key": "columns",
-            "resetOn": "table"
+            "resetOn": "table",
+            "nested": true
           }
         ]
       },

packages/client/src/components/Block.svelte

@@ -1,6 +1,6 @@
 <script>
   import { getContext, onDestroy, onMount, setContext } from "svelte"
-  import { builderStore } from "@/stores/builder.js"
+  import { builderStore } from "@/stores/builder"
   import { blockStore } from "@/stores/blocks"
 
   const component = getContext("component")

packages/client/src/components/BlockComponent.svelte

@@ -1,7 +1,7 @@
 <script>
   import { getContext, onDestroy } from "svelte"
   import { generate } from "shortid"
-  import { builderStore } from "../stores/builder.js"
+  import { builderStore } from "../stores/builder"
   import Component from "@/components/Component.svelte"
 
   export let type

packages/client/src/components/ClientApp.svelte

@@ -29,6 +29,7 @@
   import UserBindingsProvider from "./context/UserBindingsProvider.svelte"
   import DeviceBindingsProvider from "./context/DeviceBindingsProvider.svelte"
   import StateBindingsProvider from "./context/StateBindingsProvider.svelte"
+  import TestUrlBindingsProvider from "./context/TestUrlBindingsProvider.svelte"
   import RowSelectionProvider from "./context/RowSelectionProvider.svelte"
   import QueryParamsProvider from "./context/QueryParamsProvider.svelte"
   import SettingsBar from "./preview/SettingsBar.svelte"
@@ -169,108 +170,110 @@
             <StateBindingsProvider>
               <RowSelectionProvider>
                 <QueryParamsProvider>
-                  <SnippetsProvider>
-                    <!-- Settings bar can be rendered outside of device preview -->
-                    <!-- Key block needs to be outside the if statement or it breaks -->
-                    {#key $builderStore.selectedComponentId}
-                      {#if $builderStore.inBuilder}
-                        <SettingsBar />
-                      {/if}
-                    {/key}
-
-                    <!-- Clip boundary for selection indicators -->
-                    <div
-                      id="clip-root"
-                      class:preview={$builderStore.inBuilder}
-                      class:tablet-preview={$builderStore.previewDevice ===
-                        "tablet"}
-                      class:mobile-preview={$builderStore.previewDevice ===
-                        "mobile"}
-                    >
-                      <!-- Actual app -->
-                      <div id="app-root">
-                        {#if showDevTools}
-                          <DevToolsHeader />
+                  <TestUrlBindingsProvider>
+                    <SnippetsProvider>
+                      <!-- Settings bar can be rendered outside of device preview -->
+                      <!-- Key block needs to be outside the if statement or it breaks -->
+                      {#key $builderStore.selectedComponentId}
+                        {#if $builderStore.inBuilder}
+                          <SettingsBar />
                         {/if}
+                      {/key}
 
-                        <div id="app-body">
-                          {#if permissionError}
-                            <div class="error">
-                              <Layout justifyItems="center" gap="S">
-                                <!-- eslint-disable-next-line svelte/no-at-html-tags -->
-                                {@html ErrorSVG}
-                                <Heading size="L">
-                                  You don't have permission to use this app
-                                </Heading>
-                                <Body size="S">
-                                  Ask your administrator to grant you access
-                                </Body>
-                              </Layout>
-                            </div>
-                          {:else if !$screenStore.activeLayout}
-                            <div class="error">
-                              <Layout justifyItems="center" gap="S">
-                                <!-- eslint-disable-next-line svelte/no-at-html-tags -->
-                                {@html ErrorSVG}
-                                <Heading size="L">
-                                  Something went wrong rendering your app
-                                </Heading>
-                                <Body size="S">
-                                  Get in touch with support if this issue
-                                  persists
-                                </Body>
-                              </Layout>
-                            </div>
-                          {:else if embedNoScreens}
-                            <div class="error">
-                              <Layout justifyItems="center" gap="S">
-                                <!-- eslint-disable-next-line svelte/no-at-html-tags -->
-                                {@html ErrorSVG}
-                                <Heading size="L">
-                                  This Budibase app is not publicly accessible
-                                </Heading>
-                              </Layout>
-                            </div>
-                          {:else}
-                            <CustomThemeWrapper>
-                              {#key $screenStore.activeLayout._id}
-                                <Component
-                                  isLayout
-                                  instance={$screenStore.activeLayout.props}
-                                />
-                              {/key}
-
-                              <!-- Layers on top of app -->
-                              <NotificationDisplay />
-                              <ConfirmationDisplay />
-                              <PeekScreenDisplay />
-                            </CustomThemeWrapper>
+                      <!-- Clip boundary for selection indicators -->
+                      <div
+                        id="clip-root"
+                        class:preview={$builderStore.inBuilder}
+                        class:tablet-preview={$builderStore.previewDevice ===
+                          "tablet"}
+                        class:mobile-preview={$builderStore.previewDevice ===
+                          "mobile"}
+                      >
+                        <!-- Actual app -->
+                        <div id="app-root">
+                          {#if showDevTools}
+                            <DevToolsHeader />
                           {/if}
 
-                          {#if showDevTools}
-                            <DevTools />
+                          <div id="app-body">
+                            {#if permissionError}
+                              <div class="error">
+                                <Layout justifyItems="center" gap="S">
+                                  <!-- eslint-disable-next-line svelte/no-at-html-tags -->
+                                  {@html ErrorSVG}
+                                  <Heading size="L">
+                                    You don't have permission to use this app
+                                  </Heading>
+                                  <Body size="S">
+                                    Ask your administrator to grant you access
+                                  </Body>
+                                </Layout>
+                              </div>
+                            {:else if !$screenStore.activeLayout}
+                              <div class="error">
+                                <Layout justifyItems="center" gap="S">
+                                  <!-- eslint-disable-next-line svelte/no-at-html-tags -->
+                                  {@html ErrorSVG}
+                                  <Heading size="L">
+                                    Something went wrong rendering your app
+                                  </Heading>
+                                  <Body size="S">
+                                    Get in touch with support if this issue
+                                    persists
+                                  </Body>
+                                </Layout>
+                              </div>
+                            {:else if embedNoScreens}
+                              <div class="error">
+                                <Layout justifyItems="center" gap="S">
+                                  <!-- eslint-disable-next-line svelte/no-at-html-tags -->
+                                  {@html ErrorSVG}
+                                  <Heading size="L">
+                                    This Budibase app is not publicly accessible
+                                  </Heading>
+                                </Layout>
+                              </div>
+                            {:else}
+                              <CustomThemeWrapper>
+                                {#key $screenStore.activeLayout._id}
+                                  <Component
+                                    isLayout
+                                    instance={$screenStore.activeLayout.props}
+                                  />
+                                {/key}
+
+                                <!-- Layers on top of app -->
+                                <NotificationDisplay />
+                                <ConfirmationDisplay />
+                                <PeekScreenDisplay />
+                              </CustomThemeWrapper>
+                            {/if}
+
+                            {#if showDevTools}
+                              <DevTools />
+                            {/if}
+                          </div>
+
+                          {#if !$builderStore.inBuilder && $featuresStore.logoEnabled}
+                            <FreeFooter />
                           {/if}
                         </div>
 
-                        {#if !$builderStore.inBuilder && $featuresStore.logoEnabled}
-                          <FreeFooter />
+                        <!-- Preview and dev tools utilities  -->
+                        {#if $appStore.isDevApp}
+                          <SelectionIndicator />
+                        {/if}
+                        {#if $builderStore.inBuilder || $devToolsStore.allowSelection}
+                          <HoverIndicator />
+                        {/if}
+                        {#if $builderStore.inBuilder}
+                          <DNDHandler />
+                          <GridDNDHandler />
+                          <DNDSelectionIndicators />
                         {/if}
                       </div>
-
-                      <!-- Preview and dev tools utilities  -->
-                      {#if $appStore.isDevApp}
-                        <SelectionIndicator />
-                      {/if}
-                      {#if $builderStore.inBuilder || $devToolsStore.allowSelection}
-                        <HoverIndicator />
-                      {/if}
-                      {#if $builderStore.inBuilder}
-                        <DNDHandler />
-                        <GridDNDHandler />
-                        <DNDSelectionIndicators />
-                      {/if}
-                    </div>
-                  </SnippetsProvider>
+                    </SnippetsProvider>
+                  </TestUrlBindingsProvider>
                 </QueryParamsProvider>
               </RowSelectionProvider>
             </StateBindingsProvider>

packages/client/src/components/app/GridBlock.svelte

@@ -5,7 +5,7 @@
   import { get, derived, readable } from "svelte/store"
   import { featuresStore } from "@/stores"
   import { Grid } from "@budibase/frontend-core"
-  // import { processStringSync } from "@budibase/string-templates"
+  import { processStringSync } from "@budibase/string-templates"
 
   // table is actually any datasource, but called table for legacy compatibility
   export let table
@@ -47,8 +47,8 @@
   $: currentTheme = $context?.device?.theme
   $: darkMode = !currentTheme?.includes("light")
   $: parsedColumns = getParsedColumns(columns)
-  $: schemaOverrides = getSchemaOverrides(parsedColumns)
   $: enrichedButtons = enrichButtons(buttons)
+  $: schemaOverrides = getSchemaOverrides(parsedColumns, $context)
   $: selectedRows = deriveSelectedRows(gridContext)
   $: styles = patchStyles($component.styles, minHeight)
   $: data = { selectedRows: $selectedRows }
@@ -97,15 +97,19 @@
     }))
   }
 
-  const getSchemaOverrides = columns => {
+  const getSchemaOverrides = (columns, context) => {
     let overrides = {}
     columns.forEach((column, idx) => {
       overrides[column.field] = {
         displayName: column.label,
         order: idx,
-        conditions: column.conditions,
         visible: !!column.active,
-        // format: createFormatter(column),
+        conditions: enrichConditions(column.conditions, context),
+        format: createFormatter(column),
+
+        // Small hack to ensure we react to all changes, as our
+        // memoization cannot compare differences in functions
+        rand: column.conditions?.length ? Math.random() : null,
       }
       if (column.width) {
         overrides[column.field].width = column.width
@@ -114,12 +118,24 @@
     return overrides
   }
 
-  // const createFormatter = column => {
-  //   if (typeof column.format !== "string" || !column.format.trim().length) {
-  //     return null
-  //   }
-  //   return row => processStringSync(column.format, { [id]: row })
-  // }
+  const enrichConditions = (conditions, context) => {
+    return conditions?.map(condition => {
+      return {
+        ...condition,
+        referenceValue: processStringSync(
+          condition.referenceValue || "",
+          context
+        ),
+      }
+    })
+  }
+
+  const createFormatter = column => {
+    if (typeof column.format !== "string" || !column.format.trim().length) {
+      return null
+    }
+    return row => processStringSync(column.format, { [id]: row })
+  }
 
   const enrichButtons = buttons => {
     if (!buttons?.length) {

packages/client/src/components/app/Placeholder.svelte

@@ -5,7 +5,7 @@
   const component = getContext("component")
   const block = getContext("block")
 
-  export let text
+  export let text = undefined
 </script>
 
 {#if $builderStore.inBuilder}

packages/client/src/components/app/forms/Field.svelte

@@ -1,4 +1,4 @@
-<script>
+<script lang="ts">
   import { getContext, onDestroy } from "svelte"
   import { writable } from "svelte/store"
   import { Icon } from "@budibase/bbui"
@@ -6,33 +6,33 @@
   import Placeholder from "../Placeholder.svelte"
   import InnerForm from "./InnerForm.svelte"
 
-  export let label
-  export let field
-  export let fieldState
-  export let fieldApi
-  export let fieldSchema
-  export let defaultValue
-  export let type
+  export let label: string | undefined = undefined
+  export let field: string | undefined = undefined
+  export let fieldState: any
+  export let fieldApi: any
+  export let fieldSchema: any
+  export let defaultValue: string | undefined = undefined
+  export let type: any
   export let disabled = false
   export let readonly = false
-  export let validation
+  export let validation: any
   export let span = 6
-  export let helpText = null
+  export let helpText: string | undefined = undefined
 
   // Get contexts
-  const formContext = getContext("form")
-  const formStepContext = getContext("form-step")
-  const fieldGroupContext = getContext("field-group")
+  const formContext: any = getContext("form")
+  const formStepContext: any = getContext("form-step")
+  const fieldGroupContext: any = getContext("field-group")
   const { styleable, builderStore, Provider } = getContext("sdk")
-  const component = getContext("component")
+  const component: any = getContext("component")
 
   // Register field with form
   const formApi = formContext?.formApi
   const labelPos = fieldGroupContext?.labelPosition || "above"
 
-  let formField
+  let formField: any
   let touched = false
-  let labelNode
+  let labelNode: any
 
   // Memoize values required to register the field to avoid loops
   const formStep = formStepContext || writable(1)
@@ -65,7 +65,7 @@
   $: $component.editing && labelNode?.focus()
 
   // Update form properties in parent component on every store change
-  $: unsubscribe = formField?.subscribe(value => {
+  $: unsubscribe = formField?.subscribe((value: any) => {
     fieldState = value?.fieldState
     fieldApi = value?.fieldApi
     fieldSchema = value?.fieldSchema
@@ -74,7 +74,7 @@
   // Determine label class from position
   $: labelClass = labelPos === "above" ? "" : `spectrum-FieldLabel--${labelPos}`
 
-  const registerField = info => {
+  const registerField = (info: any) => {
     formField = formApi?.registerField(
       info.field,
       info.type,
@@ -86,8 +86,9 @@
     )
   }
 
-  const updateLabel = e => {
+  const updateLabel = (e: any) => {
     if (touched) {
+      // @ts-expect-error and TODO updateProp isn't recognised - need builder TS conversion
       builderStore.actions.updateProp("label", e.target.textContent)
     }
     touched = false

packages/client/src/components/app/forms/InnerForm.svelte

@@ -4,13 +4,13 @@
   import { createValidatorFromConstraints } from "./validation"
   import { Helpers } from "@budibase/bbui"
 
-  export let dataSource
+  export let dataSource = undefined
   export let disabled = false
   export let readonly = false
-  export let initialValues
-  export let size
-  export let schema
-  export let definition
+  export let initialValues = undefined
+  export let size = undefined
+  export let schema = undefined
+  export let definition = undefined
   export let disableSchemaValidation = false
   export let editAutoColumns = false
 

packages/client/src/components/app/forms/RelationshipField.svelte

@@ -1,43 +1,61 @@
-<script>
+<script lang="ts">
   import { CoreSelect, CoreMultiselect } from "@budibase/bbui"
   import { FieldType } from "@budibase/types"
   import { fetchData, Utils } from "@budibase/frontend-core"
   import { getContext } from "svelte"
   import Field from "./Field.svelte"
+  import type {
+    SearchFilter,
+    RelationshipFieldMetadata,
+    Table,
+    Row,
+  } from "@budibase/types"
 
   const { API } = getContext("sdk")
 
-  export let field
-  export let label
-  export let placeholder
-  export let disabled = false
-  export let readonly = false
-  export let validation
-  export let autocomplete = true
-  export let defaultValue
-  export let onChange
-  export let filter
-  export let datasourceType = "table"
-  export let primaryDisplay
-  export let span
-  export let helpText = null
-  export let type = FieldType.LINK
+  export let field: string | undefined = undefined
+  export let label: string | undefined = undefined
+  export let placeholder: any = undefined
+  export let disabled: boolean = false
+  export let readonly: boolean = false
+  export let validation: any
+  export let autocomplete: boolean = true
+  export let defaultValue: string | undefined = undefined
+  export let onChange: any
+  export let filter: SearchFilter[]
+  export let datasourceType: "table" | "user" | "groupUser" = "table"
+  export let primaryDisplay: string | undefined = undefined
+  export let span: number | undefined = undefined
+  export let helpText: string | undefined = undefined
+  export let type:
+    | FieldType.LINK
+    | FieldType.BB_REFERENCE
+    | FieldType.BB_REFERENCE_SINGLE = FieldType.LINK
 
-  let fieldState
-  let fieldApi
-  let fieldSchema
-  let tableDefinition
-  let searchTerm
-  let open
+  type RelationshipValue = { _id: string; [key: string]: any }
+  type OptionObj = Record<string, RelationshipValue>
+  type OptionsObjType = Record<string, OptionObj>
 
+  let fieldState: any
+  let fieldApi: any
+  let fieldSchema: RelationshipFieldMetadata | undefined
+  let tableDefinition: Table | null | undefined
+  let searchTerm: any
+  let open: boolean
+  let selectedValue: string[] | string
+
+  // need a cast version of this for reactivity, components below aren't typed
+  $: castSelectedValue = selectedValue as any
   $: multiselect =
     [FieldType.LINK, FieldType.BB_REFERENCE].includes(type) &&
     fieldSchema?.relationshipType !== "one-to-many"
-  $: linkedTableId = fieldSchema?.tableId
+  $: linkedTableId = fieldSchema?.tableId!
   $: fetch = fetchData({
     API,
     datasource: {
-      type: datasourceType,
+      // typing here doesn't seem correct - we have the correct datasourceType options
+      // but when we configure the fetchData, it seems to think only "table" is valid
+      type: datasourceType as any,
       tableId: linkedTableId,
     },
     options: {
@@ -53,7 +71,8 @@
   $: component = multiselect ? CoreMultiselect : CoreSelect
   $: primaryDisplay = primaryDisplay || tableDefinition?.primaryDisplay
 
-  let optionsObj
+  let optionsObj: OptionsObjType = {}
+  const debouncedFetchRows = Utils.debounce(fetchRows, 250)
 
   $: {
     if (primaryDisplay && fieldState && !optionsObj) {
@@ -63,27 +82,33 @@
       if (!Array.isArray(valueAsSafeArray)) {
         valueAsSafeArray = [fieldState.value]
       }
-      optionsObj = valueAsSafeArray.reduce((accumulator, value) => {
-        // fieldState has to be an array of strings to be valid for an update
-        // therefore we cannot guarantee value will be an object
-        // https://linear.app/budibase/issue/BUDI-7577/refactor-the-relationshipfield-component-to-have-better-support-for
-        if (!value._id) {
+      optionsObj = valueAsSafeArray.reduce(
+        (
+          accumulator: OptionObj,
+          value: { _id: string; primaryDisplay: any }
+        ) => {
+          // fieldState has to be an array of strings to be valid for an update
+          // therefore we cannot guarantee value will be an object
+          // https://linear.app/budibase/issue/BUDI-7577/refactor-the-relationshipfield-component-to-have-better-support-for
+          if (!value._id) {
+            return accumulator
+          }
+          accumulator[value._id] = {
+            _id: value._id,
+            [primaryDisplay]: value.primaryDisplay,
+          }
           return accumulator
-        }
-        accumulator[value._id] = {
-          _id: value._id,
-          [primaryDisplay]: value.primaryDisplay,
-        }
-        return accumulator
-      }, {})
+        },
+        {}
+      )
     }
   }
 
   $: enrichedOptions = enrichOptions(optionsObj, $fetch.rows)
-  const enrichOptions = (optionsObj, fetchResults) => {
+  const enrichOptions = (optionsObj: OptionsObjType, fetchResults: Row[]) => {
     const result = (fetchResults || [])?.reduce((accumulator, row) => {
-      if (!accumulator[row._id]) {
-        accumulator[row._id] = row
+      if (!accumulator[row._id!]) {
+        accumulator[row._id!] = row
       }
       return accumulator
     }, optionsObj || {})
@@ -92,24 +117,32 @@
   }
   $: {
     // We don't want to reorder while the dropdown is open, to avoid UX jumps
-    if (!open) {
-      enrichedOptions = enrichedOptions.sort((a, b) => {
+    if (!open && primaryDisplay) {
+      enrichedOptions = enrichedOptions.sort((a: OptionObj, b: OptionObj) => {
         const selectedValues = flatten(fieldState?.value) || []
 
-        const aIsSelected = selectedValues.find(v => v === a._id)
-        const bIsSelected = selectedValues.find(v => v === b._id)
+        const aIsSelected = selectedValues.find(
+          (v: RelationshipValue) => v === a._id
+        )
+        const bIsSelected = selectedValues.find(
+          (v: RelationshipValue) => v === b._id
+        )
         if (aIsSelected && !bIsSelected) {
           return -1
         } else if (!aIsSelected && bIsSelected) {
           return 1
         }
 
-        return a[primaryDisplay] > b[primaryDisplay]
+        return (a[primaryDisplay] > b[primaryDisplay]) as unknown as number
       })
     }
   }
 
-  $: forceFetchRows(filter)
+  $: {
+    if (filter || defaultValue) {
+      forceFetchRows()
+    }
+  }
   $: debouncedFetchRows(searchTerm, primaryDisplay, defaultValue)
 
   const forceFetchRows = async () => {
@@ -119,7 +152,11 @@
     selectedValue = []
     debouncedFetchRows(searchTerm, primaryDisplay, defaultValue)
   }
-  const fetchRows = async (searchTerm, primaryDisplay, defaultVal) => {
+  async function fetchRows(
+    searchTerm: any,
+    primaryDisplay: string,
+    defaultVal: string | string[]
+  ) {
     const allRowsFetched =
       $fetch.loaded &&
       !Object.keys($fetch.query?.string || {}).length &&
@@ -129,17 +166,39 @@
       return
     }
     // must be an array
-    if (defaultVal && !Array.isArray(defaultVal)) {
-      defaultVal = defaultVal.split(",")
-    }
-    if (defaultVal && optionsObj && defaultVal.some(val => !optionsObj[val])) {
+    const defaultValArray: string[] = !defaultVal
+      ? []
+      : !Array.isArray(defaultVal)
+      ? defaultVal.split(",")
+      : defaultVal
+
+    if (
+      defaultVal &&
+      optionsObj &&
+      defaultValArray.some(val => !optionsObj[val])
+    ) {
       await fetch.update({
-        query: { oneOf: { _id: defaultVal } },
+        query: { oneOf: { _id: defaultValArray } },
+      })
+    }
+
+    if (
+      (Array.isArray(selectedValue) &&
+        selectedValue.some(val => !optionsObj[val])) ||
+      (selectedValue && !optionsObj[selectedValue as string])
+    ) {
+      await fetch.update({
+        query: {
+          oneOf: {
+            _id: Array.isArray(selectedValue) ? selectedValue : [selectedValue],
+          },
+        },
       })
     }
 
     // Ensure we match all filters, rather than any
-    const baseFilter = (filter || []).filter(x => x.operator !== "allOr")
+    // @ts-expect-error this doesn't fit types, but don't want to change it yet
+    const baseFilter: any = (filter || []).filter(x => x.operator !== "allOr")
     await fetch.update({
       filter: [
         ...baseFilter,
@@ -152,9 +211,8 @@
       ],
     })
   }
-  const debouncedFetchRows = Utils.debounce(fetchRows, 250)
 
-  const flatten = values => {
+  const flatten = (values: any | any[]) => {
     if (!values) {
       return []
     }
@@ -162,17 +220,17 @@
     if (!Array.isArray(values)) {
       values = [values]
     }
-    values = values.map(value =>
+    values = values.map((value: any) =>
       typeof value === "object" ? value._id : value
     )
     return values
   }
 
-  const getDisplayName = row => {
-    return row?.[primaryDisplay] || "-"
+  const getDisplayName = (row: Row) => {
+    return row?.[primaryDisplay!] || "-"
   }
 
-  const handleChange = e => {
+  const handleChange = (e: any) => {
     let value = e.detail
     if (!multiselect) {
       value = value == null ? [] : [value]
@@ -220,13 +278,12 @@
       this={component}
       options={enrichedOptions}
       {autocomplete}
-      value={selectedValue}
+      value={castSelectedValue}
       on:change={handleChange}
       on:loadMore={loadMore}
       id={fieldState.fieldId}
       disabled={fieldState.disabled}
       readonly={fieldState.readonly}
-      error={fieldState.error}
       getOptionLabel={getDisplayName}
       getOptionValue={option => option._id}
       {placeholder}

packages/client/src/components/app/forms/S3Upload.svelte

@@ -2,7 +2,7 @@
   import Field from "./Field.svelte"
   import { CoreDropzone, ProgressCircle, Helpers } from "@budibase/bbui"
   import { getContext, onMount, onDestroy } from "svelte"
-  import { builderStore } from "@/stores/builder.js"
+  import { builderStore } from "@/stores/builder"
   import { processStringSync } from "@budibase/string-templates"
 
   export let datasourceId

packages/client/src/components/context/TestUrlBindingsProvider.svelte

@@ -0,0 +1,8 @@
+<script>
+  import Provider from "./Provider.svelte"
+  import { routeStore } from "@/stores"
+</script>
+
+<Provider key="url" data={$routeStore.testUrlParams}>
+  <slot />
+</Provider>

packages/client/src/index.js

@@ -0,0 +1,142 @@
+import ClientApp from "./components/ClientApp.svelte"
+import UpdatingApp from "./components/UpdatingApp.svelte"
+import {
+  builderStore,
+  appStore,
+  blockStore,
+  componentStore,
+  environmentStore,
+  dndStore,
+  eventStore,
+  hoverStore,
+  stateStore,
+  routeStore,
+} from "./stores"
+import loadSpectrumIcons from "@budibase/bbui/spectrum-icons-vite.js"
+import { get } from "svelte/store"
+import { initWebsocket } from "./websocket.js"
+
+// Provide svelte and svelte/internal as globals for custom components
+import * as svelte from "svelte"
+import * as internal from "svelte/internal"
+
+window.svelte_internal = internal
+window.svelte = svelte
+
+// Initialise spectrum icons
+loadSpectrumIcons()
+
+let app
+
+const loadBudibase = async () => {
+  // Update builder store with any builder flags
+  builderStore.set({
+    ...get(builderStore),
+    inBuilder: !!window["##BUDIBASE_IN_BUILDER##"],
+    layout: window["##BUDIBASE_PREVIEW_LAYOUT##"],
+    screen: window["##BUDIBASE_PREVIEW_SCREEN##"],
+    selectedComponentId: window["##BUDIBASE_SELECTED_COMPONENT_ID##"],
+    previewId: window["##BUDIBASE_PREVIEW_ID##"],
+    theme: window["##BUDIBASE_PREVIEW_THEME##"],
+    customTheme: window["##BUDIBASE_PREVIEW_CUSTOM_THEME##"],
+    previewDevice: window["##BUDIBASE_PREVIEW_DEVICE##"],
+    navigation: window["##BUDIBASE_PREVIEW_NAVIGATION##"],
+    hiddenComponentIds: window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"],
+    usedPlugins: window["##BUDIBASE_USED_PLUGINS##"],
+    location: window["##BUDIBASE_LOCATION##"],
+    snippets: window["##BUDIBASE_SNIPPETS##"],
+    componentErrors: window["##BUDIBASE_COMPONENT_ERRORS##"],
+  })
+
+  // Set app ID - this window flag is set by both the preview and the real
+  // server rendered app HTML
+  appStore.actions.setAppId(window["##BUDIBASE_APP_ID##"])
+
+  // Set the flag used to determine if the app is being loaded via an iframe
+  appStore.actions.setAppEmbedded(
+    window["##BUDIBASE_APP_EMBEDDED##"] === "true"
+  )
+
+  if (window.MIGRATING_APP) {
+    new UpdatingApp({
+      target: window.document.body,
+    })
+    return
+  }
+
+  // Fetch environment info
+  if (!get(environmentStore)?.loaded) {
+    await environmentStore.actions.fetchEnvironment()
+  }
+
+  // Register handler for runtime events from the builder
+  window.handleBuilderRuntimeEvent = (type, data) => {
+    if (!window["##BUDIBASE_IN_BUILDER##"]) {
+      return
+    }
+    if (type === "event-completed") {
+      eventStore.actions.resolveEvent(data)
+    } else if (type === "eject-block") {
+      const block = blockStore.actions.getBlock(data)
+      block?.eject()
+    } else if (type === "dragging-new-component") {
+      const { dragging, component } = data
+      if (dragging) {
+        const definition =
+          componentStore.actions.getComponentDefinition(component)
+        dndStore.actions.startDraggingNewComponent({ component, definition })
+      } else {
+        dndStore.actions.reset()
+      }
+    } else if (type === "request-context") {
+      const { selectedComponentInstance, screenslotInstance } =
+        get(componentStore)
+      const instance = selectedComponentInstance || screenslotInstance
+      const context = instance?.getDataContext()
+      let stringifiedContext = null
+      try {
+        stringifiedContext = JSON.stringify(context)
+      } catch (error) {
+        // Ignore - invalid context
+      }
+      eventStore.actions.dispatchEvent("provide-context", {
+        context: stringifiedContext,
+      })
+    } else if (type === "hover-component") {
+      hoverStore.actions.hoverComponent(data, false)
+    } else if (type === "builder-meta") {
+      builderStore.actions.setMetadata(data)
+    } else if (type === "builder-state") {
+      const [[key, value]] = Object.entries(data)
+      stateStore.actions.setValue(key, value)
+    } else if (type === "builder-url-test-data") {
+      const { route, testValue } = data
+      routeStore.actions.setTestUrlParams(route, testValue)
+    }
+  }
+
+  // Register any custom components
+  if (window["##BUDIBASE_CUSTOM_COMPONENTS##"]) {
+    window["##BUDIBASE_CUSTOM_COMPONENTS##"].forEach(component => {
+      componentStore.actions.registerCustomComponent(component)
+    })
+  }
+
+  // Make a callback available for custom component bundles to register
+  // themselves at runtime
+  window.registerCustomComponent =
+    componentStore.actions.registerCustomComponent
+
+  // Initialise websocket
+  initWebsocket()
+
+  // Create app if one hasn't been created yet
+  if (!app) {
+    app = new ClientApp({
+      target: window.document.body,
+    })
+  }
+}
+
+// Attach to window so the HTML template can call this when it loads
+window.loadBudibase = loadBudibase

packages/client/src/index.ts

@@ -10,6 +10,7 @@ import {
   eventStore,
   hoverStore,
   stateStore,
+  routeStore,
 } from "@/stores"
 import { get } from "svelte/store"
 import { initWebsocket } from "@/websocket"
@@ -18,7 +19,6 @@ import type { ActionTypes } from "@/constants"
 import { Readable } from "svelte/store"
 import {
   Screen,
-  Layout,
   Theme,
   AppCustomTheme,
   PreviewDevice,
@@ -47,7 +47,6 @@ declare global {
     // Data from builder
     "##BUDIBASE_APP_ID##"?: string
     "##BUDIBASE_IN_BUILDER##"?: true
-    "##BUDIBASE_PREVIEW_LAYOUT##"?: Layout
     "##BUDIBASE_PREVIEW_SCREEN##"?: Screen
     "##BUDIBASE_SELECTED_COMPONENT_ID##"?: string
     "##BUDIBASE_PREVIEW_ID##"?: number
@@ -58,13 +57,8 @@ declare global {
     "##BUDIBASE_PREVIEW_NAVIGATION##"?: AppNavigation
     "##BUDIBASE_HIDDEN_COMPONENT_IDS##"?: string[]
     "##BUDIBASE_USED_PLUGINS##"?: Plugin[]
-    "##BUDIBASE_LOCATION##"?: {
-      protocol: string
-      hostname: string
-      port: string
-    }
     "##BUDIBASE_SNIPPETS##"?: Snippet[]
-    "##BUDIBASE_COMPONENT_ERRORS##"?: Record<string, UIComponentError>[]
+    "##BUDIBASE_COMPONENT_ERRORS##"?: Record<string, UIComponentError[]>
     "##BUDIBASE_CUSTOM_COMPONENTS##"?: CustomComponent[]
 
     // Other flags
@@ -114,7 +108,6 @@ const loadBudibase = async () => {
   builderStore.set({
     ...get(builderStore),
     inBuilder: !!window["##BUDIBASE_IN_BUILDER##"],
-    layout: window["##BUDIBASE_PREVIEW_LAYOUT##"],
     screen: window["##BUDIBASE_PREVIEW_SCREEN##"],
     selectedComponentId: window["##BUDIBASE_SELECTED_COMPONENT_ID##"],
     previewId: window["##BUDIBASE_PREVIEW_ID##"],
@@ -124,7 +117,6 @@ const loadBudibase = async () => {
     navigation: window["##BUDIBASE_PREVIEW_NAVIGATION##"],
     hiddenComponentIds: window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"],
     usedPlugins: window["##BUDIBASE_USED_PLUGINS##"],
-    location: window["##BUDIBASE_LOCATION##"],
     snippets: window["##BUDIBASE_SNIPPETS##"],
     componentErrors: window["##BUDIBASE_COMPONENT_ERRORS##"],
   })
@@ -188,6 +180,9 @@ const loadBudibase = async () => {
     } else if (type === "builder-state") {
       const [[key, value]] = Object.entries(data)
       stateStore.actions.setValue(key, value)
+    } else if (type === "builder-url-test-data") {
+      const { route, testValue } = data
+      routeStore.actions.setTestUrlParams(route, testValue)
     }
   }
 

packages/client/src/stores/builder.ts

@@ -2,9 +2,39 @@ import { writable, get } from "svelte/store"
 import { API } from "@/api"
 import { devToolsStore } from "./devTools.js"
 import { eventStore } from "./events.js"
+import {
+  ComponentDefinition,
+  DropPosition,
+  PingSource,
+  PreviewDevice,
+  Screen,
+  Theme,
+  AppCustomTheme,
+  AppNavigation,
+  Plugin,
+  Snippet,
+  UIComponentError,
+} from "@budibase/types"
+
+interface BuilderStore {
+  inBuilder: boolean
+  screen?: Screen | null
+  selectedComponentId?: string | null
+  editMode: boolean
+  previewId?: number | null
+  theme?: Theme | null
+  customTheme?: AppCustomTheme | null
+  previewDevice?: PreviewDevice
+  navigation?: AppNavigation | null
+  hiddenComponentIds?: string[]
+  usedPlugins?: Plugin[] | null
+  metadata: { componentId: string; step: number } | null
+  snippets?: Snippet[] | null
+  componentErrors?: Record<string, UIComponentError[]>
+}
 
 const createBuilderStore = () => {
-  const initialState = {
+  const initialState: BuilderStore = {
     inBuilder: false,
     screen: null,
     selectedComponentId: null,
@@ -16,17 +46,13 @@ const createBuilderStore = () => {
     navigation: null,
     hiddenComponentIds: [],
     usedPlugins: null,
-    eventResolvers: {},
     metadata: null,
     snippets: null,
     componentErrors: {},
-
-    // Legacy - allow the builder to specify a layout
-    layout: null,
   }
   const store = writable(initialState)
   const actions = {
-    selectComponent: id => {
+    selectComponent: (id: string) => {
       if (id === get(store).selectedComponentId) {
         return
       }
@@ -38,46 +64,59 @@ const createBuilderStore = () => {
       devToolsStore.actions.setAllowSelection(false)
       eventStore.actions.dispatchEvent("select-component", { id })
     },
-    updateProp: (prop, value) => {
+    updateProp: (prop: string, value: any) => {
       eventStore.actions.dispatchEvent("update-prop", { prop, value })
     },
-    updateStyles: async (styles, id) => {
+    updateStyles: async (styles: Record<string, any>, id: string) => {
       await eventStore.actions.dispatchEvent("update-styles", {
         styles,
         id,
       })
     },
-    keyDown: (key, ctrlKey) => {
+    keyDown: (key: string, ctrlKey: boolean) => {
       eventStore.actions.dispatchEvent("key-down", { key, ctrlKey })
     },
-    duplicateComponent: (id, mode = "below", selectComponent = true) => {
+    duplicateComponent: (
+      id: string,
+      mode = DropPosition.BELOW,
+      selectComponent = true
+    ) => {
       eventStore.actions.dispatchEvent("duplicate-component", {
         id,
         mode,
         selectComponent,
       })
     },
-    deleteComponent: id => {
+    deleteComponent: (id: string) => {
       eventStore.actions.dispatchEvent("delete-component", { id })
     },
     notifyLoaded: () => {
       eventStore.actions.dispatchEvent("preview-loaded")
     },
-    analyticsPing: async ({ embedded }) => {
+    analyticsPing: async ({ embedded }: { embedded: boolean }) => {
       try {
-        await API.analyticsPing({ source: "app", embedded })
+        await API.analyticsPing({ source: PingSource.APP, embedded })
       } catch (error) {
         // Do nothing
       }
     },
-    moveComponent: async (componentId, destinationComponentId, mode) => {
+    moveComponent: async (
+      componentId: string,
+      destinationComponentId: string,
+      mode: DropPosition
+    ) => {
       await eventStore.actions.dispatchEvent("move-component", {
         componentId,
         destinationComponentId,
         mode,
       })
     },
-    dropNewComponent: (component, parent, index, props) => {
+    dropNewComponent: (
+      component: string,
+      parent: string,
+      index: number,
+      props: Record<string, any>
+    ) => {
       eventStore.actions.dispatchEvent("drop-new-component", {
         component,
         parent,
@@ -85,7 +124,7 @@ const createBuilderStore = () => {
         props,
       })
     },
-    setEditMode: enabled => {
+    setEditMode: (enabled: boolean) => {
       if (enabled === get(store).editMode) {
         return
       }
@@ -94,18 +133,18 @@ const createBuilderStore = () => {
     requestAddComponent: () => {
       eventStore.actions.dispatchEvent("request-add-component")
     },
-    highlightSetting: setting => {
+    highlightSetting: (setting: string) => {
       eventStore.actions.dispatchEvent("highlight-setting", { setting })
     },
-    ejectBlock: (id, definition) => {
+    ejectBlock: (id: string, definition: ComponentDefinition) => {
       eventStore.actions.dispatchEvent("eject-block", { id, definition })
     },
-    updateUsedPlugin: (name, hash) => {
+    updateUsedPlugin: (name: string, hash: string) => {
       // Check if we used this plugin
       const used = get(store)?.usedPlugins?.find(x => x.name === name)
       if (used) {
         store.update(state => {
-          state.usedPlugins = state.usedPlugins.filter(x => x.name !== name)
+          state.usedPlugins = state.usedPlugins!.filter(x => x.name !== name)
           state.usedPlugins.push({
             ...used,
             hash,
@@ -117,13 +156,13 @@ const createBuilderStore = () => {
       // Notify the builder so we can reload component definitions
       eventStore.actions.dispatchEvent("reload-plugin")
     },
-    addParentComponent: (componentId, parentType) => {
+    addParentComponent: (componentId: string, parentType: string) => {
       eventStore.actions.dispatchEvent("add-parent-component", {
         componentId,
         parentType,
       })
     },
-    setMetadata: metadata => {
+    setMetadata: (metadata: { componentId: string; step: number }) => {
       store.update(state => ({
         ...state,
         metadata,
@@ -132,7 +171,7 @@ const createBuilderStore = () => {
   }
   return {
     ...store,
-    set: state => store.set({ ...initialState, ...state }),
+    set: (state: BuilderStore) => store.set({ ...initialState, ...state }),
     actions,
   }
 }

packages/client/src/stores/derived/devToolsEnabled.js

@@ -1,6 +1,6 @@
 import { derived } from "svelte/store"
 import { appStore } from "../app.js"
-import { builderStore } from "../builder.js"
+import { builderStore } from "../builder"
 
 export const devToolsEnabled = derived(
   [appStore, builderStore],

packages/client/src/stores/derived/snippets.js

@@ -1,5 +1,5 @@
 import { appStore } from "../app.js"
-import { builderStore } from "../builder.js"
+import { builderStore } from "../builder"
 import { derivedMemo } from "@budibase/frontend-core"
 
 export const snippets = derivedMemo(

packages/client/src/stores/routes.ts

@@ -119,7 +119,39 @@ const createRouteStore = () => {
     const base = window.location.href.split("#")[0]
     return `${base}#${relativeURL}`
   }
+  const setTestUrlParams = (route: string, testValue: string) => {
+    if (route === "/") {
+      return
+    }
 
+    const [pathPart, queryPart] = testValue.split("?")
+    const routeSegments = route.split("/").filter(Boolean)
+
+    // If first segment happens to be a parameter (e.g. /:foo), include it
+    const startIndex = routeSegments[0]?.startsWith(":") ? 0 : 1
+    const segments = routeSegments.slice(startIndex)
+    const testSegments = pathPart.split("/")
+
+    const params: Record<string, string> = {}
+    segments.forEach((segment, index) => {
+      if (segment.startsWith(":") && index < testSegments.length) {
+        params[segment.slice(1)] = testSegments[index]
+      }
+    })
+
+    const queryParams: Record<string, string> = {}
+    if (queryPart) {
+      queryPart.split("&").forEach(param => {
+        const [key, value] = param.split("=")
+        if (key && value) {
+          queryParams[key] = value
+        }
+      })
+    }
+
+    setQueryParams({ ...queryParams })
+    store.update(state => ({ ...state, testUrlParams: params }))
+  }
   return {
     subscribe: store.subscribe,
     actions: {
@@ -130,6 +162,7 @@ const createRouteStore = () => {
       setQueryParams,
       setActiveRoute,
       setRouterLoaded,
+      setTestUrlParams,
     },
   }
 }

packages/client/src/stores/screens.js

@@ -36,11 +36,6 @@ const createScreenStore = () => {
         activeScreen = Helpers.cloneDeep($builderStore.screen)
         screens = [activeScreen]
 
-        // Legacy - allow the builder to specify a layout
-        if ($builderStore.layout) {
-          activeLayout = $builderStore.layout
-        }
-
         // Attach meta
         const errors = $builderStore.componentErrors || {}
         const attachComponentMeta = component => {

packages/frontend-core/src/components/grid/stores/conditions.ts

@@ -31,7 +31,7 @@ export const deriveStores = (context: StoreContext): ConditionDerivedStore => {
   // Derive and memoize the cell conditions present in our columns so that we
   // only recompute condition metadata when absolutely necessary
   const conditions = derivedMemo(columns, $columns => {
-    let newConditions = []
+    let newConditions: UICondition[] = []
     for (let column of $columns) {
       for (let condition of column.conditions || []) {
         newConditions.push({

packages/server/src/api/controllers/static/index.ts

@@ -339,10 +339,13 @@ export const getSignedUploadURL = async function (
       ctx.throw(400, "bucket and key values are required")
     }
     try {
+      let endpoint = datasource?.config?.endpoint
+      if (endpoint && !utils.urlHasProtocol(endpoint)) {
+        endpoint = `https://${endpoint}`
+      }
       const s3 = new S3({
         region: awsRegion,
-        endpoint: datasource?.config?.endpoint || undefined,
-
+        endpoint: endpoint,
         credentials: {
           accessKeyId: datasource?.config?.accessKeyId as string,
           secretAccessKey: datasource?.config?.secretAccessKey as string,
@@ -350,8 +353,8 @@ export const getSignedUploadURL = async function (
       })
       const params = { Bucket: bucket, Key: key }
       signedUrl = await getSignedUrl(s3, new PutObjectCommand(params))
-      if (datasource?.config?.endpoint) {
-        publicUrl = `${datasource.config.endpoint}/${bucket}/${key}`
+      if (endpoint) {
+        publicUrl = `${endpoint}/${bucket}/${key}`
       } else {
         publicUrl = `https://${bucket}.s3.${awsRegion}.amazonaws.com/${key}`
       }

packages/server/src/api/controllers/static/templates/preview.hbs

@@ -80,7 +80,6 @@
       // Set some flags so the app knows we're in the builder
       window["##BUDIBASE_IN_BUILDER##"] = true
       window["##BUDIBASE_APP_ID##"] = appId
-      window["##BUDIBASE_PREVIEW_LAYOUT##"] = layout
       window["##BUDIBASE_PREVIEW_SCREEN##"] = screen
       window["##BUDIBASE_SELECTED_COMPONENT_ID##"] = selectedComponentId
       window["##BUDIBASE_PREVIEW_ID##"] = Math.random()
@@ -90,7 +89,6 @@
       window["##BUDIBASE_PREVIEW_NAVIGATION##"] = navigation
       window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"] = hiddenComponentIds
       window["##BUDIBASE_USED_PLUGINS##"] = usedPlugins
-      window["##BUDIBASE_LOCATION##"] = location
       window["##BUDIBASE_SNIPPETS##"] = snippets
       window['##BUDIBASE_COMPONENT_ERRORS##'] = componentErrors
 

packages/server/src/api/routes/backup.ts

@@ -2,12 +2,14 @@ import Router from "@koa/router"
 import * as controller from "../controllers/backup"
 import authorized from "../../middleware/authorized"
 import { permissions } from "@budibase/backend-core"
+import ensureTenantAppOwnership from "../../middleware/ensureTenantAppOwnership"
 
 const router: Router = new Router()
 
 router.post(
   "/api/backups/export",
   authorized(permissions.BUILDER),
+  ensureTenantAppOwnership,
   controller.exportAppDump
 )
 

packages/server/src/automations/tests/steps/cron-automations.spec.ts

@@ -1,45 +0,0 @@
-import tk from "timekeeper"
-import "../../../environment"
-import * as automations from "../../index"
-import TestConfiguration from "../../../tests/utilities/TestConfiguration"
-import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
-
-const initialTime = Date.now()
-tk.freeze(initialTime)
-
-const oneMinuteInMs = 60 * 1000
-
-describe("cron automations", () => {
-  const config = new TestConfiguration()
-
-  beforeAll(async () => {
-    await automations.init()
-    await config.init()
-  })
-
-  afterAll(async () => {
-    await automations.shutdown()
-    config.end()
-  })
-
-  beforeEach(() => {
-    tk.freeze(initialTime)
-  })
-
-  it("should initialise the automation timestamp", async () => {
-    await createAutomationBuilder(config).onCron({ cron: "* * * * *" }).save()
-
-    tk.travel(Date.now() + oneMinuteInMs)
-    await config.publish()
-
-    const { data } = await config.getAutomationLogs()
-    expect(data).toHaveLength(1)
-    expect(data).toEqual([
-      expect.objectContaining({
-        trigger: expect.objectContaining({
-          outputs: { timestamp: initialTime + oneMinuteInMs },
-        }),
-      }),
-    ])
-  })
-})

packages/server/src/automations/tests/triggers/cron.spec.ts

@@ -1,6 +1,11 @@
 import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
-import { captureAutomationResults } from "../utilities"
+import {
+  captureAutomationQueueMessages,
+  captureAutomationResults,
+} from "../utilities"
+import { automations } from "@budibase/pro"
+import { AutomationStatus } from "@budibase/types"
 
 describe("cron trigger", () => {
   const config = new TestConfiguration()
@@ -13,6 +18,13 @@ describe("cron trigger", () => {
     config.end()
   })
 
+  beforeEach(async () => {
+    const { automations } = await config.api.automation.fetch()
+    for (const automation of automations) {
+      await config.api.automation.delete(automation)
+    }
+  })
+
   it("should queue a Bull cron job", async () => {
     const { automation } = await createAutomationBuilder(config)
       .onCron({ cron: "* * * * *" })
@@ -21,12 +33,12 @@ describe("cron trigger", () => {
       })
       .save()
 
-    const jobs = await captureAutomationResults(automation, () =>
+    const messages = await captureAutomationQueueMessages(automation, () =>
       config.api.application.publish()
     )
-    expect(jobs).toHaveLength(1)
+    expect(messages).toHaveLength(1)
 
-    const repeat = jobs[0].opts?.repeat
+    const repeat = messages[0].opts?.repeat
     if (!repeat || !("cron" in repeat)) {
       throw new Error("Expected cron repeat")
     }
@@ -49,4 +61,82 @@ describe("cron trigger", () => {
       },
     })
   })
+
+  it("should stop if the job fails more than 3 times", async () => {
+    const runner = await createAutomationBuilder(config)
+      .onCron({ cron: "* * * * *" })
+      .queryRows({
+        // @ts-expect-error intentionally sending invalid data
+        tableId: null,
+      })
+      .save()
+
+    await config.api.application.publish()
+
+    const results = await captureAutomationResults(
+      runner.automation,
+      async () => {
+        await runner.trigger({ timeout: 1000, fields: {} })
+        await runner.trigger({ timeout: 1000, fields: {} })
+        await runner.trigger({ timeout: 1000, fields: {} })
+        await runner.trigger({ timeout: 1000, fields: {} })
+        await runner.trigger({ timeout: 1000, fields: {} })
+      }
+    )
+
+    expect(results).toHaveLength(5)
+
+    await config.withProdApp(async () => {
+      const {
+        data: [latest, ..._],
+      } = await automations.logs.logSearch({
+        automationId: runner.automation._id,
+      })
+      expect(latest.status).toEqual(AutomationStatus.STOPPED_ERROR)
+    })
+  })
+
+  it("should fill in the timestamp if one is not provided", async () => {
+    const runner = await createAutomationBuilder(config)
+      .onCron({ cron: "* * * * *" })
+      .serverLog({
+        text: "Hello, world!",
+      })
+      .save()
+
+    await config.api.application.publish()
+
+    const results = await captureAutomationResults(
+      runner.automation,
+      async () => {
+        await runner.trigger({ timeout: 1000, fields: {} })
+      }
+    )
+    expect(results).toHaveLength(1)
+    expect(results[0].data.event.timestamp).toBeWithin(
+      Date.now() - 1000,
+      Date.now() + 1000
+    )
+  })
+
+  it("should use the given timestamp if one is given", async () => {
+    const timestamp = 1234
+    const runner = await createAutomationBuilder(config)
+      .onCron({ cron: "* * * * *" })
+      .serverLog({
+        text: "Hello, world!",
+      })
+      .save()
+
+    await config.api.application.publish()
+
+    const results = await captureAutomationResults(
+      runner.automation,
+      async () => {
+        await runner.trigger({ timeout: 1000, fields: {}, timestamp })
+      }
+    )
+    expect(results).toHaveLength(1)
+    expect(results[0].data.event.timestamp).toEqual(timestamp)
+  })
 })

packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts

@@ -220,10 +220,34 @@ class AutomationRunner<TStep extends AutomationTriggerStepId> {
   async trigger(
     request: TriggerAutomationRequest
   ): Promise<TriggerAutomationResponse> {
-    return await this.config.api.automation.trigger(
-      this.automation._id!,
-      request
-    )
+    if (!this.config.prodAppId) {
+      throw new Error(
+        "Automations can only be triggered in a production app context, call config.api.application.publish()"
+      )
+    }
+    // Because you can only trigger automations in a production app context, we
+    // wrap the trigger call to make tests a bit cleaner. If you really want to
+    // test triggering an automation in a dev app context, you can use the
+    // automation API directly.
+    return await this.config.withProdApp(async () => {
+      try {
+        return await this.config.api.automation.trigger(
+          this.automation._id!,
+          request
+        )
+      } catch (e: any) {
+        if (e.cause.status === 404) {
+          throw new Error(
+            `Automation with ID ${
+              this.automation._id
+            } not found in app ${this.config.getAppId()}. You may have forgotten to call config.api.application.publish().`,
+            { cause: e }
+          )
+        } else {
+          throw e
+        }
+      }
+    })
   }
 }
 

packages/server/src/automations/tests/utilities/index.ts

@@ -34,6 +34,42 @@ export async function runInProd(fn: any) {
   }
 }
 
+export async function captureAllAutomationQueueMessages(
+  f: () => Promise<unknown>
+) {
+  const messages: Job<AutomationData>[] = []
+  const queue = getQueue()
+
+  const messageListener = async (message: Job<AutomationData>) => {
+    messages.push(message)
+  }
+
+  queue.on("message", messageListener)
+  try {
+    await f()
+    // Queue messages tend to be send asynchronously in API handlers, so there's
+    // no guarantee that awaiting this function will have queued anything yet.
+    // We wait here to make sure we're queued _after_ any existing async work.
+    await helpers.wait(100)
+  } finally {
+    queue.off("message", messageListener)
+  }
+
+  return messages
+}
+
+export async function captureAutomationQueueMessages(
+  automation: Automation | string,
+  f: () => Promise<unknown>
+) {
+  const messages = await captureAllAutomationQueueMessages(f)
+  return messages.filter(
+    m =>
+      m.data.automation._id ===
+      (typeof automation === "string" ? automation : automation._id)
+  )
+}
+
 /**
  * Capture all automation runs that occur during the execution of a function.
  * This function will wait for all messages to be processed before returning.
@@ -43,14 +79,18 @@ export async function captureAllAutomationResults(
 ): Promise<Job<AutomationData>[]> {
   const runs: Job<AutomationData>[] = []
   const queue = getQueue()
-  let messagesReceived = 0
+  let messagesOutstanding = 0
 
   const completedListener = async (job: Job<AutomationData>) => {
     runs.push(job)
-    messagesReceived--
+    messagesOutstanding--
   }
-  const messageListener = async () => {
-    messagesReceived++
+  const messageListener = async (message: Job<AutomationData>) => {
+    // Don't count cron messages, as they don't get triggered automatically.
+    if (message.opts?.repeat != null) {
+      return
+    }
+    messagesOutstanding++
   }
   queue.on("message", messageListener)
   queue.on("completed", completedListener)
@@ -61,9 +101,18 @@ export async function captureAllAutomationResults(
     // We wait here to make sure we're queued _after_ any existing async work.
     await helpers.wait(100)
   } finally {
+    const waitMax = 10000
+    let waited = 0
     // eslint-disable-next-line no-unmodified-loop-condition
-    while (messagesReceived > 0) {
+    while (messagesOutstanding > 0) {
       await helpers.wait(50)
+      waited += 50
+      if (waited > waitMax) {
+        // eslint-disable-next-line no-unsafe-finally
+        throw new Error(
+          `Timed out waiting for automation runs to complete. ${messagesOutstanding} messages waiting for completion.`
+        )
+      }
     }
     queue.off("completed", completedListener)
     queue.off("message", messageListener)

packages/server/src/automations/utils.ts

@@ -72,7 +72,7 @@ export async function processEvent(job: AutomationJob) {
 
       const task = async () => {
         try {
-          if (isCronTrigger(job.data.automation)) {
+          if (isCronTrigger(job.data.automation) && !job.data.event.timestamp) {
             // Requires the timestamp at run time
             job.data.event.timestamp = Date.now()
           }

packages/server/src/integrations/s3.ts

@@ -7,7 +7,7 @@ import {
   ConnectionInfo,
 } from "@budibase/types"
 
-import { S3 } from "@aws-sdk/client-s3"
+import { S3, S3ClientConfig } from "@aws-sdk/client-s3"
 import csv from "csvtojson"
 import stream from "stream"
 
@@ -157,13 +157,20 @@ const SCHEMA: Integration = {
 }
 
 class S3Integration implements IntegrationBase {
-  private readonly config: S3Config
-  private client
+  private readonly config: S3ClientConfig
+  private client: S3
 
   constructor(config: S3Config) {
-    this.config = config
-    if (this.config.endpoint) {
-      this.config.s3ForcePathStyle = true
+    this.config = {
+      forcePathStyle: config.s3ForcePathStyle || true,
+      credentials: {
+        accessKeyId: config.accessKeyId,
+        secretAccessKey: config.secretAccessKey,
+      },
+      region: config.region,
+    }
+    if (config.endpoint) {
+      this.config.forcePathStyle = true
     } else {
       delete this.config.endpoint
     }
@@ -176,7 +183,9 @@ class S3Integration implements IntegrationBase {
       connected: false,
     }
     try {
-      await this.client.listBuckets()
+      await this.client.listBuckets({
+        MaxBuckets: 1,
+      })
       response.connected = true
     } catch (e: any) {
       response.error = e.message as string
@@ -253,7 +262,7 @@ class S3Integration implements IntegrationBase {
         .on("error", () => {
           csvError = true
         })
-      fileStream.on("finish", () => {
+      fileStream.on("end", () => {
         resolve(response)
       })
     }).catch(err => {

packages/server/src/middleware/ensureTenantAppOwnership.ts

@@ -0,0 +1,19 @@
+import { tenancy, utils, context } from "@budibase/backend-core"
+import { UserCtx } from "@budibase/types"
+
+async function ensureTenantAppOwnership(ctx: UserCtx, next: any) {
+  const appId = await utils.getAppIdFromCtx(ctx)
+  if (!appId) {
+    ctx.throw(400, "appId must be provided")
+  }
+
+  const appTenantId = context.getTenantIDFromAppID(appId)
+  const tenantId = tenancy.getTenantId()
+
+  if (appTenantId !== tenantId) {
+    ctx.throw(403, "Unauthorized")
+  }
+  await next()
+}
+
+export default ensureTenantAppOwnership

packages/server/src/middleware/tests/ensureTenantAppOwnership.spec.js

@@ -0,0 +1,75 @@
+import ensureTenantAppOwnership from "../ensureTenantAppOwnership"
+import { tenancy, utils } from "@budibase/backend-core"
+
+jest.mock("@budibase/backend-core", () => ({
+  ...jest.requireActual("@budibase/backend-core"),
+  tenancy: {
+    getTenantId: jest.fn(),
+  },
+  utils: {
+    getAppIdFromCtx: jest.fn(),
+  },
+}))
+
+class TestConfiguration {
+  constructor(appId = "tenant_1") {
+    this.next = jest.fn()
+    this.throw = jest.fn()
+    this.middleware = ensureTenantAppOwnership
+
+    this.ctx = {
+      next: this.next,
+      throw: this.throw,
+    }
+
+    utils.getAppIdFromCtx.mockResolvedValue(appId)
+  }
+
+  async executeMiddleware() {
+    return this.middleware(this.ctx, this.next)
+  }
+
+  afterEach() {
+    jest.clearAllMocks()
+  }
+}
+
+describe("Ensure Tenant Ownership Middleware", () => {
+  let config
+
+  beforeEach(() => {
+    config = new TestConfiguration()
+  })
+
+  afterEach(() => {
+    config.afterEach()
+  })
+
+  it("calls next() when appId matches tenant ID", async () => {
+    tenancy.getTenantId.mockReturnValue("tenant_1")
+
+    await config.executeMiddleware()
+
+    expect(utils.getAppIdFromCtx).toHaveBeenCalledWith(config.ctx)
+    expect(config.next).toHaveBeenCalled()
+  })
+
+  it("throws when tenant appId does not match tenant ID", async () => {
+    const appId = "app_dev_tenant3_fce449c4d75b4e4a9c7a6980d82a3e22"
+    utils.getAppIdFromCtx.mockResolvedValue(appId)
+    tenancy.getTenantId.mockReturnValue("tenant_2")
+
+    await config.executeMiddleware()
+
+    expect(utils.getAppIdFromCtx).toHaveBeenCalledWith(config.ctx)
+    expect(config.throw).toHaveBeenCalledWith(403, "Unauthorized")
+  })
+
+  it("throws 400 when appId is missing", async () => {
+    utils.getAppIdFromCtx.mockResolvedValue(null)
+
+    await config.executeMiddleware()
+
+    expect(config.throw).toHaveBeenCalledWith(400, "appId must be provided")
+  })
+})

packages/server/src/sdk/app/datasources/datasources.ts

@@ -120,7 +120,7 @@ export function areRESTVariablesValid(datasource: Datasource) {
 
 export function checkDatasourceTypes(schema: Integration, config: any) {
   for (let key of Object.keys(config)) {
-    if (!schema.datasource[key]) {
+    if (!schema.datasource?.[key]) {
       continue
     }
     const type = schema.datasource[key].type
@@ -149,7 +149,9 @@ async function enrichDatasourceWithValues(
   ) as Datasource
   processed.entities = entities
   const definition = await getDefinition(processed.source)
-  processed.config = checkDatasourceTypes(definition!, processed.config)
+  if (definition) {
+    processed.config = checkDatasourceTypes(definition, processed.config)
+  }
   return {
     datasource: processed,
     envVars: env as Record<string, string>,

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -261,11 +261,13 @@ export default class TestConfiguration {
   async withApp<R>(app: App | string, f: () => Promise<R>) {
     const oldAppId = this.appId
     this.appId = typeof app === "string" ? app : app.appId
-    try {
-      return await f()
-    } finally {
-      this.appId = oldAppId
-    }
+    return await context.doInAppContext(this.appId, async () => {
+      try {
+        return await f()
+      } finally {
+        this.appId = oldAppId
+      }
+    })
   }
 
   async withProdApp<R>(f: () => Promise<R>) {

packages/server/src/threads/automation.ts

@@ -155,23 +155,12 @@ class Orchestrator {
     return step
   }
 
-  async getMetadata(): Promise<AutomationMetadata> {
-    const metadataId = generateAutomationMetadataID(this.automation._id!)
-    const db = context.getAppDB()
-    let metadata: AutomationMetadata
-    try {
-      metadata = await db.get(metadataId)
-    } catch (err) {
-      metadata = {
-        _id: metadataId,
-        errorCount: 0,
-      }
-    }
-    return metadata
+  isCron(): boolean {
+    return isRecurring(this.automation)
   }
 
   async stopCron(reason: string) {
-    if (!this.job.opts.repeat) {
+    if (!this.isCron()) {
       return
     }
     logging.logWarn(
@@ -192,44 +181,42 @@ class Orchestrator {
     await storeLog(automation, this.executionOutput)
   }
 
-  async checkIfShouldStop(metadata: AutomationMetadata): Promise<boolean> {
-    if (!metadata.errorCount || !this.job.opts.repeat) {
+  async checkIfShouldStop(): Promise<boolean> {
+    const metadata = await this.getMetadata()
+    if (!metadata.errorCount || !this.isCron()) {
       return false
     }
     if (metadata.errorCount >= MAX_AUTOMATION_RECURRING_ERRORS) {
-      await this.stopCron("errors")
       return true
     }
     return false
   }
 
-  async updateMetadata(metadata: AutomationMetadata) {
-    const output = this.executionOutput,
-      automation = this.automation
-    if (!output || !isRecurring(automation)) {
-      return
-    }
-    const count = metadata.errorCount
-    const isError = isErrorInOutput(output)
-    // nothing to do in this scenario, escape
-    if (!count && !isError) {
-      return
-    }
-    if (isError) {
-      metadata.errorCount = count ? count + 1 : 1
-    } else {
-      metadata.errorCount = 0
-    }
+  async getMetadata(): Promise<AutomationMetadata> {
+    const metadataId = generateAutomationMetadataID(this.automation._id!)
     const db = context.getAppDB()
-    try {
-      await db.put(metadata)
-    } catch (err) {
-      logging.logAlertWithInfo(
-        "Failed to write automation metadata",
-        db.name,
-        automation._id!,
-        err
-      )
+    const metadata = await db.tryGet<AutomationMetadata>(metadataId)
+    return metadata || { _id: metadataId, errorCount: 0 }
+  }
+
+  async incrementErrorCount() {
+    for (let attempt = 0; attempt < 3; attempt++) {
+      const metadata = await this.getMetadata()
+      metadata.errorCount ||= 0
+      metadata.errorCount++
+
+      const db = context.getAppDB()
+      try {
+        await db.put(metadata)
+        return
+      } catch (err) {
+        logging.logAlertWithInfo(
+          "Failed to update error count in automation metadata",
+          db.name,
+          this.automation._id!,
+          err
+        )
+      }
     }
   }
 
@@ -293,18 +280,6 @@ class Orchestrator {
         await enrichBaseContext(this.context)
         this.context.user = this.currentUser
 
-        let metadata
-
-        // check if this is a recurring automation,
-        if (isProdAppID(this.appId) && isRecurring(this.automation)) {
-          span?.addTags({ recurring: true })
-          metadata = await this.getMetadata()
-          const shouldStop = await this.checkIfShouldStop(metadata)
-          if (shouldStop) {
-            span?.addTags({ shouldStop: true })
-            return
-          }
-        }
         const start = performance.now()
 
         await this.executeSteps(this.automation.definition.steps)
@@ -332,10 +307,15 @@ class Orchestrator {
         }
         if (
           isProdAppID(this.appId) &&
-          isRecurring(this.automation) &&
-          metadata
+          this.isCron() &&
+          isErrorInOutput(this.executionOutput)
         ) {
-          await this.updateMetadata(metadata)
+          await this.incrementErrorCount()
+          if (await this.checkIfShouldStop()) {
+            await this.stopCron("errors")
+            span?.addTags({ shouldStop: true })
+            return
+          }
         }
         return this.executionOutput
       }

packages/types/src/api/web/app/automation.ts

@@ -65,6 +65,7 @@ export interface ClearAutomationLogResponse {
 
 export interface TriggerAutomationRequest {
   fields: Record<string, any>
+  timestamp?: number
   // time in seconds
   timeout: number
 }

packages/types/src/sdk/datasources.ts

@@ -157,7 +157,7 @@ export interface Integration {
   friendlyName: string
   type?: string
   iconUrl?: string
-  datasource: DatasourceConfig
+  datasource?: DatasourceConfig
   query: {
     [key: string]: QueryDefinition
   }

packages/types/src/ui/bindings/helper.ts

@@ -1,4 +1,6 @@
 export interface Helper {
   example: string
   description: string
+  args: any[]
+  requiresBlock?: boolean
 }

packages/types/src/ui/components/index.ts

@@ -48,6 +48,11 @@ export interface ComponentSetting {
   selectAllFields?: boolean
   resetOn?: string | string[]
   settings?: ComponentSetting[]
+  nested?: boolean
   dependsOn?: DependsOnComponentSetting
   sectionDependsOn?: DependsOnComponentSetting
+  contextAccess?: {
+    global: boolean
+    self: boolean
+  }
 }

packages/types/src/ui/stores/grid/columns.ts

@@ -1,9 +1,15 @@
-import { CalculationType, FieldSchema, FieldType, UIRow } from "@budibase/types"
+import {
+  CalculationType,
+  FieldSchema,
+  FieldType,
+  UICondition,
+  UIRow,
+} from "@budibase/types"
 
 export type UIColumn = FieldSchema & {
   label: string
   readonly: boolean
-  conditions: any
+  conditions?: UICondition[]
   format?: (row: UIRow) => any
   related?: {
     field: string

packages/types/src/ui/stores/grid/condition.ts

@@ -3,7 +3,7 @@ import { FieldType, SearchFilter } from "@budibase/types"
 export interface UICondition {
   column: string
   type: FieldType
-  referenceValue: string
+  referenceValue: any
   operator: SearchFilter["operator"]
   metadataKey: string
   metadataValue: string

packages/types/src/ui/stores/misc.ts

@@ -1,2 +1,8 @@
 // type purely to capture structures that the type is unknown, but maybe known later
 export type UIObject = Record<string, any>
+
+export const enum DropPosition {
+  ABOVE = "above",
+  BELOW = "below",
+  INSIDE = "inside",
+}

packages/types/src/ui/stores/preview.ts

@@ -1 +1,2 @@
 export type PreviewDevice = "desktop" | "tablet" | "mobile"
+export type ComponentContext = Record<string, any>