Merge branch 'master' into redis-sessions

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -3,6 +3,7 @@ import { newid } from "../utils"
 import { Queue, QueueOptions, JobOptions } from "./queue"
 import { helpers } from "@budibase/shared-core"
 import { Job, JobId, JobInformation } from "bull"
+import { cloneDeep } from "lodash"
 
 function jobToJobInformation(job: Job): JobInformation {
   let cron = ""
@@ -33,12 +34,13 @@ function jobToJobInformation(job: Job): JobInformation {
   }
 }
 
-interface JobMessage<T = any> extends Partial<Job<T>> {
+export interface TestQueueMessage<T = any> extends Partial<Job<T>> {
   id: string
   timestamp: number
   queue: Queue<T>
   data: any
   opts?: JobOptions
+  manualTrigger?: boolean
 }
 
 /**
@@ -47,12 +49,16 @@ interface JobMessage<T = any> extends Partial<Job<T>> {
  * internally to register when messages are available to the consumers - in can
  * support many inputs and many consumers.
  */
-class InMemoryQueue implements Partial<Queue> {
+export class InMemoryQueue<T = any> implements Partial<Queue<T>> {
   _name: string
   _opts?: QueueOptions
-  _messages: JobMessage[]
+  _messages: TestQueueMessage<T>[]
   _queuedJobIds: Set<string>
-  _emitter: NodeJS.EventEmitter<{ message: [JobMessage]; completed: [Job] }>
+  _emitter: NodeJS.EventEmitter<{
+    message: [TestQueueMessage<T>]
+    completed: [Job<T>]
+    removed: [TestQueueMessage<T>]
+  }>
   _runCount: number
   _addCount: number
 
@@ -82,7 +88,15 @@ class InMemoryQueue implements Partial<Queue> {
    */
   async process(concurrencyOrFunc: number | any, func?: any) {
     func = typeof concurrencyOrFunc === "number" ? func : concurrencyOrFunc
-    this._emitter.on("message", async message => {
+    this._emitter.on("message", async msg => {
+      const message = cloneDeep(msg)
+
+      // For the purpose of testing, don't trigger cron jobs immediately.
+      // Require the test to trigger them manually with timestamps.
+      if (!message.manualTrigger && message.opts?.repeat != null) {
+        return
+      }
+
       let resp = func(message)
 
       async function retryFunc(fnc: any) {
@@ -97,7 +111,7 @@ class InMemoryQueue implements Partial<Queue> {
       if (resp.then != null) {
         try {
           await retryFunc(resp)
-          this._emitter.emit("completed", message as Job)
+          this._emitter.emit("completed", message as Job<T>)
         } catch (e: any) {
           console.error(e)
         }
@@ -114,7 +128,6 @@ class InMemoryQueue implements Partial<Queue> {
     return this as any
   }
 
-  // simply puts a message to the queue and emits to the queue for processing
   /**
    * Simple function to replicate the add message functionality of Bull, putting
    * a new message on the queue. This then emits an event which will be used to
@@ -123,7 +136,13 @@ class InMemoryQueue implements Partial<Queue> {
    * a JSON message as this is required by Bull.
    * @param repeat serves no purpose for the import queue.
    */
-  async add(data: any, opts?: JobOptions) {
+  async add(data: T | string, optsOrT?: JobOptions | T) {
+    if (typeof data === "string") {
+      throw new Error("doesn't support named jobs")
+    }
+
+    const opts = optsOrT as JobOptions
+
     const jobId = opts?.jobId?.toString()
     if (jobId && this._queuedJobIds.has(jobId)) {
       console.log(`Ignoring already queued job ${jobId}`)
@@ -138,7 +157,7 @@ class InMemoryQueue implements Partial<Queue> {
     }
 
     const pushMessage = () => {
-      const message: JobMessage = {
+      const message: TestQueueMessage = {
         id: newid(),
         timestamp: Date.now(),
         queue: this as unknown as Queue,
@@ -164,13 +183,14 @@ class InMemoryQueue implements Partial<Queue> {
    */
   async close() {}
 
-  /**
-   * This removes a cron which has been implemented, this is part of Bull API.
-   * @param cronJobId The cron which is to be removed.
-   */
-  async removeRepeatableByKey(cronJobId: string) {
-    // TODO: implement for testing
-    console.log(cronJobId)
+  async removeRepeatableByKey(id: string) {
+    for (const [idx, message] of this._messages.entries()) {
+      if (message.id === id) {
+        this._messages.splice(idx, 1)
+        this._emitter.emit("removed", message)
+        return
+      }
+    }
   }
 
   async removeJobs(_pattern: string) {
@@ -193,6 +213,16 @@ class InMemoryQueue implements Partial<Queue> {
     return null
   }
 
+  manualTrigger(id: JobId) {
+    for (const message of this._messages) {
+      if (message.id === id) {
+        this._emitter.emit("message", { ...message, manualTrigger: true })
+        return
+      }
+    }
+    throw new Error(`Job with id ${id} not found`)
+  }
+
   on(event: string, callback: (...args: any[]) => void): Queue {
     // @ts-expect-error - this callback can be one of many types
     this._emitter.on(event, callback)
@@ -214,7 +244,9 @@ class InMemoryQueue implements Partial<Queue> {
   }
 
   async getRepeatableJobs() {
-    return this._messages.map(job => jobToJobInformation(job as Job))
+    return this._messages
+      .filter(job => job.opts?.repeat != null)
+      .map(job => jobToJobInformation(job as Job))
   }
 }
 

packages/backend-core/src/queue/index.ts

@@ -1,2 +1,3 @@
 export * from "./queue"
 export * from "./constants"
+export * from "./inMemoryQueue"

packages/bbui/src/Form/Core/Multiselect.svelte

@@ -14,7 +14,7 @@
   export let sort = false
   export let autoWidth = false
   export let searchTerm = null
-  export let customPopoverHeight
+  export let customPopoverHeight = undefined
   export let open = false
   export let loading
   export let onOptionMouseenter = () => {}

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -386,7 +386,7 @@
       editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
     } else if (editableColumn.type === FieldType.FORMULA) {
       editableColumn.formulaType = "dynamic"
-      editableColumn.responseType = field.responseType || FIELDS.STRING.type
+      editableColumn.responseType = field?.responseType || FIELDS.STRING.type
     }
   }
 

packages/builder/src/components/common/CodeEditor/CodeEditor.svelte

@@ -40,16 +40,19 @@
     indentMore,
     indentLess,
   } from "@codemirror/commands"
+  import { setDiagnostics } from "@codemirror/lint"
   import { Compartment, EditorState } from "@codemirror/state"
+  import type { Extension } from "@codemirror/state"
   import { javascript } from "@codemirror/lang-javascript"
   import { EditorModes } from "./"
   import { themeStore } from "@/stores/portal"
   import type { EditorMode } from "@budibase/types"
-  import type { BindingCompletion } from "@/types"
+  import type { BindingCompletion, CodeValidator } from "@/types"
+  import { validateHbsTemplate } from "./validator/hbs"
 
   export let label: string | undefined = undefined
-  // TODO: work out what best type fits this
   export let completions: BindingCompletion[] = []
+  export let validations: CodeValidator | null = null
   export let mode: EditorMode = EditorModes.Handlebars
   export let value: string | null = ""
   export let placeholder: string | null = null
@@ -248,7 +251,7 @@
   // None of this is reactive, but it never has been, so we just assume most
   // config flags aren't changed at runtime
   // TODO: work out type for base
-  const buildExtensions = (base: any[]) => {
+  const buildExtensions = (base: Extension[]) => {
     let complete = [...base]
 
     if (autocompleteEnabled) {
@@ -340,6 +343,24 @@
     return complete
   }
 
+  function validate(
+    value: string | null,
+    editor: EditorView | undefined,
+    mode: EditorMode,
+    validations: CodeValidator | null
+  ) {
+    if (!value || !validations || !editor) {
+      return
+    }
+
+    if (mode === EditorModes.Handlebars) {
+      const diagnostics = validateHbsTemplate(value, validations)
+      editor.dispatch(setDiagnostics(editor.state, diagnostics))
+    }
+  }
+
+  $: validate(value, editor, mode, validations)
+
   const initEditor = () => {
     const baseExtensions = buildBaseExtensions()
 
@@ -366,7 +387,6 @@
     <Label size="S">{label}</Label>
   </div>
 {/if}
-
 <div class={`code-editor ${mode?.name || ""}`}>
   <div tabindex="-1" bind:this={textarea} />
 </div>

packages/builder/src/components/common/CodeEditor/index.ts

@@ -83,6 +83,8 @@ const helpersToCompletion = (
     const helper = helpers[helperName]
     return {
       label: helperName,
+      args: helper.args,
+      requiresBlock: helper.requiresBlock,
       info: () => buildHelperInfoNode(helper),
       type: "helper",
       section: helperSection,
@@ -136,9 +138,13 @@ export const hbAutocomplete = (
   baseCompletions: BindingCompletionOption[]
 ): BindingCompletion => {
   function coreCompletion(context: CompletionContext) {
-    let bindingStart = context.matchBefore(EditorModes.Handlebars.match)
+    if (!baseCompletions.length) {
+      return null
+    }
 
-    let options = baseCompletions || []
+    const bindingStart = context.matchBefore(EditorModes.Handlebars.match)
+
+    const options = baseCompletions
 
     if (!bindingStart) {
       return null
@@ -149,7 +155,7 @@ export const hbAutocomplete = (
       return null
     }
     const query = bindingStart.text.replace(match[0], "")
-    let filtered = bindingFilter(options, query)
+    const filtered = bindingFilter(options, query)
 
     return {
       from: bindingStart.from + match[0].length,
@@ -169,8 +175,12 @@ export const jsAutocomplete = (
   baseCompletions: BindingCompletionOption[]
 ): BindingCompletion => {
   function coreCompletion(context: CompletionContext) {
-    let jsBinding = wrappedAutocompleteMatch(context)
-    let options = baseCompletions || []
+    if (!baseCompletions.length) {
+      return null
+    }
+
+    const jsBinding = wrappedAutocompleteMatch(context)
+    const options = baseCompletions
 
     if (jsBinding) {
       // Accommodate spaces
@@ -209,6 +219,10 @@ function setAutocomplete(
   options: BindingCompletionOption[]
 ): BindingCompletion {
   return function (context: CompletionContext) {
+    if (!options.length) {
+      return null
+    }
+
     if (wrappedAutocompleteMatch(context)) {
       return null
     }

packages/builder/src/components/common/CodeEditor/validator/hbs.ts

@@ -0,0 +1,113 @@
+/* global hbs */
+import Handlebars from "handlebars"
+import type { Diagnostic } from "@codemirror/lint"
+import { CodeValidator } from "@/types"
+
+function isMustacheStatement(
+  node: hbs.AST.Statement
+): node is hbs.AST.MustacheStatement {
+  return node.type === "MustacheStatement"
+}
+
+function isBlockStatement(
+  node: hbs.AST.Statement
+): node is hbs.AST.BlockStatement {
+  return node.type === "BlockStatement"
+}
+
+function isPathExpression(
+  node: hbs.AST.Statement
+): node is hbs.AST.PathExpression {
+  return node.type === "PathExpression"
+}
+
+export function validateHbsTemplate(
+  text: string,
+  validations: CodeValidator
+): Diagnostic[] {
+  const diagnostics: Diagnostic[] = []
+
+  try {
+    const ast = Handlebars.parse(text, {})
+
+    const lineOffsets: number[] = []
+    let offset = 0
+    for (const line of text.split("\n")) {
+      lineOffsets.push(offset)
+      offset += line.length + 1 // +1 for newline character
+    }
+
+    function traverseNodes(
+      nodes: hbs.AST.Statement[],
+      options?: {
+        ignoreMissing?: boolean
+      }
+    ) {
+      const ignoreMissing = options?.ignoreMissing || false
+      nodes.forEach(node => {
+        if (isMustacheStatement(node) && isPathExpression(node.path)) {
+          const helperName = node.path.original
+
+          const from =
+            lineOffsets[node.loc.start.line - 1] + node.loc.start.column
+          const to = lineOffsets[node.loc.end.line - 1] + node.loc.end.column
+
+          if (!(helperName in validations)) {
+            if (!ignoreMissing) {
+              diagnostics.push({
+                from,
+                to,
+                severity: "warning",
+                message: `"${helperName}" handler does not exist.`,
+              })
+            }
+            return
+          }
+
+          const { arguments: expectedArguments = [], requiresBlock } =
+            validations[helperName]
+
+          if (requiresBlock && !isBlockStatement(node)) {
+            diagnostics.push({
+              from,
+              to,
+              severity: "error",
+              message: `Helper "${helperName}" requires a body:\n{{#${helperName} ...}} [body] {{/${helperName}}}`,
+            })
+            return
+          }
+
+          const providedParams = node.params
+
+          if (providedParams.length !== expectedArguments.length) {
+            diagnostics.push({
+              from,
+              to,
+              severity: "error",
+              message: `Helper "${helperName}" expects ${
+                expectedArguments.length
+              } parameters (${expectedArguments.join(", ")}), but got ${
+                providedParams.length
+              }.`,
+            })
+          }
+        }
+
+        if (isBlockStatement(node)) {
+          traverseNodes(node.program.body, { ignoreMissing: true })
+        }
+      })
+    }
+
+    traverseNodes(ast.body, { ignoreMissing: true })
+  } catch (e: any) {
+    diagnostics.push({
+      from: 0,
+      to: text.length,
+      severity: "error",
+      message: `The handlebars code is not valid:\n${e.message}`,
+    })
+  }
+
+  return diagnostics
+}

packages/builder/src/components/common/CodeEditor/validator/tests/hbs.spec.ts

@@ -0,0 +1,142 @@
+import { validateHbsTemplate } from "../hbs"
+import { CodeValidator } from "@/types"
+
+describe("hbs validator", () => {
+  it("validate empty strings", () => {
+    const text = ""
+    const validators = {}
+
+    const result = validateHbsTemplate(text, validators)
+    expect(result).toHaveLength(0)
+  })
+
+  it("validate strings without hbs expressions", () => {
+    const text = "first line\nand another one"
+    const validators = {}
+
+    const result = validateHbsTemplate(text, validators)
+    expect(result).toHaveLength(0)
+  })
+
+  describe("basic expressions", () => {
+    const validators = {
+      fieldName: {},
+    }
+
+    it("validate valid expressions", () => {
+      const text = "{{ fieldName }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toHaveLength(0)
+    })
+
+    it("does not throw on missing validations", () => {
+      const text = "{{ anotherFieldName }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toHaveLength(0)
+    })
+
+    // Waiting for missing fields validation
+    it.skip("throws on untrimmed invalid expressions", () => {
+      const text = "    {{ anotherFieldName }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 4,
+          message: `"anotherFieldName" handler does not exist.`,
+          severity: "warning",
+          to: 26,
+        },
+      ])
+    })
+
+    // Waiting for missing fields validation
+    it.skip("throws on invalid expressions between valid lines", () => {
+      const text =
+        "literal expression\nthe value is {{ anotherFieldName }}\nanother expression"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 32,
+          message: `"anotherFieldName" handler does not exist.`,
+          severity: "warning",
+          to: 54,
+        },
+      ])
+    })
+
+    describe("expressions with whitespaces", () => {
+      const validators = {
+        [`field name`]: {},
+      }
+
+      it("validates expressions with whitespaces", () => {
+        const text = `{{ [field name] }}`
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toHaveLength(0)
+      })
+
+      // Waiting for missing fields validation
+      it.skip("throws if not wrapped between brackets", () => {
+        const text = `{{ field name }}`
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toEqual([
+          {
+            from: 0,
+            message: `"field" handler does not exist.`,
+            severity: "warning",
+            to: 16,
+          },
+        ])
+      })
+    })
+  })
+
+  describe("expressions with parameters", () => {
+    const validators: CodeValidator = {
+      helperFunction: {
+        arguments: ["a", "b", "c"],
+      },
+    }
+
+    it("validate valid params", () => {
+      const text = "{{ helperFunction 1 99 'a' }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toHaveLength(0)
+    })
+
+    it("throws on too few params", () => {
+      const text = "{{ helperFunction 100 }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 0,
+          message: `Helper "helperFunction" expects 3 parameters (a, b, c), but got 1.`,
+          severity: "error",
+          to: 24,
+        },
+      ])
+    })
+
+    it("throws on too many params", () => {
+      const text = "{{ helperFunction  1 99 'a' 100 }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 0,
+          message: `Helper "helperFunction" expects 3 parameters (a, b, c), but got 4.`,
+          severity: "error",
+          to: 34,
+        },
+      ])
+    })
+  })
+})

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -42,7 +42,7 @@
     JSONValue,
   } from "@budibase/types"
   import type { Log } from "@budibase/string-templates"
-  import type { BindingCompletion, BindingCompletionOption } from "@/types"
+  import type { CodeValidator } from "@/types"
 
   const dispatch = createEventDispatcher()
 
@@ -58,7 +58,7 @@
   export let placeholder = null
   export let showTabBar = true
 
-  let mode: BindingMode | null
+  let mode: BindingMode
   let sidePanel: SidePanel | null
   let initialValueJS = value?.startsWith?.("{{ js ")
   let jsValue: string | null = initialValueJS ? value : null
@@ -88,13 +88,37 @@
     | null
   $: runtimeExpression = readableToRuntimeBinding(enrichedBindings, value)
   $: requestEval(runtimeExpression, context, snippets)
-  $: bindingCompletions = bindingsToCompletions(enrichedBindings, editorMode)
   $: bindingHelpers = new BindingHelpers(getCaretPosition, insertAtPos)
-  $: hbsCompletions = getHBSCompletions(bindingCompletions)
-  $: jsCompletions = getJSCompletions(bindingCompletions, snippets, {
-    useHelpers: allowHelpers,
-    useSnippets,
-  })
+
+  $: bindingOptions = bindingsToCompletions(bindings, editorMode)
+  $: helperOptions = allowHelpers ? getHelperCompletions(editorMode) : []
+  $: snippetsOptions =
+    usingJS && useSnippets && snippets?.length ? snippets : []
+
+  $: completions = !usingJS
+    ? [hbAutocomplete([...bindingOptions, ...helperOptions])]
+    : [
+        jsAutocomplete(bindingOptions),
+        jsHelperAutocomplete(helperOptions),
+        snippetAutoComplete(snippetsOptions),
+      ]
+
+  $: validations = {
+    ...bindingOptions.reduce<CodeValidator>((validations, option) => {
+      validations[option.label] = {
+        arguments: [],
+      }
+      return validations
+    }, {}),
+    ...helperOptions.reduce<CodeValidator>((validations, option) => {
+      validations[option.label] = {
+        arguments: option.args,
+        requiresBlock: option.requiresBlock,
+      }
+      return validations
+    }, {}),
+  }
+
   $: {
     // Ensure a valid side panel option is always selected
     if (sidePanel && !sidePanelOptions.includes(sidePanel)) {
@@ -102,38 +126,6 @@
     }
   }
 
-  const getHBSCompletions = (bindingCompletions: BindingCompletionOption[]) => {
-    return [
-      hbAutocomplete([
-        ...bindingCompletions,
-        ...getHelperCompletions(EditorModes.Handlebars),
-      ]),
-    ]
-  }
-
-  const getJSCompletions = (
-    bindingCompletions: BindingCompletionOption[],
-    snippets: Snippet[] | null,
-    config: {
-      useHelpers: boolean
-      useSnippets: boolean
-    }
-  ) => {
-    const completions: BindingCompletion[] = []
-    if (bindingCompletions.length) {
-      completions.push(jsAutocomplete([...bindingCompletions]))
-    }
-    if (config.useHelpers) {
-      completions.push(
-        jsHelperAutocomplete([...getHelperCompletions(EditorModes.JS)])
-      )
-    }
-    if (config.useSnippets && snippets) {
-      completions.push(snippetAutoComplete(snippets))
-    }
-    return completions
-  }
-
   const getModeOptions = (allowHBS: boolean, allowJS: boolean) => {
     let options = []
     if (allowHBS) {
@@ -213,7 +205,7 @@
     bindings: EnrichedBinding[],
     context: any,
     snippets: Snippet[] | null
-  ) => {
+  ): EnrichedBinding[] => {
     // Create a single big array to enrich in one go
     const bindingStrings = bindings.map(binding => {
       if (binding.runtimeBinding.startsWith('trim "')) {
@@ -290,7 +282,7 @@
     jsValue = null
     hbsValue = null
     updateValue(null)
-    mode = targetMode
+    mode = targetMode!
     targetMode = null
   }
 
@@ -365,13 +357,14 @@
       {/if}
       <div class="editor">
         {#if mode === BindingMode.Text}
-          {#key hbsCompletions}
+          {#key completions}
             <CodeEditor
               value={hbsValue}
               on:change={onChangeHBSValue}
               bind:getCaretPosition
               bind:insertAtPos
-              completions={hbsCompletions}
+              {completions}
+              {validations}
               autofocus={autofocusEditor}
               placeholder={placeholder ||
                 "Add bindings by typing {{ or use the menu on the right"}
@@ -379,18 +372,18 @@
             />
           {/key}
         {:else if mode === BindingMode.JavaScript}
-          {#key jsCompletions}
+          {#key completions}
             <CodeEditor
               value={jsValue ? decodeJSBinding(jsValue) : jsValue}
               on:change={onChangeJSValue}
-              completions={jsCompletions}
+              {completions}
               mode={EditorModes.JS}
               bind:getCaretPosition
               bind:insertAtPos
               autofocus={autofocusEditor}
               placeholder={placeholder ||
                 "Add bindings by typing $ or use the menu on the right"}
-              jsBindingWrapping={bindingCompletions.length > 0}
+              jsBindingWrapping={completions.length > 0}
             />
           {/key}
         {/if}

packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte

@@ -43,7 +43,6 @@
     <EditComponentPopover
       {anchor}
       componentInstance={item}
-      {componentBindings}
       {bindings}
       on:change
       parseSettings={updatedNestedFlags}

packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte

@@ -1,13 +1,13 @@
 <script>
   import { Icon, Popover, Layout } from "@budibase/bbui"
-  import { componentStore } from "@/stores/builder"
+  import { componentStore, selectedScreen } from "@/stores/builder"
   import { cloneDeep } from "lodash/fp"
   import { createEventDispatcher, getContext } from "svelte"
   import ComponentSettingsSection from "@/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte"
+  import { getComponentBindableProperties } from "@/dataBinding"
 
   export let anchor
   export let componentInstance
-  export let componentBindings
   export let bindings
   export let parseSettings
 
@@ -28,6 +28,10 @@
   }
   $: componentDef = componentStore.getDefinition(componentInstance._component)
   $: parsedComponentDef = processComponentDefinitionSettings(componentDef)
+  $: componentBindings = getComponentBindableProperties(
+    $selectedScreen,
+    $componentStore.selectedComponentId
+  )
 
   const open = () => {
     isOpen = true

packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte

@@ -45,7 +45,6 @@
     <EditComponentPopover
       {anchor}
       componentInstance={item}
-      {componentBindings}
       {bindings}
       {parseSettings}
       on:change

packages/builder/src/components/design/settings/controls/PropertyControl.svelte

@@ -22,25 +22,59 @@
   export let propertyFocus = false
   export let info = null
   export let disableBindings = false
-  export let wide
+  export let wide = false
+  export let contextAccess = null
 
   let highlightType
   let domElement
 
   $: highlightedProp = $builderStore.highlightedSetting
-  $: allBindings = getAllBindings(bindings, componentBindings, nested)
+  $: allBindings = getAllBindings(
+    bindings,
+    componentBindings,
+    nested,
+    contextAccess
+  )
   $: safeValue = getSafeValue(value, defaultValue, allBindings)
   $: replaceBindings = val => readableToRuntimeBinding(allBindings, val)
-
   $: isHighlighted = highlightedProp?.key === key
-
   $: highlightType = isHighlighted ? `highlighted-${highlightedProp?.type}` : ""
+  $: highlightedProp && isHighlighted && scrollToElement(domElement)
 
-  const getAllBindings = (bindings, componentBindings, nested) => {
-    if (!nested) {
+  const getAllBindings = (
+    bindings,
+    componentBindings,
+    nested,
+    contextAccess
+  ) => {
+    // contextAccess is a bit of an escape hatch to get around how we render
+    // certain settings types by using a pseudo component definition, leading
+    // to problems with the nested flag
+    if (contextAccess != null) {
+      // Optionally include global bindings
+      let allBindings = contextAccess.global ? bindings : []
+
+      // Optionally include or exclude self (component) bindings.
+      // If this is a nested setting then we will already have our own context
+      // bindings mixed in, so if we don't want self context we need to filter
+      // them out.
+      if (contextAccess.self) {
+        return [...allBindings, ...componentBindings]
+      } else {
+        return allBindings.filter(binding => {
+          return !componentBindings.some(componentBinding => {
+            return componentBinding.runtimeBinding === binding.runtimeBinding
+          })
+        })
+      }
+    }
+
+    // Otherwise just honour the normal nested flag
+    if (nested) {
+      return [...bindings, ...componentBindings]
+    } else {
       return bindings
     }
-    return [...(componentBindings || []), ...(bindings || [])]
   }
 
   // Handle a value change of any type
@@ -81,8 +115,6 @@
       block: "center",
     })
   }
-
-  $: highlightedProp && isHighlighted && scrollToElement(domElement)
 </script>
 
 <div

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte

@@ -147,6 +147,7 @@
           {componentInstance}
           {componentDefinition}
           {bindings}
+          {componentBindings}
         />
       {/if}
     </Panel>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte

@@ -151,6 +151,7 @@
               propertyFocus={$builderStore.propertyFocus === setting.key}
               info={setting.info}
               disableBindings={setting.disableBindings}
+              contextAccess={setting.contextAccess}
               props={{
                 // Generic settings
                 placeholder: setting.placeholder || null,

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUIDrawer.svelte

@@ -19,6 +19,7 @@
 
   export let conditions = []
   export let bindings = []
+  export let componentBindings = []
 
   const flipDurationMs = 150
   const actionOptions = [
@@ -55,6 +56,7 @@
   ]
 
   let dragDisabled = true
+
   $: settings = componentStore
     .getComponentSettings($selectedComponent?._component)
     ?.concat({
@@ -213,7 +215,10 @@
                       options: definition.options,
                       placeholder: definition.placeholder,
                     }}
+                    nested={definition.nested}
+                    contextAccess={definition.contextAccess}
                     {bindings}
+                    {componentBindings}
                   />
                 {:else}
                   <Select disabled placeholder=" " />

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUISection.svelte

@@ -64,7 +64,12 @@
     Show, hide and update components in response to conditions being met.
   </svelte:fragment>
   <Button cta slot="buttons" on:click={() => save()}>Save</Button>
-  <ConditionalUIDrawer slot="body" bind:conditions={tempValue} {bindings} />
+  <ConditionalUIDrawer
+    slot="body"
+    bind:conditions={tempValue}
+    {bindings}
+    {componentBindings}
+  />
 </Drawer>
 
 <style>

packages/builder/src/types/bindings.ts

@@ -5,4 +5,15 @@ export type BindingCompletion = (context: CompletionContext) => {
   options: Completion[]
 } | null
 
-export type BindingCompletionOption = Completion
+export interface BindingCompletionOption extends Completion {
+  args?: any[]
+  requiresBlock?: boolean
+}
+
+export type CodeValidator = Record<
+  string,
+  {
+    arguments?: any[]
+    requiresBlock?: boolean
+  }
+>

packages/builder/tsconfig.json

@@ -14,5 +14,6 @@
       "assets/*": ["assets/*"],
       "@/*": ["src/*"]
     }
-  }
+  },
+  "exclude": []
 }

packages/client/manifest.json

@@ -3089,7 +3089,21 @@
       {
         "type": "tableConditions",
         "label": "Conditions",
-        "key": "conditions"
+        "key": "conditions",
+        "contextAccess": {
+          "global": true,
+          "self": false
+        }
+      },
+      {
+        "type": "text",
+        "label": "Format",
+        "key": "format",
+        "info": "Changing format will display values as text",
+        "contextAccess": {
+          "global": false,
+          "self": true
+        }
       }
     ]
   },
@@ -7686,7 +7700,8 @@
           {
             "type": "columns/grid",
             "key": "columns",
-            "resetOn": "table"
+            "resetOn": "table",
+            "nested": true
           }
         ]
       },

packages/client/src/components/app/GridBlock.svelte

@@ -5,7 +5,7 @@
   import { get, derived, readable } from "svelte/store"
   import { featuresStore } from "@/stores"
   import { Grid } from "@budibase/frontend-core"
-  // import { processStringSync } from "@budibase/string-templates"
+  import { processStringSync } from "@budibase/string-templates"
 
   // table is actually any datasource, but called table for legacy compatibility
   export let table
@@ -47,8 +47,8 @@
   $: currentTheme = $context?.device?.theme
   $: darkMode = !currentTheme?.includes("light")
   $: parsedColumns = getParsedColumns(columns)
-  $: schemaOverrides = getSchemaOverrides(parsedColumns)
   $: enrichedButtons = enrichButtons(buttons)
+  $: schemaOverrides = getSchemaOverrides(parsedColumns, $context)
   $: selectedRows = deriveSelectedRows(gridContext)
   $: styles = patchStyles($component.styles, minHeight)
   $: data = { selectedRows: $selectedRows }
@@ -97,15 +97,19 @@
     }))
   }
 
-  const getSchemaOverrides = columns => {
+  const getSchemaOverrides = (columns, context) => {
     let overrides = {}
     columns.forEach((column, idx) => {
       overrides[column.field] = {
         displayName: column.label,
         order: idx,
-        conditions: column.conditions,
         visible: !!column.active,
-        // format: createFormatter(column),
+        conditions: enrichConditions(column.conditions, context),
+        format: createFormatter(column),
+
+        // Small hack to ensure we react to all changes, as our
+        // memoization cannot compare differences in functions
+        rand: column.conditions?.length ? Math.random() : null,
       }
       if (column.width) {
         overrides[column.field].width = column.width
@@ -114,12 +118,24 @@
     return overrides
   }
 
-  // const createFormatter = column => {
-  //   if (typeof column.format !== "string" || !column.format.trim().length) {
-  //     return null
-  //   }
-  //   return row => processStringSync(column.format, { [id]: row })
-  // }
+  const enrichConditions = (conditions, context) => {
+    return conditions?.map(condition => {
+      return {
+        ...condition,
+        referenceValue: processStringSync(
+          condition.referenceValue || "",
+          context
+        ),
+      }
+    })
+  }
+
+  const createFormatter = column => {
+    if (typeof column.format !== "string" || !column.format.trim().length) {
+      return null
+    }
+    return row => processStringSync(column.format, { [id]: row })
+  }
 
   const enrichButtons = buttons => {
     if (!buttons?.length) {

packages/client/src/components/app/Placeholder.svelte

@@ -5,7 +5,7 @@
   const component = getContext("component")
   const block = getContext("block")
 
-  export let text
+  export let text = undefined
 </script>
 
 {#if $builderStore.inBuilder}

packages/client/src/components/app/forms/Field.svelte

@@ -1,4 +1,4 @@
-<script>
+<script lang="ts">
   import { getContext, onDestroy } from "svelte"
   import { writable } from "svelte/store"
   import { Icon } from "@budibase/bbui"
@@ -6,33 +6,33 @@
   import Placeholder from "../Placeholder.svelte"
   import InnerForm from "./InnerForm.svelte"
 
-  export let label
-  export let field
-  export let fieldState
-  export let fieldApi
-  export let fieldSchema
-  export let defaultValue
-  export let type
+  export let label: string | undefined = undefined
+  export let field: string | undefined = undefined
+  export let fieldState: any
+  export let fieldApi: any
+  export let fieldSchema: any
+  export let defaultValue: string | undefined = undefined
+  export let type: any
   export let disabled = false
   export let readonly = false
-  export let validation
+  export let validation: any
   export let span = 6
-  export let helpText = null
+  export let helpText: string | undefined = undefined
 
   // Get contexts
-  const formContext = getContext("form")
-  const formStepContext = getContext("form-step")
-  const fieldGroupContext = getContext("field-group")
+  const formContext: any = getContext("form")
+  const formStepContext: any = getContext("form-step")
+  const fieldGroupContext: any = getContext("field-group")
   const { styleable, builderStore, Provider } = getContext("sdk")
-  const component = getContext("component")
+  const component: any = getContext("component")
 
   // Register field with form
   const formApi = formContext?.formApi
   const labelPos = fieldGroupContext?.labelPosition || "above"
 
-  let formField
+  let formField: any
   let touched = false
-  let labelNode
+  let labelNode: any
 
   // Memoize values required to register the field to avoid loops
   const formStep = formStepContext || writable(1)
@@ -65,7 +65,7 @@
   $: $component.editing && labelNode?.focus()
 
   // Update form properties in parent component on every store change
-  $: unsubscribe = formField?.subscribe(value => {
+  $: unsubscribe = formField?.subscribe((value: any) => {
     fieldState = value?.fieldState
     fieldApi = value?.fieldApi
     fieldSchema = value?.fieldSchema
@@ -74,7 +74,7 @@
   // Determine label class from position
   $: labelClass = labelPos === "above" ? "" : `spectrum-FieldLabel--${labelPos}`
 
-  const registerField = info => {
+  const registerField = (info: any) => {
     formField = formApi?.registerField(
       info.field,
       info.type,
@@ -86,8 +86,9 @@
     )
   }
 
-  const updateLabel = e => {
+  const updateLabel = (e: any) => {
     if (touched) {
+      // @ts-expect-error and TODO updateProp isn't recognised - need builder TS conversion
       builderStore.actions.updateProp("label", e.target.textContent)
     }
     touched = false

packages/client/src/components/app/forms/InnerForm.svelte

@@ -4,13 +4,13 @@
   import { createValidatorFromConstraints } from "./validation"
   import { Helpers } from "@budibase/bbui"
 
-  export let dataSource
+  export let dataSource = undefined
   export let disabled = false
   export let readonly = false
-  export let initialValues
-  export let size
-  export let schema
-  export let definition
+  export let initialValues = undefined
+  export let size = undefined
+  export let schema = undefined
+  export let definition = undefined
   export let disableSchemaValidation = false
   export let editAutoColumns = false
 

packages/client/src/components/app/forms/RelationshipField.svelte

@@ -1,43 +1,61 @@
-<script>
+<script lang="ts">
   import { CoreSelect, CoreMultiselect } from "@budibase/bbui"
   import { FieldType } from "@budibase/types"
   import { fetchData, Utils } from "@budibase/frontend-core"
   import { getContext } from "svelte"
   import Field from "./Field.svelte"
+  import type {
+    SearchFilter,
+    RelationshipFieldMetadata,
+    Table,
+    Row,
+  } from "@budibase/types"
 
   const { API } = getContext("sdk")
 
-  export let field
-  export let label
-  export let placeholder
-  export let disabled = false
-  export let readonly = false
-  export let validation
-  export let autocomplete = true
-  export let defaultValue
-  export let onChange
-  export let filter
-  export let datasourceType = "table"
-  export let primaryDisplay
-  export let span
-  export let helpText = null
-  export let type = FieldType.LINK
+  export let field: string | undefined = undefined
+  export let label: string | undefined = undefined
+  export let placeholder: any = undefined
+  export let disabled: boolean = false
+  export let readonly: boolean = false
+  export let validation: any
+  export let autocomplete: boolean = true
+  export let defaultValue: string | undefined = undefined
+  export let onChange: any
+  export let filter: SearchFilter[]
+  export let datasourceType: "table" | "user" | "groupUser" = "table"
+  export let primaryDisplay: string | undefined = undefined
+  export let span: number | undefined = undefined
+  export let helpText: string | undefined = undefined
+  export let type:
+    | FieldType.LINK
+    | FieldType.BB_REFERENCE
+    | FieldType.BB_REFERENCE_SINGLE = FieldType.LINK
 
-  let fieldState
-  let fieldApi
-  let fieldSchema
-  let tableDefinition
-  let searchTerm
-  let open
+  type RelationshipValue = { _id: string; [key: string]: any }
+  type OptionObj = Record<string, RelationshipValue>
+  type OptionsObjType = Record<string, OptionObj>
 
+  let fieldState: any
+  let fieldApi: any
+  let fieldSchema: RelationshipFieldMetadata | undefined
+  let tableDefinition: Table | null | undefined
+  let searchTerm: any
+  let open: boolean
+  let selectedValue: string[] | string
+
+  // need a cast version of this for reactivity, components below aren't typed
+  $: castSelectedValue = selectedValue as any
   $: multiselect =
     [FieldType.LINK, FieldType.BB_REFERENCE].includes(type) &&
     fieldSchema?.relationshipType !== "one-to-many"
-  $: linkedTableId = fieldSchema?.tableId
+  $: linkedTableId = fieldSchema?.tableId!
   $: fetch = fetchData({
     API,
     datasource: {
-      type: datasourceType,
+      // typing here doesn't seem correct - we have the correct datasourceType options
+      // but when we configure the fetchData, it seems to think only "table" is valid
+      type: datasourceType as any,
       tableId: linkedTableId,
     },
     options: {
@@ -53,7 +71,8 @@
   $: component = multiselect ? CoreMultiselect : CoreSelect
   $: primaryDisplay = primaryDisplay || tableDefinition?.primaryDisplay
 
-  let optionsObj
+  let optionsObj: OptionsObjType = {}
+  const debouncedFetchRows = Utils.debounce(fetchRows, 250)
 
   $: {
     if (primaryDisplay && fieldState && !optionsObj) {
@@ -63,27 +82,33 @@
       if (!Array.isArray(valueAsSafeArray)) {
         valueAsSafeArray = [fieldState.value]
       }
-      optionsObj = valueAsSafeArray.reduce((accumulator, value) => {
-        // fieldState has to be an array of strings to be valid for an update
-        // therefore we cannot guarantee value will be an object
-        // https://linear.app/budibase/issue/BUDI-7577/refactor-the-relationshipfield-component-to-have-better-support-for
-        if (!value._id) {
+      optionsObj = valueAsSafeArray.reduce(
+        (
+          accumulator: OptionObj,
+          value: { _id: string; primaryDisplay: any }
+        ) => {
+          // fieldState has to be an array of strings to be valid for an update
+          // therefore we cannot guarantee value will be an object
+          // https://linear.app/budibase/issue/BUDI-7577/refactor-the-relationshipfield-component-to-have-better-support-for
+          if (!value._id) {
+            return accumulator
+          }
+          accumulator[value._id] = {
+            _id: value._id,
+            [primaryDisplay]: value.primaryDisplay,
+          }
           return accumulator
-        }
-        accumulator[value._id] = {
-          _id: value._id,
-          [primaryDisplay]: value.primaryDisplay,
-        }
-        return accumulator
-      }, {})
+        },
+        {}
+      )
     }
   }
 
   $: enrichedOptions = enrichOptions(optionsObj, $fetch.rows)
-  const enrichOptions = (optionsObj, fetchResults) => {
+  const enrichOptions = (optionsObj: OptionsObjType, fetchResults: Row[]) => {
     const result = (fetchResults || [])?.reduce((accumulator, row) => {
-      if (!accumulator[row._id]) {
-        accumulator[row._id] = row
+      if (!accumulator[row._id!]) {
+        accumulator[row._id!] = row
       }
       return accumulator
     }, optionsObj || {})
@@ -92,24 +117,32 @@
   }
   $: {
     // We don't want to reorder while the dropdown is open, to avoid UX jumps
-    if (!open) {
-      enrichedOptions = enrichedOptions.sort((a, b) => {
+    if (!open && primaryDisplay) {
+      enrichedOptions = enrichedOptions.sort((a: OptionObj, b: OptionObj) => {
         const selectedValues = flatten(fieldState?.value) || []
 
-        const aIsSelected = selectedValues.find(v => v === a._id)
-        const bIsSelected = selectedValues.find(v => v === b._id)
+        const aIsSelected = selectedValues.find(
+          (v: RelationshipValue) => v === a._id
+        )
+        const bIsSelected = selectedValues.find(
+          (v: RelationshipValue) => v === b._id
+        )
         if (aIsSelected && !bIsSelected) {
           return -1
         } else if (!aIsSelected && bIsSelected) {
           return 1
         }
 
-        return a[primaryDisplay] > b[primaryDisplay]
+        return (a[primaryDisplay] > b[primaryDisplay]) as unknown as number
       })
     }
   }
 
-  $: forceFetchRows(filter)
+  $: {
+    if (filter || defaultValue) {
+      forceFetchRows()
+    }
+  }
   $: debouncedFetchRows(searchTerm, primaryDisplay, defaultValue)
 
   const forceFetchRows = async () => {
@@ -119,7 +152,11 @@
     selectedValue = []
     debouncedFetchRows(searchTerm, primaryDisplay, defaultValue)
   }
-  const fetchRows = async (searchTerm, primaryDisplay, defaultVal) => {
+  async function fetchRows(
+    searchTerm: any,
+    primaryDisplay: string,
+    defaultVal: string | string[]
+  ) {
     const allRowsFetched =
       $fetch.loaded &&
       !Object.keys($fetch.query?.string || {}).length &&
@@ -129,17 +166,39 @@
       return
     }
     // must be an array
-    if (defaultVal && !Array.isArray(defaultVal)) {
-      defaultVal = defaultVal.split(",")
-    }
-    if (defaultVal && optionsObj && defaultVal.some(val => !optionsObj[val])) {
+    const defaultValArray: string[] = !defaultVal
+      ? []
+      : !Array.isArray(defaultVal)
+      ? defaultVal.split(",")
+      : defaultVal
+
+    if (
+      defaultVal &&
+      optionsObj &&
+      defaultValArray.some(val => !optionsObj[val])
+    ) {
       await fetch.update({
-        query: { oneOf: { _id: defaultVal } },
+        query: { oneOf: { _id: defaultValArray } },
+      })
+    }
+
+    if (
+      (Array.isArray(selectedValue) &&
+        selectedValue.some(val => !optionsObj[val])) ||
+      (selectedValue && !optionsObj[selectedValue as string])
+    ) {
+      await fetch.update({
+        query: {
+          oneOf: {
+            _id: Array.isArray(selectedValue) ? selectedValue : [selectedValue],
+          },
+        },
       })
     }
 
     // Ensure we match all filters, rather than any
-    const baseFilter = (filter || []).filter(x => x.operator !== "allOr")
+    // @ts-expect-error this doesn't fit types, but don't want to change it yet
+    const baseFilter: any = (filter || []).filter(x => x.operator !== "allOr")
     await fetch.update({
       filter: [
         ...baseFilter,
@@ -152,9 +211,8 @@
       ],
     })
   }
-  const debouncedFetchRows = Utils.debounce(fetchRows, 250)
 
-  const flatten = values => {
+  const flatten = (values: any | any[]) => {
     if (!values) {
       return []
     }
@@ -162,17 +220,17 @@
     if (!Array.isArray(values)) {
       values = [values]
     }
-    values = values.map(value =>
+    values = values.map((value: any) =>
       typeof value === "object" ? value._id : value
     )
     return values
   }
 
-  const getDisplayName = row => {
-    return row?.[primaryDisplay] || "-"
+  const getDisplayName = (row: Row) => {
+    return row?.[primaryDisplay!] || "-"
   }
 
-  const handleChange = e => {
+  const handleChange = (e: any) => {
     let value = e.detail
     if (!multiselect) {
       value = value == null ? [] : [value]
@@ -220,13 +278,12 @@
       this={component}
       options={enrichedOptions}
       {autocomplete}
-      value={selectedValue}
+      value={castSelectedValue}
       on:change={handleChange}
       on:loadMore={loadMore}
       id={fieldState.fieldId}
       disabled={fieldState.disabled}
       readonly={fieldState.readonly}
-      error={fieldState.error}
       getOptionLabel={getDisplayName}
       getOptionValue={option => option._id}
       {placeholder}

packages/frontend-core/src/components/grid/stores/conditions.ts

@@ -31,7 +31,7 @@ export const deriveStores = (context: StoreContext): ConditionDerivedStore => {
   // Derive and memoize the cell conditions present in our columns so that we
   // only recompute condition metadata when absolutely necessary
   const conditions = derivedMemo(columns, $columns => {
-    let newConditions = []
+    let newConditions: UICondition[] = []
     for (let column of $columns) {
       for (let condition of column.conditions || []) {
         newConditions.push({

packages/server/src/automations/logging/index.ts

@@ -1,7 +1,7 @@
 import env from "../../environment"
 import { AutomationResults, Automation, App } from "@budibase/types"
 import { automations } from "@budibase/pro"
-import { db as dbUtils } from "@budibase/backend-core"
+import { db as dbUtils, logging } from "@budibase/backend-core"
 import sizeof from "object-sizeof"
 
 const MAX_LOG_SIZE_MB = 5
@@ -32,7 +32,16 @@ export async function storeLog(
   if (bytes / MB_IN_BYTES > MAX_LOG_SIZE_MB) {
     sanitiseResults(results)
   }
-  await automations.logs.storeLog(automation, results)
+  try {
+    await automations.logs.storeLog(automation, results)
+  } catch (e: any) {
+    if (e.status === 413 && e.request?.data) {
+      // if content is too large we shouldn't log it
+      delete e.request.data
+      e.request.data = { message: "removed due to large size" }
+    }
+    logging.logAlert("Error writing automation log", e)
+  }
 }
 
 export async function checkAppMetadata(apps: App[]) {

packages/server/src/automations/tests/steps/cron-automations.spec.ts

@@ -1,45 +0,0 @@
-import tk from "timekeeper"
-import "../../../environment"
-import * as automations from "../../index"
-import TestConfiguration from "../../../tests/utilities/TestConfiguration"
-import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
-
-const initialTime = Date.now()
-tk.freeze(initialTime)
-
-const oneMinuteInMs = 60 * 1000
-
-describe("cron automations", () => {
-  const config = new TestConfiguration()
-
-  beforeAll(async () => {
-    await automations.init()
-    await config.init()
-  })
-
-  afterAll(async () => {
-    await automations.shutdown()
-    config.end()
-  })
-
-  beforeEach(() => {
-    tk.freeze(initialTime)
-  })
-
-  it("should initialise the automation timestamp", async () => {
-    await createAutomationBuilder(config).onCron({ cron: "* * * * *" }).save()
-
-    tk.travel(Date.now() + oneMinuteInMs)
-    await config.publish()
-
-    const { data } = await config.getAutomationLogs()
-    expect(data).toHaveLength(1)
-    expect(data).toEqual([
-      expect.objectContaining({
-        trigger: expect.objectContaining({
-          outputs: { timestamp: initialTime + oneMinuteInMs },
-        }),
-      }),
-    ])
-  })
-})

packages/server/src/automations/tests/steps/loop.spec.ts

@@ -21,6 +21,11 @@ describe("Attempt to run a basic loop automation", () => {
   })
 
   beforeEach(async () => {
+    const { automations } = await config.api.automation.fetch()
+    for (const automation of automations) {
+      await config.api.automation.delete(automation)
+    }
+
     table = await config.api.table.save(basicTable())
     await config.api.row.save(table._id!, {})
   })

packages/server/src/automations/tests/triggers/cron.spec.ts

@@ -1,6 +1,15 @@
 import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
-import { captureAutomationResults } from "../utilities"
+import {
+  captureAutomationMessages,
+  captureAutomationRemovals,
+  captureAutomationResults,
+  triggerCron,
+} from "../utilities"
+import { automations } from "@budibase/pro"
+import { AutomationData, AutomationStatus } from "@budibase/types"
+import { MAX_AUTOMATION_RECURRING_ERRORS } from "../../../constants"
+import { queue } from "@budibase/backend-core"
 
 describe("cron trigger", () => {
   const config = new TestConfiguration()
@@ -13,6 +22,13 @@ describe("cron trigger", () => {
     config.end()
   })
 
+  beforeEach(async () => {
+    const { automations } = await config.api.automation.fetch()
+    for (const automation of automations) {
+      await config.api.automation.delete(automation)
+    }
+  })
+
   it("should queue a Bull cron job", async () => {
     const { automation } = await createAutomationBuilder(config)
       .onCron({ cron: "* * * * *" })
@@ -21,12 +37,12 @@ describe("cron trigger", () => {
       })
       .save()
 
-    const jobs = await captureAutomationResults(automation, () =>
+    const messages = await captureAutomationMessages(automation, () =>
       config.api.application.publish()
     )
-    expect(jobs).toHaveLength(1)
+    expect(messages).toHaveLength(1)
 
-    const repeat = jobs[0].opts?.repeat
+    const repeat = messages[0].opts?.repeat
     if (!repeat || !("cron" in repeat)) {
       throw new Error("Expected cron repeat")
     }
@@ -49,4 +65,85 @@ describe("cron trigger", () => {
       },
     })
   })
+
+  it("should stop if the job fails more than 3 times", async () => {
+    const { automation } = await createAutomationBuilder(config)
+      .onCron({ cron: "* * * * *" })
+      .queryRows({
+        // @ts-expect-error intentionally sending invalid data
+        tableId: null,
+      })
+      .save()
+
+    const [message] = await captureAutomationMessages(automation, () =>
+      config.api.application.publish()
+    )
+
+    await config.withProdApp(async () => {
+      let results: queue.TestQueueMessage<AutomationData>[] = []
+      const removed = await captureAutomationRemovals(automation, async () => {
+        results = await captureAutomationResults(automation, async () => {
+          for (let i = 0; i < MAX_AUTOMATION_RECURRING_ERRORS; i++) {
+            triggerCron(message)
+          }
+        })
+      })
+
+      expect(removed).toHaveLength(1)
+      expect(removed[0].id).toEqual(message.id)
+
+      expect(results).toHaveLength(5)
+
+      const search = await automations.logs.logSearch({
+        automationId: automation._id,
+        status: AutomationStatus.STOPPED_ERROR,
+      })
+      expect(search.data).toHaveLength(1)
+      expect(search.data[0].status).toEqual(AutomationStatus.STOPPED_ERROR)
+    })
+  })
+
+  it("should fill in the timestamp if one is not provided", async () => {
+    const runner = await createAutomationBuilder(config)
+      .onCron({ cron: "* * * * *" })
+      .serverLog({
+        text: "Hello, world!",
+      })
+      .save()
+
+    await config.api.application.publish()
+
+    const results = await captureAutomationResults(
+      runner.automation,
+      async () => {
+        await runner.trigger({ timeout: 1000, fields: {} })
+      }
+    )
+    expect(results).toHaveLength(1)
+    expect(results[0].data.event.timestamp).toBeWithin(
+      Date.now() - 1000,
+      Date.now() + 1000
+    )
+  })
+
+  it("should use the given timestamp if one is given", async () => {
+    const timestamp = 1234
+    const runner = await createAutomationBuilder(config)
+      .onCron({ cron: "* * * * *" })
+      .serverLog({
+        text: "Hello, world!",
+      })
+      .save()
+
+    await config.api.application.publish()
+
+    const results = await captureAutomationResults(
+      runner.automation,
+      async () => {
+        await runner.trigger({ timeout: 1000, fields: {}, timestamp })
+      }
+    )
+    expect(results).toHaveLength(1)
+    expect(results[0].data.event.timestamp).toEqual(timestamp)
+  })
 })

packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts

@@ -220,10 +220,34 @@ class AutomationRunner<TStep extends AutomationTriggerStepId> {
   async trigger(
     request: TriggerAutomationRequest
   ): Promise<TriggerAutomationResponse> {
-    return await this.config.api.automation.trigger(
-      this.automation._id!,
-      request
-    )
+    if (!this.config.prodAppId) {
+      throw new Error(
+        "Automations can only be triggered in a production app context, call config.api.application.publish()"
+      )
+    }
+    // Because you can only trigger automations in a production app context, we
+    // wrap the trigger call to make tests a bit cleaner. If you really want to
+    // test triggering an automation in a dev app context, you can use the
+    // automation API directly.
+    return await this.config.withProdApp(async () => {
+      try {
+        return await this.config.api.automation.trigger(
+          this.automation._id!,
+          request
+        )
+      } catch (e: any) {
+        if (e.cause.status === 404) {
+          throw new Error(
+            `Automation with ID ${
+              this.automation._id
+            } not found in app ${this.config.getAppId()}. You may have forgotten to call config.api.application.publish().`,
+            { cause: e }
+          )
+        } else {
+          throw e
+        }
+      }
+    })
   }
 }
 

packages/server/src/automations/tests/utilities/index.ts

@@ -6,6 +6,7 @@ import { Knex } from "knex"
 import { getQueue } from "../.."
 import { Job } from "bull"
 import { helpers } from "@budibase/shared-core"
+import { queue } from "@budibase/backend-core"
 
 let config: TestConfiguration
 
@@ -20,6 +21,17 @@ export function afterAll() {
   config.end()
 }
 
+export function getTestQueue(): queue.InMemoryQueue<AutomationData> {
+  return getQueue() as unknown as queue.InMemoryQueue<AutomationData>
+}
+
+export function triggerCron(message: Job<AutomationData>) {
+  if (!message.opts?.repeat || !("cron" in message.opts.repeat)) {
+    throw new Error("Expected cron message")
+  }
+  getTestQueue().manualTrigger(message.id)
+}
+
 export async function runInProd(fn: any) {
   env._set("NODE_ENV", "production")
   let error
@@ -34,23 +46,99 @@ export async function runInProd(fn: any) {
   }
 }
 
+export async function captureAllAutomationRemovals(f: () => Promise<unknown>) {
+  const messages: Job<AutomationData>[] = []
+  const queue = getQueue()
+
+  const messageListener = async (message: Job<AutomationData>) => {
+    messages.push(message)
+  }
+
+  queue.on("removed", messageListener)
+  try {
+    await f()
+    // Queue messages tend to be send asynchronously in API handlers, so there's
+    // no guarantee that awaiting this function will have queued anything yet.
+    // We wait here to make sure we're queued _after_ any existing async work.
+    await helpers.wait(100)
+  } finally {
+    queue.off("removed", messageListener)
+  }
+
+  return messages
+}
+
+export async function captureAutomationRemovals(
+  automation: Automation | string,
+  f: () => Promise<unknown>
+) {
+  const messages = await captureAllAutomationRemovals(f)
+  return messages.filter(
+    m =>
+      m.data.automation._id ===
+      (typeof automation === "string" ? automation : automation._id)
+  )
+}
+
+export async function captureAllAutomationMessages(f: () => Promise<unknown>) {
+  const messages: Job<AutomationData>[] = []
+  const queue = getQueue()
+
+  const messageListener = async (message: Job<AutomationData>) => {
+    messages.push(message)
+  }
+
+  queue.on("message", messageListener)
+  try {
+    await f()
+    // Queue messages tend to be send asynchronously in API handlers, so there's
+    // no guarantee that awaiting this function will have queued anything yet.
+    // We wait here to make sure we're queued _after_ any existing async work.
+    await helpers.wait(100)
+  } finally {
+    queue.off("message", messageListener)
+  }
+
+  return messages
+}
+
+export async function captureAutomationMessages(
+  automation: Automation | string,
+  f: () => Promise<unknown>
+) {
+  const messages = await captureAllAutomationMessages(f)
+  return messages.filter(
+    m =>
+      m.data.automation._id ===
+      (typeof automation === "string" ? automation : automation._id)
+  )
+}
+
 /**
  * Capture all automation runs that occur during the execution of a function.
  * This function will wait for all messages to be processed before returning.
  */
 export async function captureAllAutomationResults(
   f: () => Promise<unknown>
-): Promise<Job<AutomationData>[]> {
-  const runs: Job<AutomationData>[] = []
+): Promise<queue.TestQueueMessage<AutomationData>[]> {
+  const runs: queue.TestQueueMessage<AutomationData>[] = []
   const queue = getQueue()
-  let messagesReceived = 0
+  let messagesOutstanding = 0
 
-  const completedListener = async (job: Job<AutomationData>) => {
+  const completedListener = async (
+    job: queue.TestQueueMessage<AutomationData>
+  ) => {
     runs.push(job)
-    messagesReceived--
+    messagesOutstanding--
   }
-  const messageListener = async () => {
-    messagesReceived++
+  const messageListener = async (
+    message: queue.TestQueueMessage<AutomationData>
+  ) => {
+    // Don't count cron messages, as they don't get triggered automatically.
+    if (!message.manualTrigger && message.opts?.repeat != null) {
+      return
+    }
+    messagesOutstanding++
   }
   queue.on("message", messageListener)
   queue.on("completed", completedListener)
@@ -61,9 +149,18 @@ export async function captureAllAutomationResults(
     // We wait here to make sure we're queued _after_ any existing async work.
     await helpers.wait(100)
   } finally {
+    const waitMax = 10000
+    let waited = 0
     // eslint-disable-next-line no-unmodified-loop-condition
-    while (messagesReceived > 0) {
+    while (messagesOutstanding > 0) {
       await helpers.wait(50)
+      waited += 50
+      if (waited > waitMax) {
+        // eslint-disable-next-line no-unsafe-finally
+        throw new Error(
+          `Timed out waiting for automation runs to complete. ${messagesOutstanding} messages waiting for completion.`
+        )
+      }
     }
     queue.off("completed", completedListener)
     queue.off("message", messageListener)

packages/server/src/automations/utils.ts

@@ -72,7 +72,7 @@ export async function processEvent(job: AutomationJob) {
 
       const task = async () => {
         try {
-          if (isCronTrigger(job.data.automation)) {
+          if (isCronTrigger(job.data.automation) && !job.data.event.timestamp) {
             // Requires the timestamp at run time
             job.data.event.timestamp = Date.now()
           }

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -261,11 +261,13 @@ export default class TestConfiguration {
   async withApp<R>(app: App | string, f: () => Promise<R>) {
     const oldAppId = this.appId
     this.appId = typeof app === "string" ? app : app.appId
-    try {
-      return await f()
-    } finally {
-      this.appId = oldAppId
-    }
+    return await context.doInAppContext(this.appId, async () => {
+      try {
+        return await f()
+      } finally {
+        this.appId = oldAppId
+      }
+    })
   }
 
   async withProdApp<R>(f: () => Promise<R>) {

packages/server/src/threads/automation.ts

@@ -155,23 +155,12 @@ class Orchestrator {
     return step
   }
 
-  async getMetadata(): Promise<AutomationMetadata> {
-    const metadataId = generateAutomationMetadataID(this.automation._id!)
-    const db = context.getAppDB()
-    let metadata: AutomationMetadata
-    try {
-      metadata = await db.get(metadataId)
-    } catch (err) {
-      metadata = {
-        _id: metadataId,
-        errorCount: 0,
-      }
-    }
-    return metadata
+  isCron(): boolean {
+    return isRecurring(this.automation)
   }
 
   async stopCron(reason: string) {
-    if (!this.job.opts.repeat) {
+    if (!this.isCron()) {
       return
     }
     logging.logWarn(
@@ -192,45 +181,37 @@ class Orchestrator {
     await storeLog(automation, this.executionOutput)
   }
 
-  async checkIfShouldStop(metadata: AutomationMetadata): Promise<boolean> {
-    if (!metadata.errorCount || !this.job.opts.repeat) {
-      return false
-    }
-    if (metadata.errorCount >= MAX_AUTOMATION_RECURRING_ERRORS) {
-      await this.stopCron("errors")
-      return true
-    }
-    return false
+  async getMetadata(): Promise<AutomationMetadata> {
+    const metadataId = generateAutomationMetadataID(this.automation._id!)
+    const db = context.getAppDB()
+    const metadata = await db.tryGet<AutomationMetadata>(metadataId)
+    return metadata || { _id: metadataId, errorCount: 0 }
   }
 
-  async updateMetadata(metadata: AutomationMetadata) {
-    const output = this.executionOutput,
-      automation = this.automation
-    if (!output || !isRecurring(automation)) {
-      return
-    }
-    const count = metadata.errorCount
-    const isError = isErrorInOutput(output)
-    // nothing to do in this scenario, escape
-    if (!count && !isError) {
-      return
-    }
-    if (isError) {
-      metadata.errorCount = count ? count + 1 : 1
-    } else {
-      metadata.errorCount = 0
-    }
+  async incrementErrorCount() {
     const db = context.getAppDB()
-    try {
-      await db.put(metadata)
-    } catch (err) {
-      logging.logAlertWithInfo(
-        "Failed to write automation metadata",
-        db.name,
-        automation._id!,
-        err
-      )
+    let err: Error | undefined = undefined
+    for (let attempt = 0; attempt < 10; attempt++) {
+      const metadata = await this.getMetadata()
+      metadata.errorCount ||= 0
+      metadata.errorCount++
+
+      try {
+        await db.put(metadata)
+        return metadata.errorCount
+      } catch (error: any) {
+        err = error
+        await helpers.wait(1000 + Math.random() * 1000)
+      }
     }
+
+    logging.logAlertWithInfo(
+      "Failed to update error count in automation metadata",
+      db.name,
+      this.automation._id!,
+      err
+    )
+    return undefined
   }
 
   updateExecutionOutput(id: string, stepId: string, inputs: any, outputs: any) {
@@ -293,18 +274,6 @@ class Orchestrator {
         await enrichBaseContext(this.context)
         this.context.user = this.currentUser
 
-        let metadata
-
-        // check if this is a recurring automation,
-        if (isProdAppID(this.appId) && isRecurring(this.automation)) {
-          span?.addTags({ recurring: true })
-          metadata = await this.getMetadata()
-          const shouldStop = await this.checkIfShouldStop(metadata)
-          if (shouldStop) {
-            span?.addTags({ shouldStop: true })
-            return
-          }
-        }
         const start = performance.now()
 
         await this.executeSteps(this.automation.definition.steps)
@@ -320,23 +289,22 @@ class Orchestrator {
           }
         )
 
-        try {
-          await storeLog(this.automation, this.executionOutput)
-        } catch (e: any) {
-          if (e.status === 413 && e.request?.data) {
-            // if content is too large we shouldn't log it
-            delete e.request.data
-            e.request.data = { message: "removed due to large size" }
-          }
-          logging.logAlert("Error writing automation log", e)
-        }
+        let errorCount = 0
         if (
           isProdAppID(this.appId) &&
-          isRecurring(this.automation) &&
-          metadata
+          this.isCron() &&
+          isErrorInOutput(this.executionOutput)
         ) {
-          await this.updateMetadata(metadata)
+          errorCount = (await this.incrementErrorCount()) || 0
         }
+
+        if (errorCount >= MAX_AUTOMATION_RECURRING_ERRORS) {
+          await this.stopCron("errors")
+          span?.addTags({ shouldStop: true })
+        } else {
+          await storeLog(this.automation, this.executionOutput)
+        }
+
         return this.executionOutput
       }
     )
@@ -763,7 +731,7 @@ export async function executeInThread(
   })) as AutomationResponse
 }
 
-export const removeStalled = async (job: Job) => {
+export const removeStalled = async (job: Job<AutomationData>) => {
   const appId = job.data.event.appId
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")

packages/types/src/api/web/app/automation.ts

@@ -65,6 +65,7 @@ export interface ClearAutomationLogResponse {
 
 export interface TriggerAutomationRequest {
   fields: Record<string, any>
+  timestamp?: number
   // time in seconds
   timeout: number
 }

packages/types/src/ui/bindings/helper.ts

@@ -1,4 +1,6 @@
 export interface Helper {
   example: string
   description: string
+  args: any[]
+  requiresBlock?: boolean
 }

packages/types/src/ui/components/index.ts

@@ -48,6 +48,11 @@ export interface ComponentSetting {
   selectAllFields?: boolean
   resetOn?: string | string[]
   settings?: ComponentSetting[]
+  nested?: boolean
   dependsOn?: DependsOnComponentSetting
   sectionDependsOn?: DependsOnComponentSetting
+  contextAccess?: {
+    global: boolean
+    self: boolean
+  }
 }

packages/types/src/ui/stores/grid/columns.ts

@@ -1,9 +1,15 @@
-import { CalculationType, FieldSchema, FieldType, UIRow } from "@budibase/types"
+import {
+  CalculationType,
+  FieldSchema,
+  FieldType,
+  UICondition,
+  UIRow,
+} from "@budibase/types"
 
 export type UIColumn = FieldSchema & {
   label: string
   readonly: boolean
-  conditions: any
+  conditions?: UICondition[]
   format?: (row: UIRow) => any
   related?: {
     field: string

packages/types/src/ui/stores/grid/condition.ts

@@ -3,7 +3,7 @@ import { FieldType, SearchFilter } from "@budibase/types"
 export interface UICondition {
   column: string
   type: FieldType
-  referenceValue: string
+  referenceValue: any
   operator: SearchFilter["operator"]
   metadataKey: string
   metadataValue: string