Merge remote-tracking branch 'origin/execute-script-v2' into execute-script-v2-frontend

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.4.9",
+  "version": "3.4.11",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -52,7 +52,11 @@ class InMemoryQueue implements Partial<Queue> {
   _opts?: QueueOptions
   _messages: JobMessage[]
   _queuedJobIds: Set<string>
-  _emitter: NodeJS.EventEmitter<{ message: [JobMessage]; completed: [Job] }>
+  _emitter: NodeJS.EventEmitter<{
+    message: [JobMessage]
+    completed: [Job]
+    removed: [JobMessage]
+  }>
   _runCount: number
   _addCount: number
 
@@ -83,6 +87,12 @@ class InMemoryQueue implements Partial<Queue> {
   async process(concurrencyOrFunc: number | any, func?: any) {
     func = typeof concurrencyOrFunc === "number" ? func : concurrencyOrFunc
     this._emitter.on("message", async message => {
+      // For the purpose of testing, don't trigger cron jobs immediately.
+      // Require the test to trigger them manually with timestamps.
+      if (message.opts?.repeat != null) {
+        return
+      }
+
       let resp = func(message)
 
       async function retryFunc(fnc: any) {
@@ -164,13 +174,14 @@ class InMemoryQueue implements Partial<Queue> {
    */
   async close() {}
 
-  /**
-   * This removes a cron which has been implemented, this is part of Bull API.
-   * @param cronJobId The cron which is to be removed.
-   */
-  async removeRepeatableByKey(cronJobId: string) {
-    // TODO: implement for testing
-    console.log(cronJobId)
+  async removeRepeatableByKey(id: string) {
+    for (const [idx, message] of this._messages.entries()) {
+      if (message.opts?.jobId?.toString() === id) {
+        this._messages.splice(idx, 1)
+        this._emitter.emit("removed", message)
+        return
+      }
+    }
   }
 
   async removeJobs(_pattern: string) {
@@ -214,7 +225,9 @@ class InMemoryQueue implements Partial<Queue> {
   }
 
   async getRepeatableJobs() {
-    return this._messages.map(job => jobToJobInformation(job as Job))
+    return this._messages
+      .filter(job => job.opts?.repeat != null)
+      .map(job => jobToJobInformation(job as Job))
   }
 }
 

packages/backend-core/src/utils/tests/utils.spec.ts

@@ -67,6 +67,15 @@ describe("utils", () => {
       })
     })
 
+    it("gets appId from query params", async () => {
+      const ctx = structures.koa.newContext()
+      const expected = db.generateAppID()
+      ctx.query = { appId: expected }
+
+      const actual = await utils.getAppIdFromCtx(ctx)
+      expect(actual).toBe(expected)
+    })
+
     it("doesn't get appId from url when previewing", async () => {
       const ctx = structures.koa.newContext()
       const appId = db.generateAppID()

packages/backend-core/src/utils/utils.ts

@@ -101,6 +101,11 @@ export async function getAppIdFromCtx(ctx: Ctx) {
     appId = confirmAppId(pathId)
   }
 
+  // look in queryParams
+  if (!appId && ctx.query?.appId) {
+    appId = confirmAppId(ctx.query?.appId as string)
+  }
+
   // lookup using custom url - prod apps only
   // filter out the builder preview path which collides with the prod app path
   // to ensure we don't load all apps excessively

packages/bbui/src/Form/Core/Multiselect.svelte

@@ -14,7 +14,7 @@
   export let sort = false
   export let autoWidth = false
   export let searchTerm = null
-  export let customPopoverHeight
+  export let customPopoverHeight = undefined
   export let open = false
   export let loading
   export let onOptionMouseenter = () => {}

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -386,7 +386,7 @@
       editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
     } else if (editableColumn.type === FieldType.FORMULA) {
       editableColumn.formulaType = "dynamic"
-      editableColumn.responseType = field.responseType || FIELDS.STRING.type
+      editableColumn.responseType = field?.responseType || FIELDS.STRING.type
     }
   }
 

packages/builder/src/components/common/CodeEditor/CodeEditor.svelte

@@ -47,17 +47,20 @@
     indentMore,
     indentLess,
   } from "@codemirror/commands"
+  import { setDiagnostics } from "@codemirror/lint"
   import { Compartment, EditorState } from "@codemirror/state"
+  import type { Extension } from "@codemirror/state"
   import { javascript } from "@codemirror/lang-javascript"
   import { EditorModes } from "./"
   import { themeStore } from "@/stores/portal"
   import type { EditorMode } from "@budibase/types"
   import { tooltips } from "@codemirror/view"
-  import type { BindingCompletion } from "@/types"
+  import type { BindingCompletion, CodeValidator } from "@/types"
+  import { validateHbsTemplate } from "./validator/hbs"
 
   export let label: string | undefined = undefined
-  // TODO: work out what best type fits this
   export let completions: BindingCompletion[] = []
+  export let validations: CodeValidator | null = null
   export let mode: EditorMode = EditorModes.Handlebars
   export let value: string | null = ""
   export let placeholder: string | null = null
@@ -257,7 +260,7 @@
   // None of this is reactive, but it never has been, so we just assume most
   // config flags aren't changed at runtime
   // TODO: work out type for base
-  const buildExtensions = (base: any[]) => {
+  const buildExtensions = (base: Extension[]) => {
     let complete = [...base]
 
     if (autocompleteEnabled) {
@@ -349,6 +352,24 @@
     return complete
   }
 
+  function validate(
+    value: string | null,
+    editor: EditorView | undefined,
+    mode: EditorMode,
+    validations: CodeValidator | null
+  ) {
+    if (!value || !validations || !editor) {
+      return
+    }
+
+    if (mode === EditorModes.Handlebars) {
+      const diagnostics = validateHbsTemplate(value, validations)
+      editor.dispatch(setDiagnostics(editor.state, diagnostics))
+    }
+  }
+
+  $: validate(value, editor, mode, validations)
+
   const initEditor = () => {
     const baseExtensions = buildBaseExtensions()
 

packages/builder/src/components/common/CodeEditor/index.ts

@@ -83,6 +83,8 @@ const helpersToCompletion = (
     const helper = helpers[helperName]
     return {
       label: helperName,
+      args: helper.args,
+      requiresBlock: helper.requiresBlock,
       info: () => buildHelperInfoNode(helper),
       type: "helper",
       section: helperSection,
@@ -136,9 +138,13 @@ export const hbAutocomplete = (
   baseCompletions: BindingCompletionOption[]
 ): BindingCompletion => {
   function coreCompletion(context: CompletionContext) {
-    let bindingStart = context.matchBefore(EditorModes.Handlebars.match)
+    if (!baseCompletions.length) {
+      return null
+    }
 
-    let options = baseCompletions || []
+    const bindingStart = context.matchBefore(EditorModes.Handlebars.match)
+
+    const options = baseCompletions
 
     if (!bindingStart) {
       return null
@@ -149,7 +155,7 @@ export const hbAutocomplete = (
       return null
     }
     const query = bindingStart.text.replace(match[0], "")
-    let filtered = bindingFilter(options, query)
+    const filtered = bindingFilter(options, query)
 
     return {
       from: bindingStart.from + match[0].length,
@@ -169,8 +175,12 @@ export const jsAutocomplete = (
   baseCompletions: BindingCompletionOption[]
 ): BindingCompletion => {
   function coreCompletion(context: CompletionContext) {
-    let jsBinding = wrappedAutocompleteMatch(context)
-    let options = baseCompletions || []
+    if (!baseCompletions.length) {
+      return null
+    }
+
+    const jsBinding = wrappedAutocompleteMatch(context)
+    const options = baseCompletions
 
     if (jsBinding) {
       // Accommodate spaces
@@ -209,6 +219,10 @@ function setAutocomplete(
   options: BindingCompletionOption[]
 ): BindingCompletion {
   return function (context: CompletionContext) {
+    if (!options.length) {
+      return null
+    }
+
     if (wrappedAutocompleteMatch(context)) {
       return null
     }

packages/builder/src/components/common/CodeEditor/validator/hbs.ts

@@ -0,0 +1,113 @@
+/* global hbs */
+import Handlebars from "handlebars"
+import type { Diagnostic } from "@codemirror/lint"
+import { CodeValidator } from "@/types"
+
+function isMustacheStatement(
+  node: hbs.AST.Statement
+): node is hbs.AST.MustacheStatement {
+  return node.type === "MustacheStatement"
+}
+
+function isBlockStatement(
+  node: hbs.AST.Statement
+): node is hbs.AST.BlockStatement {
+  return node.type === "BlockStatement"
+}
+
+function isPathExpression(
+  node: hbs.AST.Statement
+): node is hbs.AST.PathExpression {
+  return node.type === "PathExpression"
+}
+
+export function validateHbsTemplate(
+  text: string,
+  validations: CodeValidator
+): Diagnostic[] {
+  const diagnostics: Diagnostic[] = []
+
+  try {
+    const ast = Handlebars.parse(text, {})
+
+    const lineOffsets: number[] = []
+    let offset = 0
+    for (const line of text.split("\n")) {
+      lineOffsets.push(offset)
+      offset += line.length + 1 // +1 for newline character
+    }
+
+    function traverseNodes(
+      nodes: hbs.AST.Statement[],
+      options?: {
+        ignoreMissing?: boolean
+      }
+    ) {
+      const ignoreMissing = options?.ignoreMissing || false
+      nodes.forEach(node => {
+        if (isMustacheStatement(node) && isPathExpression(node.path)) {
+          const helperName = node.path.original
+
+          const from =
+            lineOffsets[node.loc.start.line - 1] + node.loc.start.column
+          const to = lineOffsets[node.loc.end.line - 1] + node.loc.end.column
+
+          if (!(helperName in validations)) {
+            if (!ignoreMissing) {
+              diagnostics.push({
+                from,
+                to,
+                severity: "warning",
+                message: `"${helperName}" handler does not exist.`,
+              })
+            }
+            return
+          }
+
+          const { arguments: expectedArguments = [], requiresBlock } =
+            validations[helperName]
+
+          if (requiresBlock && !isBlockStatement(node)) {
+            diagnostics.push({
+              from,
+              to,
+              severity: "error",
+              message: `Helper "${helperName}" requires a body:\n{{#${helperName} ...}} [body] {{/${helperName}}}`,
+            })
+            return
+          }
+
+          const providedParams = node.params
+
+          if (providedParams.length !== expectedArguments.length) {
+            diagnostics.push({
+              from,
+              to,
+              severity: "error",
+              message: `Helper "${helperName}" expects ${
+                expectedArguments.length
+              } parameters (${expectedArguments.join(", ")}), but got ${
+                providedParams.length
+              }.`,
+            })
+          }
+        }
+
+        if (isBlockStatement(node)) {
+          traverseNodes(node.program.body, { ignoreMissing: true })
+        }
+      })
+    }
+
+    traverseNodes(ast.body, { ignoreMissing: true })
+  } catch (e: any) {
+    diagnostics.push({
+      from: 0,
+      to: text.length,
+      severity: "error",
+      message: `The handlebars code is not valid:\n${e.message}`,
+    })
+  }
+
+  return diagnostics
+}

packages/builder/src/components/common/CodeEditor/validator/tests/hbs.spec.ts

@@ -0,0 +1,142 @@
+import { validateHbsTemplate } from "../hbs"
+import { CodeValidator } from "@/types"
+
+describe("hbs validator", () => {
+  it("validate empty strings", () => {
+    const text = ""
+    const validators = {}
+
+    const result = validateHbsTemplate(text, validators)
+    expect(result).toHaveLength(0)
+  })
+
+  it("validate strings without hbs expressions", () => {
+    const text = "first line\nand another one"
+    const validators = {}
+
+    const result = validateHbsTemplate(text, validators)
+    expect(result).toHaveLength(0)
+  })
+
+  describe("basic expressions", () => {
+    const validators = {
+      fieldName: {},
+    }
+
+    it("validate valid expressions", () => {
+      const text = "{{ fieldName }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toHaveLength(0)
+    })
+
+    it("does not throw on missing validations", () => {
+      const text = "{{ anotherFieldName }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toHaveLength(0)
+    })
+
+    // Waiting for missing fields validation
+    it.skip("throws on untrimmed invalid expressions", () => {
+      const text = "    {{ anotherFieldName }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 4,
+          message: `"anotherFieldName" handler does not exist.`,
+          severity: "warning",
+          to: 26,
+        },
+      ])
+    })
+
+    // Waiting for missing fields validation
+    it.skip("throws on invalid expressions between valid lines", () => {
+      const text =
+        "literal expression\nthe value is {{ anotherFieldName }}\nanother expression"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 32,
+          message: `"anotherFieldName" handler does not exist.`,
+          severity: "warning",
+          to: 54,
+        },
+      ])
+    })
+
+    describe("expressions with whitespaces", () => {
+      const validators = {
+        [`field name`]: {},
+      }
+
+      it("validates expressions with whitespaces", () => {
+        const text = `{{ [field name] }}`
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toHaveLength(0)
+      })
+
+      // Waiting for missing fields validation
+      it.skip("throws if not wrapped between brackets", () => {
+        const text = `{{ field name }}`
+
+        const result = validateHbsTemplate(text, validators)
+        expect(result).toEqual([
+          {
+            from: 0,
+            message: `"field" handler does not exist.`,
+            severity: "warning",
+            to: 16,
+          },
+        ])
+      })
+    })
+  })
+
+  describe("expressions with parameters", () => {
+    const validators: CodeValidator = {
+      helperFunction: {
+        arguments: ["a", "b", "c"],
+      },
+    }
+
+    it("validate valid params", () => {
+      const text = "{{ helperFunction 1 99 'a' }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toHaveLength(0)
+    })
+
+    it("throws on too few params", () => {
+      const text = "{{ helperFunction 100 }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 0,
+          message: `Helper "helperFunction" expects 3 parameters (a, b, c), but got 1.`,
+          severity: "error",
+          to: 24,
+        },
+      ])
+    })
+
+    it("throws on too many params", () => {
+      const text = "{{ helperFunction  1 99 'a' 100 }}"
+
+      const result = validateHbsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 0,
+          message: `Helper "helperFunction" expects 3 parameters (a, b, c), but got 4.`,
+          severity: "error",
+          to: 34,
+        },
+      ])
+    })
+  })
+})

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -42,7 +42,7 @@
     JSONValue,
   } from "@budibase/types"
   import type { Log } from "@budibase/string-templates"
-  import type { BindingCompletion, BindingCompletionOption } from "@/types"
+  import type { CodeValidator } from "@/types"
 
   const dispatch = createEventDispatcher()
 
@@ -58,7 +58,7 @@
   export let placeholder = null
   export let showTabBar = true
 
-  let mode: BindingMode | null
+  let mode: BindingMode
   let sidePanel: SidePanel | null
   let initialValueJS = value?.startsWith?.("{{ js ")
   let jsValue: string | null = initialValueJS ? value : null
@@ -88,13 +88,37 @@
     | null
   $: runtimeExpression = readableToRuntimeBinding(enrichedBindings, value)
   $: requestEval(runtimeExpression, context, snippets)
-  $: bindingCompletions = bindingsToCompletions(enrichedBindings, editorMode)
   $: bindingHelpers = new BindingHelpers(getCaretPosition, insertAtPos)
-  $: hbsCompletions = getHBSCompletions(bindingCompletions)
-  $: jsCompletions = getJSCompletions(bindingCompletions, snippets, {
-    useHelpers: allowHelpers,
-    useSnippets,
-  })
+
+  $: bindingOptions = bindingsToCompletions(bindings, editorMode)
+  $: helperOptions = allowHelpers ? getHelperCompletions(editorMode) : []
+  $: snippetsOptions =
+    usingJS && useSnippets && snippets?.length ? snippets : []
+
+  $: completions = !usingJS
+    ? [hbAutocomplete([...bindingOptions, ...helperOptions])]
+    : [
+        jsAutocomplete(bindingOptions),
+        jsHelperAutocomplete(helperOptions),
+        snippetAutoComplete(snippetsOptions),
+      ]
+
+  $: validations = {
+    ...bindingOptions.reduce<CodeValidator>((validations, option) => {
+      validations[option.label] = {
+        arguments: [],
+      }
+      return validations
+    }, {}),
+    ...helperOptions.reduce<CodeValidator>((validations, option) => {
+      validations[option.label] = {
+        arguments: option.args,
+        requiresBlock: option.requiresBlock,
+      }
+      return validations
+    }, {}),
+  }
+
   $: {
     // Ensure a valid side panel option is always selected
     if (sidePanel && !sidePanelOptions.includes(sidePanel)) {
@@ -102,38 +126,6 @@
     }
   }
 
-  const getHBSCompletions = (bindingCompletions: BindingCompletionOption[]) => {
-    return [
-      hbAutocomplete([
-        ...bindingCompletions,
-        ...getHelperCompletions(EditorModes.Handlebars),
-      ]),
-    ]
-  }
-
-  const getJSCompletions = (
-    bindingCompletions: BindingCompletionOption[],
-    snippets: Snippet[] | null,
-    config: {
-      useHelpers: boolean
-      useSnippets: boolean
-    }
-  ) => {
-    const completions: BindingCompletion[] = []
-    if (bindingCompletions.length) {
-      completions.push(jsAutocomplete([...bindingCompletions]))
-    }
-    if (config.useHelpers) {
-      completions.push(
-        jsHelperAutocomplete([...getHelperCompletions(EditorModes.JS)])
-      )
-    }
-    if (config.useSnippets && snippets) {
-      completions.push(snippetAutoComplete(snippets))
-    }
-    return completions
-  }
-
   const getModeOptions = (allowHBS: boolean, allowJS: boolean) => {
     let options = []
     if (allowHBS) {
@@ -213,7 +205,7 @@
     bindings: EnrichedBinding[],
     context: any,
     snippets: Snippet[] | null
-  ) => {
+  ): EnrichedBinding[] => {
     // Create a single big array to enrich in one go
     const bindingStrings = bindings.map(binding => {
       if (binding.runtimeBinding.startsWith('trim "')) {
@@ -290,7 +282,7 @@
     jsValue = null
     hbsValue = null
     updateValue(null)
-    mode = targetMode
+    mode = targetMode!
     targetMode = null
   }
 
@@ -365,13 +357,14 @@
       {/if}
       <div class="editor">
         {#if mode === BindingMode.Text}
-          {#key hbsCompletions}
+          {#key completions}
             <CodeEditor
               value={hbsValue || ""}
               on:change={onChangeHBSValue}
               bind:getCaretPosition
               bind:insertAtPos
-              completions={hbsCompletions}
+              {completions}
+              {validations}
               autofocus={autofocusEditor}
               placeholder={placeholder ||
                 "Add bindings by typing {{ or use the menu on the right"}
@@ -379,18 +372,18 @@
             />
           {/key}
         {:else if mode === BindingMode.JavaScript}
-          {#key jsCompletions}
+          {#key completions}
             <CodeEditor
               value={jsValue ? decodeJSBinding(jsValue) : ""}
               on:change={onChangeJSValue}
-              completions={jsCompletions}
+              {completions}
               mode={EditorModes.JS}
               bind:getCaretPosition
               bind:insertAtPos
               autofocus={autofocusEditor}
               placeholder={placeholder ||
                 "Add bindings by typing $ or use the menu on the right"}
-              jsBindingWrapping={bindingCompletions.length > 0}
+              jsBindingWrapping={completions.length > 0}
             />
           {/key}
         {/if}

packages/builder/src/components/design/settings/controls/ButtonConfiguration/ButtonSetting.svelte

@@ -43,7 +43,6 @@
     <EditComponentPopover
       {anchor}
       componentInstance={item}
-      {componentBindings}
       {bindings}
       on:change
       parseSettings={updatedNestedFlags}

packages/builder/src/components/design/settings/controls/EditComponentPopover.svelte

@@ -1,13 +1,13 @@
 <script>
   import { Icon, Popover, Layout } from "@budibase/bbui"
-  import { componentStore } from "@/stores/builder"
+  import { componentStore, selectedScreen } from "@/stores/builder"
   import { cloneDeep } from "lodash/fp"
   import { createEventDispatcher, getContext } from "svelte"
   import ComponentSettingsSection from "@/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte"
+  import { getComponentBindableProperties } from "@/dataBinding"
 
   export let anchor
   export let componentInstance
-  export let componentBindings
   export let bindings
   export let parseSettings
 
@@ -28,6 +28,10 @@
   }
   $: componentDef = componentStore.getDefinition(componentInstance._component)
   $: parsedComponentDef = processComponentDefinitionSettings(componentDef)
+  $: componentBindings = getComponentBindableProperties(
+    $selectedScreen,
+    $componentStore.selectedComponentId
+  )
 
   const open = () => {
     isOpen = true

packages/builder/src/components/design/settings/controls/FieldConfiguration/FieldSetting.svelte

@@ -45,7 +45,6 @@
     <EditComponentPopover
       {anchor}
       componentInstance={item}
-      {componentBindings}
       {bindings}
       {parseSettings}
       on:change

packages/builder/src/components/design/settings/controls/PropertyControl.svelte

@@ -22,25 +22,59 @@
   export let propertyFocus = false
   export let info = null
   export let disableBindings = false
-  export let wide
+  export let wide = false
+  export let contextAccess = null
 
   let highlightType
   let domElement
 
   $: highlightedProp = $builderStore.highlightedSetting
-  $: allBindings = getAllBindings(bindings, componentBindings, nested)
+  $: allBindings = getAllBindings(
+    bindings,
+    componentBindings,
+    nested,
+    contextAccess
+  )
   $: safeValue = getSafeValue(value, defaultValue, allBindings)
   $: replaceBindings = val => readableToRuntimeBinding(allBindings, val)
-
   $: isHighlighted = highlightedProp?.key === key
-
   $: highlightType = isHighlighted ? `highlighted-${highlightedProp?.type}` : ""
+  $: highlightedProp && isHighlighted && scrollToElement(domElement)
 
-  const getAllBindings = (bindings, componentBindings, nested) => {
-    if (!nested) {
+  const getAllBindings = (
+    bindings,
+    componentBindings,
+    nested,
+    contextAccess
+  ) => {
+    // contextAccess is a bit of an escape hatch to get around how we render
+    // certain settings types by using a pseudo component definition, leading
+    // to problems with the nested flag
+    if (contextAccess != null) {
+      // Optionally include global bindings
+      let allBindings = contextAccess.global ? bindings : []
+
+      // Optionally include or exclude self (component) bindings.
+      // If this is a nested setting then we will already have our own context
+      // bindings mixed in, so if we don't want self context we need to filter
+      // them out.
+      if (contextAccess.self) {
+        return [...allBindings, ...componentBindings]
+      } else {
+        return allBindings.filter(binding => {
+          return !componentBindings.some(componentBinding => {
+            return componentBinding.runtimeBinding === binding.runtimeBinding
+          })
+        })
+      }
+    }
+
+    // Otherwise just honour the normal nested flag
+    if (nested) {
+      return [...bindings, ...componentBindings]
+    } else {
       return bindings
     }
-    return [...(componentBindings || []), ...(bindings || [])]
   }
 
   // Handle a value change of any type
@@ -81,8 +115,6 @@
       block: "center",
     })
   }
-
-  $: highlightedProp && isHighlighted && scrollToElement(domElement)
 </script>
 
 <div

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte

@@ -147,6 +147,7 @@
           {componentInstance}
           {componentDefinition}
           {bindings}
+          {componentBindings}
         />
       {/if}
     </Panel>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte

@@ -151,6 +151,7 @@
               propertyFocus={$builderStore.propertyFocus === setting.key}
               info={setting.info}
               disableBindings={setting.disableBindings}
+              contextAccess={setting.contextAccess}
               props={{
                 // Generic settings
                 placeholder: setting.placeholder || null,

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUIDrawer.svelte

@@ -19,6 +19,7 @@
 
   export let conditions = []
   export let bindings = []
+  export let componentBindings = []
 
   const flipDurationMs = 150
   const actionOptions = [
@@ -55,6 +56,7 @@
   ]
 
   let dragDisabled = true
+
   $: settings = componentStore
     .getComponentSettings($selectedComponent?._component)
     ?.concat({
@@ -213,7 +215,10 @@
                       options: definition.options,
                       placeholder: definition.placeholder,
                     }}
+                    nested={definition.nested}
+                    contextAccess={definition.contextAccess}
                     {bindings}
+                    {componentBindings}
                   />
                 {:else}
                   <Select disabled placeholder=" " />

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUISection.svelte

@@ -64,7 +64,12 @@
     Show, hide and update components in response to conditions being met.
   </svelte:fragment>
   <Button cta slot="buttons" on:click={() => save()}>Save</Button>
-  <ConditionalUIDrawer slot="body" bind:conditions={tempValue} {bindings} />
+  <ConditionalUIDrawer
+    slot="body"
+    bind:conditions={tempValue}
+    {bindings}
+    {componentBindings}
+  />
 </Drawer>
 
 <style>

packages/builder/src/types/bindings.ts

@@ -5,4 +5,15 @@ export type BindingCompletion = (context: CompletionContext) => {
   options: Completion[]
 } | null
 
-export type BindingCompletionOption = Completion
+export interface BindingCompletionOption extends Completion {
+  args?: any[]
+  requiresBlock?: boolean
+}
+
+export type CodeValidator = Record<
+  string,
+  {
+    arguments?: any[]
+    requiresBlock?: boolean
+  }
+>

packages/builder/tsconfig.json

@@ -14,5 +14,6 @@
       "assets/*": ["assets/*"],
       "@/*": ["src/*"]
     }
-  }
+  },
+  "exclude": []
 }

packages/client/manifest.json

@@ -3089,7 +3089,21 @@
       {
         "type": "tableConditions",
         "label": "Conditions",
-        "key": "conditions"
+        "key": "conditions",
+        "contextAccess": {
+          "global": true,
+          "self": false
+        }
+      },
+      {
+        "type": "text",
+        "label": "Format",
+        "key": "format",
+        "info": "Changing format will display values as text",
+        "contextAccess": {
+          "global": false,
+          "self": true
+        }
       }
     ]
   },
@@ -7686,7 +7700,8 @@
           {
             "type": "columns/grid",
             "key": "columns",
-            "resetOn": "table"
+            "resetOn": "table",
+            "nested": true
           }
         ]
       },

packages/client/src/components/app/GridBlock.svelte

@@ -5,7 +5,7 @@
   import { get, derived, readable } from "svelte/store"
   import { featuresStore } from "@/stores"
   import { Grid } from "@budibase/frontend-core"
-  // import { processStringSync } from "@budibase/string-templates"
+  import { processStringSync } from "@budibase/string-templates"
 
   // table is actually any datasource, but called table for legacy compatibility
   export let table
@@ -47,8 +47,8 @@
   $: currentTheme = $context?.device?.theme
   $: darkMode = !currentTheme?.includes("light")
   $: parsedColumns = getParsedColumns(columns)
-  $: schemaOverrides = getSchemaOverrides(parsedColumns)
   $: enrichedButtons = enrichButtons(buttons)
+  $: schemaOverrides = getSchemaOverrides(parsedColumns, $context)
   $: selectedRows = deriveSelectedRows(gridContext)
   $: styles = patchStyles($component.styles, minHeight)
   $: data = { selectedRows: $selectedRows }
@@ -97,15 +97,19 @@
     }))
   }
 
-  const getSchemaOverrides = columns => {
+  const getSchemaOverrides = (columns, context) => {
     let overrides = {}
     columns.forEach((column, idx) => {
       overrides[column.field] = {
         displayName: column.label,
         order: idx,
-        conditions: column.conditions,
         visible: !!column.active,
-        // format: createFormatter(column),
+        conditions: enrichConditions(column.conditions, context),
+        format: createFormatter(column),
+
+        // Small hack to ensure we react to all changes, as our
+        // memoization cannot compare differences in functions
+        rand: column.conditions?.length ? Math.random() : null,
       }
       if (column.width) {
         overrides[column.field].width = column.width
@@ -114,12 +118,24 @@
     return overrides
   }
 
-  // const createFormatter = column => {
-  //   if (typeof column.format !== "string" || !column.format.trim().length) {
-  //     return null
-  //   }
-  //   return row => processStringSync(column.format, { [id]: row })
-  // }
+  const enrichConditions = (conditions, context) => {
+    return conditions?.map(condition => {
+      return {
+        ...condition,
+        referenceValue: processStringSync(
+          condition.referenceValue || "",
+          context
+        ),
+      }
+    })
+  }
+
+  const createFormatter = column => {
+    if (typeof column.format !== "string" || !column.format.trim().length) {
+      return null
+    }
+    return row => processStringSync(column.format, { [id]: row })
+  }
 
   const enrichButtons = buttons => {
     if (!buttons?.length) {

packages/client/src/components/app/Placeholder.svelte

@@ -5,7 +5,7 @@
   const component = getContext("component")
   const block = getContext("block")
 
-  export let text
+  export let text = undefined
 </script>
 
 {#if $builderStore.inBuilder}

packages/client/src/components/app/forms/Field.svelte

@@ -1,4 +1,4 @@
-<script>
+<script lang="ts">
   import { getContext, onDestroy } from "svelte"
   import { writable } from "svelte/store"
   import { Icon } from "@budibase/bbui"
@@ -6,33 +6,33 @@
   import Placeholder from "../Placeholder.svelte"
   import InnerForm from "./InnerForm.svelte"
 
-  export let label
-  export let field
-  export let fieldState
-  export let fieldApi
-  export let fieldSchema
-  export let defaultValue
-  export let type
+  export let label: string | undefined = undefined
+  export let field: string | undefined = undefined
+  export let fieldState: any
+  export let fieldApi: any
+  export let fieldSchema: any
+  export let defaultValue: string | undefined = undefined
+  export let type: any
   export let disabled = false
   export let readonly = false
-  export let validation
+  export let validation: any
   export let span = 6
-  export let helpText = null
+  export let helpText: string | undefined = undefined
 
   // Get contexts
-  const formContext = getContext("form")
-  const formStepContext = getContext("form-step")
-  const fieldGroupContext = getContext("field-group")
+  const formContext: any = getContext("form")
+  const formStepContext: any = getContext("form-step")
+  const fieldGroupContext: any = getContext("field-group")
   const { styleable, builderStore, Provider } = getContext("sdk")
-  const component = getContext("component")
+  const component: any = getContext("component")
 
   // Register field with form
   const formApi = formContext?.formApi
   const labelPos = fieldGroupContext?.labelPosition || "above"
 
-  let formField
+  let formField: any
   let touched = false
-  let labelNode
+  let labelNode: any
 
   // Memoize values required to register the field to avoid loops
   const formStep = formStepContext || writable(1)
@@ -65,7 +65,7 @@
   $: $component.editing && labelNode?.focus()
 
   // Update form properties in parent component on every store change
-  $: unsubscribe = formField?.subscribe(value => {
+  $: unsubscribe = formField?.subscribe((value: any) => {
     fieldState = value?.fieldState
     fieldApi = value?.fieldApi
     fieldSchema = value?.fieldSchema
@@ -74,7 +74,7 @@
   // Determine label class from position
   $: labelClass = labelPos === "above" ? "" : `spectrum-FieldLabel--${labelPos}`
 
-  const registerField = info => {
+  const registerField = (info: any) => {
     formField = formApi?.registerField(
       info.field,
       info.type,
@@ -86,8 +86,9 @@
     )
   }
 
-  const updateLabel = e => {
+  const updateLabel = (e: any) => {
     if (touched) {
+      // @ts-expect-error and TODO updateProp isn't recognised - need builder TS conversion
       builderStore.actions.updateProp("label", e.target.textContent)
     }
     touched = false

packages/client/src/components/app/forms/InnerForm.svelte

@@ -4,13 +4,13 @@
   import { createValidatorFromConstraints } from "./validation"
   import { Helpers } from "@budibase/bbui"
 
-  export let dataSource
+  export let dataSource = undefined
   export let disabled = false
   export let readonly = false
-  export let initialValues
-  export let size
-  export let schema
-  export let definition
+  export let initialValues = undefined
+  export let size = undefined
+  export let schema = undefined
+  export let definition = undefined
   export let disableSchemaValidation = false
   export let editAutoColumns = false
 

packages/client/src/components/app/forms/RelationshipField.svelte

@@ -1,43 +1,61 @@
-<script>
+<script lang="ts">
   import { CoreSelect, CoreMultiselect } from "@budibase/bbui"
   import { FieldType } from "@budibase/types"
   import { fetchData, Utils } from "@budibase/frontend-core"
   import { getContext } from "svelte"
   import Field from "./Field.svelte"
+  import type {
+    SearchFilter,
+    RelationshipFieldMetadata,
+    Table,
+    Row,
+  } from "@budibase/types"
 
   const { API } = getContext("sdk")
 
-  export let field
-  export let label
-  export let placeholder
-  export let disabled = false
-  export let readonly = false
-  export let validation
-  export let autocomplete = true
-  export let defaultValue
-  export let onChange
-  export let filter
-  export let datasourceType = "table"
-  export let primaryDisplay
-  export let span
-  export let helpText = null
-  export let type = FieldType.LINK
+  export let field: string | undefined = undefined
+  export let label: string | undefined = undefined
+  export let placeholder: any = undefined
+  export let disabled: boolean = false
+  export let readonly: boolean = false
+  export let validation: any
+  export let autocomplete: boolean = true
+  export let defaultValue: string | undefined = undefined
+  export let onChange: any
+  export let filter: SearchFilter[]
+  export let datasourceType: "table" | "user" | "groupUser" = "table"
+  export let primaryDisplay: string | undefined = undefined
+  export let span: number | undefined = undefined
+  export let helpText: string | undefined = undefined
+  export let type:
+    | FieldType.LINK
+    | FieldType.BB_REFERENCE
+    | FieldType.BB_REFERENCE_SINGLE = FieldType.LINK
 
-  let fieldState
-  let fieldApi
-  let fieldSchema
-  let tableDefinition
-  let searchTerm
-  let open
+  type RelationshipValue = { _id: string; [key: string]: any }
+  type OptionObj = Record<string, RelationshipValue>
+  type OptionsObjType = Record<string, OptionObj>
 
+  let fieldState: any
+  let fieldApi: any
+  let fieldSchema: RelationshipFieldMetadata | undefined
+  let tableDefinition: Table | null | undefined
+  let searchTerm: any
+  let open: boolean
+  let selectedValue: string[] | string
+
+  // need a cast version of this for reactivity, components below aren't typed
+  $: castSelectedValue = selectedValue as any
   $: multiselect =
     [FieldType.LINK, FieldType.BB_REFERENCE].includes(type) &&
     fieldSchema?.relationshipType !== "one-to-many"
-  $: linkedTableId = fieldSchema?.tableId
+  $: linkedTableId = fieldSchema?.tableId!
   $: fetch = fetchData({
     API,
     datasource: {
-      type: datasourceType,
+      // typing here doesn't seem correct - we have the correct datasourceType options
+      // but when we configure the fetchData, it seems to think only "table" is valid
+      type: datasourceType as any,
       tableId: linkedTableId,
     },
     options: {
@@ -53,7 +71,8 @@
   $: component = multiselect ? CoreMultiselect : CoreSelect
   $: primaryDisplay = primaryDisplay || tableDefinition?.primaryDisplay
 
-  let optionsObj
+  let optionsObj: OptionsObjType = {}
+  const debouncedFetchRows = Utils.debounce(fetchRows, 250)
 
   $: {
     if (primaryDisplay && fieldState && !optionsObj) {
@@ -63,27 +82,33 @@
       if (!Array.isArray(valueAsSafeArray)) {
         valueAsSafeArray = [fieldState.value]
       }
-      optionsObj = valueAsSafeArray.reduce((accumulator, value) => {
-        // fieldState has to be an array of strings to be valid for an update
-        // therefore we cannot guarantee value will be an object
-        // https://linear.app/budibase/issue/BUDI-7577/refactor-the-relationshipfield-component-to-have-better-support-for
-        if (!value._id) {
+      optionsObj = valueAsSafeArray.reduce(
+        (
+          accumulator: OptionObj,
+          value: { _id: string; primaryDisplay: any }
+        ) => {
+          // fieldState has to be an array of strings to be valid for an update
+          // therefore we cannot guarantee value will be an object
+          // https://linear.app/budibase/issue/BUDI-7577/refactor-the-relationshipfield-component-to-have-better-support-for
+          if (!value._id) {
+            return accumulator
+          }
+          accumulator[value._id] = {
+            _id: value._id,
+            [primaryDisplay]: value.primaryDisplay,
+          }
           return accumulator
-        }
-        accumulator[value._id] = {
-          _id: value._id,
-          [primaryDisplay]: value.primaryDisplay,
-        }
-        return accumulator
-      }, {})
+        },
+        {}
+      )
     }
   }
 
   $: enrichedOptions = enrichOptions(optionsObj, $fetch.rows)
-  const enrichOptions = (optionsObj, fetchResults) => {
+  const enrichOptions = (optionsObj: OptionsObjType, fetchResults: Row[]) => {
     const result = (fetchResults || [])?.reduce((accumulator, row) => {
-      if (!accumulator[row._id]) {
-        accumulator[row._id] = row
+      if (!accumulator[row._id!]) {
+        accumulator[row._id!] = row
       }
       return accumulator
     }, optionsObj || {})
@@ -92,24 +117,32 @@
   }
   $: {
     // We don't want to reorder while the dropdown is open, to avoid UX jumps
-    if (!open) {
-      enrichedOptions = enrichedOptions.sort((a, b) => {
+    if (!open && primaryDisplay) {
+      enrichedOptions = enrichedOptions.sort((a: OptionObj, b: OptionObj) => {
         const selectedValues = flatten(fieldState?.value) || []
 
-        const aIsSelected = selectedValues.find(v => v === a._id)
-        const bIsSelected = selectedValues.find(v => v === b._id)
+        const aIsSelected = selectedValues.find(
+          (v: RelationshipValue) => v === a._id
+        )
+        const bIsSelected = selectedValues.find(
+          (v: RelationshipValue) => v === b._id
+        )
         if (aIsSelected && !bIsSelected) {
           return -1
         } else if (!aIsSelected && bIsSelected) {
           return 1
         }
 
-        return a[primaryDisplay] > b[primaryDisplay]
+        return (a[primaryDisplay] > b[primaryDisplay]) as unknown as number
       })
     }
   }
 
-  $: forceFetchRows(filter)
+  $: {
+    if (filter || defaultValue) {
+      forceFetchRows()
+    }
+  }
   $: debouncedFetchRows(searchTerm, primaryDisplay, defaultValue)
 
   const forceFetchRows = async () => {
@@ -119,7 +152,11 @@
     selectedValue = []
     debouncedFetchRows(searchTerm, primaryDisplay, defaultValue)
   }
-  const fetchRows = async (searchTerm, primaryDisplay, defaultVal) => {
+  async function fetchRows(
+    searchTerm: any,
+    primaryDisplay: string,
+    defaultVal: string | string[]
+  ) {
     const allRowsFetched =
       $fetch.loaded &&
       !Object.keys($fetch.query?.string || {}).length &&
@@ -129,17 +166,39 @@
       return
     }
     // must be an array
-    if (defaultVal && !Array.isArray(defaultVal)) {
-      defaultVal = defaultVal.split(",")
-    }
-    if (defaultVal && optionsObj && defaultVal.some(val => !optionsObj[val])) {
+    const defaultValArray: string[] = !defaultVal
+      ? []
+      : !Array.isArray(defaultVal)
+      ? defaultVal.split(",")
+      : defaultVal
+
+    if (
+      defaultVal &&
+      optionsObj &&
+      defaultValArray.some(val => !optionsObj[val])
+    ) {
       await fetch.update({
-        query: { oneOf: { _id: defaultVal } },
+        query: { oneOf: { _id: defaultValArray } },
+      })
+    }
+
+    if (
+      (Array.isArray(selectedValue) &&
+        selectedValue.some(val => !optionsObj[val])) ||
+      (selectedValue && !optionsObj[selectedValue as string])
+    ) {
+      await fetch.update({
+        query: {
+          oneOf: {
+            _id: Array.isArray(selectedValue) ? selectedValue : [selectedValue],
+          },
+        },
       })
     }
 
     // Ensure we match all filters, rather than any
-    const baseFilter = (filter || []).filter(x => x.operator !== "allOr")
+    // @ts-expect-error this doesn't fit types, but don't want to change it yet
+    const baseFilter: any = (filter || []).filter(x => x.operator !== "allOr")
     await fetch.update({
       filter: [
         ...baseFilter,
@@ -152,9 +211,8 @@
       ],
     })
   }
-  const debouncedFetchRows = Utils.debounce(fetchRows, 250)
 
-  const flatten = values => {
+  const flatten = (values: any | any[]) => {
     if (!values) {
       return []
     }
@@ -162,17 +220,17 @@
     if (!Array.isArray(values)) {
       values = [values]
     }
-    values = values.map(value =>
+    values = values.map((value: any) =>
       typeof value === "object" ? value._id : value
     )
     return values
   }
 
-  const getDisplayName = row => {
-    return row?.[primaryDisplay] || "-"
+  const getDisplayName = (row: Row) => {
+    return row?.[primaryDisplay!] || "-"
   }
 
-  const handleChange = e => {
+  const handleChange = (e: any) => {
     let value = e.detail
     if (!multiselect) {
       value = value == null ? [] : [value]
@@ -220,13 +278,12 @@
       this={component}
       options={enrichedOptions}
       {autocomplete}
-      value={selectedValue}
+      value={castSelectedValue}
       on:change={handleChange}
       on:loadMore={loadMore}
       id={fieldState.fieldId}
       disabled={fieldState.disabled}
       readonly={fieldState.readonly}
-      error={fieldState.error}
       getOptionLabel={getDisplayName}
       getOptionValue={option => option._id}
       {placeholder}

packages/frontend-core/src/components/grid/stores/conditions.ts

@@ -31,7 +31,7 @@ export const deriveStores = (context: StoreContext): ConditionDerivedStore => {
   // Derive and memoize the cell conditions present in our columns so that we
   // only recompute condition metadata when absolutely necessary
   const conditions = derivedMemo(columns, $columns => {
-    let newConditions = []
+    let newConditions: UICondition[] = []
     for (let column of $columns) {
       for (let condition of column.conditions || []) {
         newConditions.push({

packages/server/src/api/routes/backup.ts

@@ -2,12 +2,14 @@ import Router from "@koa/router"
 import * as controller from "../controllers/backup"
 import authorized from "../../middleware/authorized"
 import { permissions } from "@budibase/backend-core"
+import ensureTenantAppOwnership from "../../middleware/ensureTenantAppOwnership"
 
 const router: Router = new Router()
 
 router.post(
   "/api/backups/export",
   authorized(permissions.BUILDER),
+  ensureTenantAppOwnership,
   controller.exportAppDump
 )
 

packages/server/src/automations/tests/steps/cron-automations.spec.ts

@@ -1,45 +0,0 @@
-import tk from "timekeeper"
-import "../../../environment"
-import * as automations from "../../index"
-import TestConfiguration from "../../../tests/utilities/TestConfiguration"
-import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
-
-const initialTime = Date.now()
-tk.freeze(initialTime)
-
-const oneMinuteInMs = 60 * 1000
-
-describe("cron automations", () => {
-  const config = new TestConfiguration()
-
-  beforeAll(async () => {
-    await automations.init()
-    await config.init()
-  })
-
-  afterAll(async () => {
-    await automations.shutdown()
-    config.end()
-  })
-
-  beforeEach(() => {
-    tk.freeze(initialTime)
-  })
-
-  it("should initialise the automation timestamp", async () => {
-    await createAutomationBuilder(config).onCron({ cron: "* * * * *" }).save()
-
-    tk.travel(Date.now() + oneMinuteInMs)
-    await config.publish()
-
-    const { data } = await config.getAutomationLogs()
-    expect(data).toHaveLength(1)
-    expect(data).toEqual([
-      expect.objectContaining({
-        trigger: expect.objectContaining({
-          outputs: { timestamp: initialTime + oneMinuteInMs },
-        }),
-      }),
-    ])
-  })
-})

packages/server/src/automations/tests/triggers/cron.spec.ts

@@ -1,6 +1,11 @@
 import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
-import { captureAutomationResults } from "../utilities"
+import {
+  captureAutomationQueueMessages,
+  captureAutomationResults,
+} from "../utilities"
+import { automations } from "@budibase/pro"
+import { AutomationStatus } from "@budibase/types"
 
 describe("cron trigger", () => {
   const config = new TestConfiguration()
@@ -13,6 +18,13 @@ describe("cron trigger", () => {
     config.end()
   })
 
+  beforeEach(async () => {
+    const { automations } = await config.api.automation.fetch()
+    for (const automation of automations) {
+      await config.api.automation.delete(automation)
+    }
+  })
+
   it("should queue a Bull cron job", async () => {
     const { automation } = await createAutomationBuilder(config)
       .onCron({ cron: "* * * * *" })
@@ -21,12 +33,12 @@ describe("cron trigger", () => {
       })
       .save()
 
-    const jobs = await captureAutomationResults(automation, () =>
+    const messages = await captureAutomationQueueMessages(automation, () =>
       config.api.application.publish()
     )
-    expect(jobs).toHaveLength(1)
+    expect(messages).toHaveLength(1)
 
-    const repeat = jobs[0].opts?.repeat
+    const repeat = messages[0].opts?.repeat
     if (!repeat || !("cron" in repeat)) {
       throw new Error("Expected cron repeat")
     }
@@ -49,4 +61,82 @@ describe("cron trigger", () => {
       },
     })
   })
+
+  it("should stop if the job fails more than 3 times", async () => {
+    const runner = await createAutomationBuilder(config)
+      .onCron({ cron: "* * * * *" })
+      .queryRows({
+        // @ts-expect-error intentionally sending invalid data
+        tableId: null,
+      })
+      .save()
+
+    await config.api.application.publish()
+
+    const results = await captureAutomationResults(
+      runner.automation,
+      async () => {
+        await runner.trigger({ timeout: 1000, fields: {} })
+        await runner.trigger({ timeout: 1000, fields: {} })
+        await runner.trigger({ timeout: 1000, fields: {} })
+        await runner.trigger({ timeout: 1000, fields: {} })
+        await runner.trigger({ timeout: 1000, fields: {} })
+      }
+    )
+
+    expect(results).toHaveLength(5)
+
+    await config.withProdApp(async () => {
+      const {
+        data: [latest, ..._],
+      } = await automations.logs.logSearch({
+        automationId: runner.automation._id,
+      })
+      expect(latest.status).toEqual(AutomationStatus.STOPPED_ERROR)
+    })
+  })
+
+  it("should fill in the timestamp if one is not provided", async () => {
+    const runner = await createAutomationBuilder(config)
+      .onCron({ cron: "* * * * *" })
+      .serverLog({
+        text: "Hello, world!",
+      })
+      .save()
+
+    await config.api.application.publish()
+
+    const results = await captureAutomationResults(
+      runner.automation,
+      async () => {
+        await runner.trigger({ timeout: 1000, fields: {} })
+      }
+    )
+    expect(results).toHaveLength(1)
+    expect(results[0].data.event.timestamp).toBeWithin(
+      Date.now() - 1000,
+      Date.now() + 1000
+    )
+  })
+
+  it("should use the given timestamp if one is given", async () => {
+    const timestamp = 1234
+    const runner = await createAutomationBuilder(config)
+      .onCron({ cron: "* * * * *" })
+      .serverLog({
+        text: "Hello, world!",
+      })
+      .save()
+
+    await config.api.application.publish()
+
+    const results = await captureAutomationResults(
+      runner.automation,
+      async () => {
+        await runner.trigger({ timeout: 1000, fields: {}, timestamp })
+      }
+    )
+    expect(results).toHaveLength(1)
+    expect(results[0].data.event.timestamp).toEqual(timestamp)
+  })
 })

packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts

@@ -223,10 +223,34 @@ class AutomationRunner<TStep extends AutomationTriggerStepId> {
   async trigger(
     request: TriggerAutomationRequest
   ): Promise<TriggerAutomationResponse> {
-    return await this.config.api.automation.trigger(
-      this.automation._id!,
-      request
-    )
+    if (!this.config.prodAppId) {
+      throw new Error(
+        "Automations can only be triggered in a production app context, call config.api.application.publish()"
+      )
+    }
+    // Because you can only trigger automations in a production app context, we
+    // wrap the trigger call to make tests a bit cleaner. If you really want to
+    // test triggering an automation in a dev app context, you can use the
+    // automation API directly.
+    return await this.config.withProdApp(async () => {
+      try {
+        return await this.config.api.automation.trigger(
+          this.automation._id!,
+          request
+        )
+      } catch (e: any) {
+        if (e.cause.status === 404) {
+          throw new Error(
+            `Automation with ID ${
+              this.automation._id
+            } not found in app ${this.config.getAppId()}. You may have forgotten to call config.api.application.publish().`,
+            { cause: e }
+          )
+        } else {
+          throw e
+        }
+      }
+    })
   }
 }
 

packages/server/src/automations/tests/utilities/index.ts

@@ -34,6 +34,42 @@ export async function runInProd(fn: any) {
   }
 }
 
+export async function captureAllAutomationQueueMessages(
+  f: () => Promise<unknown>
+) {
+  const messages: Job<AutomationData>[] = []
+  const queue = getQueue()
+
+  const messageListener = async (message: Job<AutomationData>) => {
+    messages.push(message)
+  }
+
+  queue.on("message", messageListener)
+  try {
+    await f()
+    // Queue messages tend to be send asynchronously in API handlers, so there's
+    // no guarantee that awaiting this function will have queued anything yet.
+    // We wait here to make sure we're queued _after_ any existing async work.
+    await helpers.wait(100)
+  } finally {
+    queue.off("message", messageListener)
+  }
+
+  return messages
+}
+
+export async function captureAutomationQueueMessages(
+  automation: Automation | string,
+  f: () => Promise<unknown>
+) {
+  const messages = await captureAllAutomationQueueMessages(f)
+  return messages.filter(
+    m =>
+      m.data.automation._id ===
+      (typeof automation === "string" ? automation : automation._id)
+  )
+}
+
 /**
  * Capture all automation runs that occur during the execution of a function.
  * This function will wait for all messages to be processed before returning.
@@ -43,14 +79,18 @@ export async function captureAllAutomationResults(
 ): Promise<Job<AutomationData>[]> {
   const runs: Job<AutomationData>[] = []
   const queue = getQueue()
-  let messagesReceived = 0
+  let messagesOutstanding = 0
 
   const completedListener = async (job: Job<AutomationData>) => {
     runs.push(job)
-    messagesReceived--
+    messagesOutstanding--
   }
-  const messageListener = async () => {
-    messagesReceived++
+  const messageListener = async (message: Job<AutomationData>) => {
+    // Don't count cron messages, as they don't get triggered automatically.
+    if (message.opts?.repeat != null) {
+      return
+    }
+    messagesOutstanding++
   }
   queue.on("message", messageListener)
   queue.on("completed", completedListener)
@@ -61,9 +101,18 @@ export async function captureAllAutomationResults(
     // We wait here to make sure we're queued _after_ any existing async work.
     await helpers.wait(100)
   } finally {
+    const waitMax = 10000
+    let waited = 0
     // eslint-disable-next-line no-unmodified-loop-condition
-    while (messagesReceived > 0) {
+    while (messagesOutstanding > 0) {
       await helpers.wait(50)
+      waited += 50
+      if (waited > waitMax) {
+        // eslint-disable-next-line no-unsafe-finally
+        throw new Error(
+          `Timed out waiting for automation runs to complete. ${messagesOutstanding} messages waiting for completion.`
+        )
+      }
     }
     queue.off("completed", completedListener)
     queue.off("message", messageListener)

packages/server/src/automations/utils.ts

@@ -63,7 +63,7 @@ export async function processEvent(job: AutomationJob) {
 
       const task = async () => {
         try {
-          if (isCronTrigger(job.data.automation)) {
+          if (isCronTrigger(job.data.automation) && !job.data.event.timestamp) {
             // Requires the timestamp at run time
             job.data.event.timestamp = Date.now()
           }

packages/server/src/middleware/ensureTenantAppOwnership.ts

@@ -0,0 +1,19 @@
+import { tenancy, utils, context } from "@budibase/backend-core"
+import { UserCtx } from "@budibase/types"
+
+async function ensureTenantAppOwnership(ctx: UserCtx, next: any) {
+  const appId = await utils.getAppIdFromCtx(ctx)
+  if (!appId) {
+    ctx.throw(400, "appId must be provided")
+  }
+
+  const appTenantId = context.getTenantIDFromAppID(appId)
+  const tenantId = tenancy.getTenantId()
+
+  if (appTenantId !== tenantId) {
+    ctx.throw(403, "Unauthorized")
+  }
+  await next()
+}
+
+export default ensureTenantAppOwnership

packages/server/src/middleware/tests/ensureTenantAppOwnership.spec.js

@@ -0,0 +1,75 @@
+import ensureTenantAppOwnership from "../ensureTenantAppOwnership"
+import { tenancy, utils } from "@budibase/backend-core"
+
+jest.mock("@budibase/backend-core", () => ({
+  ...jest.requireActual("@budibase/backend-core"),
+  tenancy: {
+    getTenantId: jest.fn(),
+  },
+  utils: {
+    getAppIdFromCtx: jest.fn(),
+  },
+}))
+
+class TestConfiguration {
+  constructor(appId = "tenant_1") {
+    this.next = jest.fn()
+    this.throw = jest.fn()
+    this.middleware = ensureTenantAppOwnership
+
+    this.ctx = {
+      next: this.next,
+      throw: this.throw,
+    }
+
+    utils.getAppIdFromCtx.mockResolvedValue(appId)
+  }
+
+  async executeMiddleware() {
+    return this.middleware(this.ctx, this.next)
+  }
+
+  afterEach() {
+    jest.clearAllMocks()
+  }
+}
+
+describe("Ensure Tenant Ownership Middleware", () => {
+  let config
+
+  beforeEach(() => {
+    config = new TestConfiguration()
+  })
+
+  afterEach(() => {
+    config.afterEach()
+  })
+
+  it("calls next() when appId matches tenant ID", async () => {
+    tenancy.getTenantId.mockReturnValue("tenant_1")
+
+    await config.executeMiddleware()
+
+    expect(utils.getAppIdFromCtx).toHaveBeenCalledWith(config.ctx)
+    expect(config.next).toHaveBeenCalled()
+  })
+
+  it("throws when tenant appId does not match tenant ID", async () => {
+    const appId = "app_dev_tenant3_fce449c4d75b4e4a9c7a6980d82a3e22"
+    utils.getAppIdFromCtx.mockResolvedValue(appId)
+    tenancy.getTenantId.mockReturnValue("tenant_2")
+
+    await config.executeMiddleware()
+
+    expect(utils.getAppIdFromCtx).toHaveBeenCalledWith(config.ctx)
+    expect(config.throw).toHaveBeenCalledWith(403, "Unauthorized")
+  })
+
+  it("throws 400 when appId is missing", async () => {
+    utils.getAppIdFromCtx.mockResolvedValue(null)
+
+    await config.executeMiddleware()
+
+    expect(config.throw).toHaveBeenCalledWith(400, "appId must be provided")
+  })
+})

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -262,11 +262,13 @@ export default class TestConfiguration {
   async withApp<R>(app: App | string, f: () => Promise<R>) {
     const oldAppId = this.appId
     this.appId = typeof app === "string" ? app : app.appId
-    try {
-      return await f()
-    } finally {
-      this.appId = oldAppId
-    }
+    return await context.doInAppContext(this.appId, async () => {
+      try {
+        return await f()
+      } finally {
+        this.appId = oldAppId
+      }
+    })
   }
 
   async withProdApp<R>(f: () => Promise<R>) {

packages/server/src/threads/automation.ts

@@ -155,23 +155,12 @@ class Orchestrator {
     return step
   }
 
-  async getMetadata(): Promise<AutomationMetadata> {
-    const metadataId = generateAutomationMetadataID(this.automation._id!)
-    const db = context.getAppDB()
-    let metadata: AutomationMetadata
-    try {
-      metadata = await db.get(metadataId)
-    } catch (err) {
-      metadata = {
-        _id: metadataId,
-        errorCount: 0,
-      }
-    }
-    return metadata
+  isCron(): boolean {
+    return isRecurring(this.automation)
   }
 
   async stopCron(reason: string) {
-    if (!this.job.opts.repeat) {
+    if (!this.isCron()) {
       return
     }
     logging.logWarn(
@@ -192,44 +181,42 @@ class Orchestrator {
     await storeLog(automation, this.executionOutput)
   }
 
-  async checkIfShouldStop(metadata: AutomationMetadata): Promise<boolean> {
-    if (!metadata.errorCount || !this.job.opts.repeat) {
+  async checkIfShouldStop(): Promise<boolean> {
+    const metadata = await this.getMetadata()
+    if (!metadata.errorCount || !this.isCron()) {
       return false
     }
     if (metadata.errorCount >= MAX_AUTOMATION_RECURRING_ERRORS) {
-      await this.stopCron("errors")
       return true
     }
     return false
   }
 
-  async updateMetadata(metadata: AutomationMetadata) {
-    const output = this.executionOutput,
-      automation = this.automation
-    if (!output || !isRecurring(automation)) {
-      return
-    }
-    const count = metadata.errorCount
-    const isError = isErrorInOutput(output)
-    // nothing to do in this scenario, escape
-    if (!count && !isError) {
-      return
-    }
-    if (isError) {
-      metadata.errorCount = count ? count + 1 : 1
-    } else {
-      metadata.errorCount = 0
-    }
+  async getMetadata(): Promise<AutomationMetadata> {
+    const metadataId = generateAutomationMetadataID(this.automation._id!)
     const db = context.getAppDB()
-    try {
-      await db.put(metadata)
-    } catch (err) {
-      logging.logAlertWithInfo(
-        "Failed to write automation metadata",
-        db.name,
-        automation._id!,
-        err
-      )
+    const metadata = await db.tryGet<AutomationMetadata>(metadataId)
+    return metadata || { _id: metadataId, errorCount: 0 }
+  }
+
+  async incrementErrorCount() {
+    for (let attempt = 0; attempt < 3; attempt++) {
+      const metadata = await this.getMetadata()
+      metadata.errorCount ||= 0
+      metadata.errorCount++
+
+      const db = context.getAppDB()
+      try {
+        await db.put(metadata)
+        return
+      } catch (err) {
+        logging.logAlertWithInfo(
+          "Failed to update error count in automation metadata",
+          db.name,
+          this.automation._id!,
+          err
+        )
+      }
     }
   }
 
@@ -293,18 +280,6 @@ class Orchestrator {
         await enrichBaseContext(this.context)
         this.context.user = this.currentUser
 
-        let metadata
-
-        // check if this is a recurring automation,
-        if (isProdAppID(this.appId) && isRecurring(this.automation)) {
-          span?.addTags({ recurring: true })
-          metadata = await this.getMetadata()
-          const shouldStop = await this.checkIfShouldStop(metadata)
-          if (shouldStop) {
-            span?.addTags({ shouldStop: true })
-            return
-          }
-        }
         const start = performance.now()
 
         await this.executeSteps(this.automation.definition.steps)
@@ -332,10 +307,15 @@ class Orchestrator {
         }
         if (
           isProdAppID(this.appId) &&
-          isRecurring(this.automation) &&
-          metadata
+          this.isCron() &&
+          isErrorInOutput(this.executionOutput)
         ) {
-          await this.updateMetadata(metadata)
+          await this.incrementErrorCount()
+          if (await this.checkIfShouldStop()) {
+            await this.stopCron("errors")
+            span?.addTags({ shouldStop: true })
+            return
+          }
         }
         return this.executionOutput
       }

packages/types/src/api/web/app/automation.ts

@@ -66,6 +66,7 @@ export interface ClearAutomationLogResponse {
 
 export interface TriggerAutomationRequest {
   fields: Record<string, any>
+  timestamp?: number
   // time in seconds
   timeout: number
 }

packages/types/src/ui/bindings/helper.ts

@@ -1,4 +1,6 @@
 export interface Helper {
   example: string
   description: string
+  args: any[]
+  requiresBlock?: boolean
 }

packages/types/src/ui/components/index.ts

@@ -48,6 +48,11 @@ export interface ComponentSetting {
   selectAllFields?: boolean
   resetOn?: string | string[]
   settings?: ComponentSetting[]
+  nested?: boolean
   dependsOn?: DependsOnComponentSetting
   sectionDependsOn?: DependsOnComponentSetting
+  contextAccess?: {
+    global: boolean
+    self: boolean
+  }
 }

packages/types/src/ui/stores/grid/columns.ts

@@ -1,9 +1,15 @@
-import { CalculationType, FieldSchema, FieldType, UIRow } from "@budibase/types"
+import {
+  CalculationType,
+  FieldSchema,
+  FieldType,
+  UICondition,
+  UIRow,
+} from "@budibase/types"
 
 export type UIColumn = FieldSchema & {
   label: string
   readonly: boolean
-  conditions: any
+  conditions?: UICondition[]
   format?: (row: UIRow) => any
   related?: {
     field: string

packages/types/src/ui/stores/grid/condition.ts

@@ -3,7 +3,7 @@ import { FieldType, SearchFilter } from "@budibase/types"
 export interface UICondition {
   column: string
   type: FieldType
-  referenceValue: string
+  referenceValue: any
   operator: SearchFilter["operator"]
   metadataKey: string
   metadataValue: string