Merge branch 'master' into BUDI-9011

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.4.11",
+  "version": "3.4.12",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -3,6 +3,7 @@ import { newid } from "../utils"
 import { Queue, QueueOptions, JobOptions } from "./queue"
 import { helpers } from "@budibase/shared-core"
 import { Job, JobId, JobInformation } from "bull"
+import { cloneDeep } from "lodash"
 
 function jobToJobInformation(job: Job): JobInformation {
   let cron = ""
@@ -33,12 +34,13 @@ function jobToJobInformation(job: Job): JobInformation {
   }
 }
 
-interface JobMessage<T = any> extends Partial<Job<T>> {
+export interface TestQueueMessage<T = any> extends Partial<Job<T>> {
   id: string
   timestamp: number
   queue: Queue<T>
   data: any
   opts?: JobOptions
+  manualTrigger?: boolean
 }
 
 /**
@@ -47,15 +49,15 @@ interface JobMessage<T = any> extends Partial<Job<T>> {
  * internally to register when messages are available to the consumers - in can
  * support many inputs and many consumers.
  */
-class InMemoryQueue implements Partial<Queue> {
+export class InMemoryQueue<T = any> implements Partial<Queue<T>> {
   _name: string
   _opts?: QueueOptions
-  _messages: JobMessage[]
+  _messages: TestQueueMessage<T>[]
   _queuedJobIds: Set<string>
   _emitter: NodeJS.EventEmitter<{
-    message: [JobMessage]
+    message: [TestQueueMessage<T>]
-    completed: [Job]
+    completed: [Job<T>]
-    removed: [JobMessage]
+    removed: [TestQueueMessage<T>]
   }>
   _runCount: number
   _addCount: number
@@ -86,10 +88,12 @@ class InMemoryQueue implements Partial<Queue> {
    */
   async process(concurrencyOrFunc: number | any, func?: any) {
     func = typeof concurrencyOrFunc === "number" ? func : concurrencyOrFunc
-    this._emitter.on("message", async message => {
+    this._emitter.on("message", async msg => {
+      const message = cloneDeep(msg)
+
       // For the purpose of testing, don't trigger cron jobs immediately.
       // Require the test to trigger them manually with timestamps.
-      if (message.opts?.repeat != null) {
+      if (!message.manualTrigger && message.opts?.repeat != null) {
         return
       }
 
@@ -107,7 +111,7 @@ class InMemoryQueue implements Partial<Queue> {
       if (resp.then != null) {
         try {
           await retryFunc(resp)
-          this._emitter.emit("completed", message as Job)
+          this._emitter.emit("completed", message as Job<T>)
         } catch (e: any) {
           console.error(e)
         }
@@ -124,7 +128,6 @@ class InMemoryQueue implements Partial<Queue> {
     return this as any
   }
 
-  // simply puts a message to the queue and emits to the queue for processing
   /**
    * Simple function to replicate the add message functionality of Bull, putting
    * a new message on the queue. This then emits an event which will be used to
@@ -133,7 +136,13 @@ class InMemoryQueue implements Partial<Queue> {
    * a JSON message as this is required by Bull.
    * @param repeat serves no purpose for the import queue.
    */
-  async add(data: any, opts?: JobOptions) {
+  async add(data: T | string, optsOrT?: JobOptions | T) {
+    if (typeof data === "string") {
+      throw new Error("doesn't support named jobs")
+    }
+
+    const opts = optsOrT as JobOptions
+
     const jobId = opts?.jobId?.toString()
     if (jobId && this._queuedJobIds.has(jobId)) {
       console.log(`Ignoring already queued job ${jobId}`)
@@ -148,7 +157,7 @@ class InMemoryQueue implements Partial<Queue> {
     }
 
     const pushMessage = () => {
-      const message: JobMessage = {
+      const message: TestQueueMessage = {
         id: newid(),
         timestamp: Date.now(),
         queue: this as unknown as Queue,
@@ -176,7 +185,7 @@ class InMemoryQueue implements Partial<Queue> {
 
   async removeRepeatableByKey(id: string) {
     for (const [idx, message] of this._messages.entries()) {
-      if (message.opts?.jobId?.toString() === id) {
+      if (message.id === id) {
         this._messages.splice(idx, 1)
         this._emitter.emit("removed", message)
         return
@@ -204,6 +213,16 @@ class InMemoryQueue implements Partial<Queue> {
     return null
   }
 
+  manualTrigger(id: JobId) {
+    for (const message of this._messages) {
+      if (message.id === id) {
+        this._emitter.emit("message", { ...message, manualTrigger: true })
+        return
+      }
+    }
+    throw new Error(`Job with id ${id} not found`)
+  }
+
   on(event: string, callback: (...args: any[]) => void): Queue {
     // @ts-expect-error - this callback can be one of many types
     this._emitter.on(event, callback)

packages/backend-core/src/queue/index.ts

@@ -1,2 +1,3 @@
 export * from "./queue"
 export * from "./constants"
+export * from "./inMemoryQueue"

packages/builder/src/components/design/settings/controls/URLVariableTestInput.svelte

@@ -2,7 +2,7 @@
   import { onMount } from "svelte"
   import { Input, Label } from "@budibase/bbui"
   import { previewStore, selectedScreen } from "@/stores/builder"
-  import type { ComponentContext } from "@budibase/types"
+  import type { AppContext } from "@budibase/types"
 
   export let baseRoute = ""
 
@@ -31,7 +31,7 @@
 
   // This function is needed to repopulate the test value from componentContext
   // when a user navigates to another component and then back again
-  const updateTestValueFromContext = (context: ComponentContext | null) => {
+  const updateTestValueFromContext = (context: AppContext | null) => {
     if (context?.url && !testValue) {
       const { wild, ...urlParams } = context.url
       const queryParams = context.query

packages/builder/src/helpers/components.ts

@@ -8,27 +8,32 @@ import {
 } from "@budibase/string-templates"
 import { capitalise } from "@/helpers"
 import { Constants } from "@budibase/frontend-core"
+import { Component, ComponentContext } from "@budibase/types"
 
 const { ContextScopes } = Constants
 
 /**
  * Recursively searches for a specific component ID
  */
-export const findComponent = (rootComponent, id) => {
+export const findComponent = (rootComponent: Component, id: string) => {
   return searchComponentTree(rootComponent, comp => comp._id === id)
 }
 
 /**
  * Recursively searches for a specific component type
  */
-export const findComponentType = (rootComponent, type) => {
+export const findComponentType = (rootComponent: Component, type: string) => {
   return searchComponentTree(rootComponent, comp => comp._component === type)
 }
 
 /**
  * Recursively searches for the parent component of a specific component ID
  */
-export const findComponentParent = (rootComponent, id, parentComponent) => {
+export const findComponentParent = (
+  rootComponent: Component | undefined,
+  id: string | undefined,
+  parentComponent: Component | null = null
+): Component | null => {
   if (!rootComponent || !id) {
     return null
   }
@@ -51,7 +56,11 @@ export const findComponentParent = (rootComponent, id, parentComponent) => {
  * Recursively searches for a specific component ID and records the component
  * path to this component
  */
-export const findComponentPath = (rootComponent, id, path = []) => {
+export const findComponentPath = (
+  rootComponent: Component,
+  id: string | undefined,
+  path: Component[] = []
+): Component[] => {
   if (!rootComponent || !id) {
     return []
   }
@@ -75,11 +84,14 @@ export const findComponentPath = (rootComponent, id, path = []) => {
  * Recurses through the component tree and finds all components which match
  * a certain selector
  */
-export const findAllMatchingComponents = (rootComponent, selector) => {
+export const findAllMatchingComponents = (
+  rootComponent: Component | null,
+  selector: (component: Component) => boolean
+) => {
   if (!rootComponent || !selector) {
     return []
   }
-  let components = []
+  let components: Component[] = []
   if (rootComponent._children) {
     rootComponent._children.forEach(child => {
       components = [
@@ -97,7 +109,7 @@ export const findAllMatchingComponents = (rootComponent, selector) => {
 /**
  * Recurses through the component tree and finds all components.
  */
-export const findAllComponents = rootComponent => {
+export const findAllComponents = (rootComponent: Component) => {
   return findAllMatchingComponents(rootComponent, () => true)
 }
 
@@ -105,9 +117,9 @@ export const findAllComponents = rootComponent => {
  * Finds the closest parent component which matches certain criteria
  */
 export const findClosestMatchingComponent = (
-  rootComponent,
+  rootComponent: Component,
-  componentId,
+  componentId: string | undefined,
-  selector
+  selector: (component: Component) => boolean
 ) => {
   if (!selector) {
     return null
@@ -125,7 +137,10 @@ export const findClosestMatchingComponent = (
  * Recurses through a component tree evaluating a matching function against
  * components until a match is found
  */
-const searchComponentTree = (rootComponent, matchComponent) => {
+const searchComponentTree = (
+  rootComponent: Component,
+  matchComponent: (component: Component) => boolean
+): Component | null => {
   if (!rootComponent || !matchComponent) {
     return null
   }
@@ -150,15 +165,18 @@ const searchComponentTree = (rootComponent, matchComponent) => {
  * This mutates the object in place.
  * @param component the component to randomise
  */
-export const makeComponentUnique = component => {
+export const makeComponentUnique = (component: Component) => {
   if (!component) {
     return
   }
 
   // Generate a full set of component ID replacements in this tree
-  const idReplacements = []
+  const idReplacements: [string, string][] = []
-  const generateIdReplacements = (component, replacements) => {
+  const generateIdReplacements = (
-    const oldId = component._id
+    component: Component,
+    replacements: [string, string][]
+  ) => {
+    const oldId = component._id!
     const newId = Helpers.uuid()
     replacements.push([oldId, newId])
     component._children?.forEach(x => generateIdReplacements(x, replacements))
@@ -182,9 +200,9 @@ export const makeComponentUnique = component => {
     let js = decodeJSBinding(sanitizedBinding)
     if (js != null) {
       // Replace ID inside JS binding
-      idReplacements.forEach(([oldId, newId]) => {
+      for (const [oldId, newId] of idReplacements) {
         js = js.replace(new RegExp(oldId, "g"), newId)
-      })
+      }
 
       // Create new valid JS binding
       let newBinding = encodeJSBinding(js)
@@ -204,7 +222,7 @@ export const makeComponentUnique = component => {
   return JSON.parse(definition)
 }
 
-export const getComponentText = component => {
+export const getComponentText = (component: Component) => {
   if (component == null) {
     return ""
   }
@@ -218,7 +236,7 @@ export const getComponentText = component => {
   return capitalise(type)
 }
 
-export const getComponentName = component => {
+export const getComponentName = (component: Component) => {
   if (component == null) {
     return ""
   }
@@ -229,9 +247,9 @@ export const getComponentName = component => {
 }
 
 // Gets all contexts exposed by a certain component type, including actions
-export const getComponentContexts = component => {
+export const getComponentContexts = (component: string) => {
   const def = componentStore.getDefinition(component)
-  let contexts = []
+  let contexts: ComponentContext[] = []
   if (def?.context) {
     contexts = Array.isArray(def.context) ? [...def.context] : [def.context]
   }
@@ -251,9 +269,9 @@ export const getComponentContexts = component => {
  * Recurses through the component tree and builds a tree of contexts provided
  * by components.
  */
-export const buildContextTree = (
+const buildContextTree = (
-  rootComponent,
+  rootComponent: Component,
-  tree = { root: [] },
+  tree: Record<string, string[]> = { root: [] },
   currentBranch = "root"
 ) => {
   // Sanity check
@@ -264,12 +282,12 @@ export const buildContextTree = (
   // Process this component's contexts
   const contexts = getComponentContexts(rootComponent._component)
   if (contexts.length) {
-    tree[currentBranch].push(rootComponent._id)
+    tree[currentBranch].push(rootComponent._id!)
 
     // If we provide local context, start a new branch for our children
     if (contexts.some(context => context.scope === ContextScopes.Local)) {
-      currentBranch = rootComponent._id
+      currentBranch = rootComponent._id!
-      tree[rootComponent._id] = []
+      tree[rootComponent._id!] = []
     }
   }
 
@@ -287,9 +305,9 @@ export const buildContextTree = (
  * Generates a lookup map of which context branch all components in a component
  * tree are inside.
  */
-export const buildContextTreeLookupMap = rootComponent => {
+export const buildContextTreeLookupMap = (rootComponent: Component) => {
   const tree = buildContextTree(rootComponent)
-  let map = {}
+  const map: Record<string, string> = {}
   Object.entries(tree).forEach(([branch, ids]) => {
     ids.forEach(id => {
       map[id] = branch
@@ -299,9 +317,9 @@ export const buildContextTreeLookupMap = rootComponent => {
 }
 
 // Get a flat list of ids for all descendants of a component
-export const getChildIdsForComponent = component => {
+export const getChildIdsForComponent = (component: Component): string[] => {
   return [
-    component._id,
+    component._id!,
     ...(component?._children ?? []).map(getChildIdsForComponent).flat(1),
   ]
 }

packages/builder/src/stores/builder/componentTreeNodes.ts

@@ -58,7 +58,7 @@ export class ComponentTreeNodesStore extends BudiStore<OpenNodesState> {
 
     const path = findComponentPath(selectedScreen.props, componentId)
 
-    const componentIds = path.map((component: Component) => component._id)
+    const componentIds = path.map((component: Component) => component._id!)
 
     this.update((openNodes: OpenNodesState) => {
       const newNodes = Object.fromEntries(

packages/builder/src/stores/builder/components.ts

@@ -1,3 +1,5 @@
+// TODO: analise and fix all the undefined ! and ?
+
 import { get, derived } from "svelte/store"
 import { cloneDeep } from "lodash/fp"
 import { API } from "@/api"
@@ -36,7 +38,7 @@ import { Utils } from "@budibase/frontend-core"
 import {
   ComponentDefinition,
   ComponentSetting,
-  Component as ComponentType,
+  Component,
   ComponentCondition,
   FieldType,
   Screen,
@@ -45,10 +47,6 @@ import {
 import { utils } from "@budibase/shared-core"
 import { getSequentialName } from "@/helpers/duplicate"
 
-interface Component extends ComponentType {
-  _id: string
-}
-
 export interface ComponentState {
   components: Record<string, ComponentDefinition>
   customComponents: string[]
@@ -182,7 +180,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
    * Takes an enriched component instance and applies any required migration
    * logic
    */
-  migrateSettings(enrichedComponent: Component) {
+  migrateSettings(enrichedComponent: Component | null) {
     const componentPrefix = "@budibase/standard-components"
     let migrated = false
 
@@ -232,7 +230,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
 
   enrichEmptySettings(
     component: Component,
-    opts: { screen?: Screen; parent?: Component; useDefaultValues?: boolean }
+    opts: { screen?: Screen; parent?: string; useDefaultValues?: boolean }
   ) {
     if (!component?._component) {
       return
@@ -240,7 +238,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
     const defaultDS = this.getDefaultDatasource()
     const settings = this.getComponentSettings(component._component)
     const { parent, screen, useDefaultValues } = opts || {}
-    const treeId = parent?._id || component._id
+    const treeId = parent || component._id
     if (!screen) {
       return
     }
@@ -425,7 +423,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
   createInstance(
     componentType: string,
     presetProps?: Record<string, any>,
-    parent?: Component
+    parent?: string
   ): Component | null {
     const screen = get(selectedScreen)
     if (!screen) {
@@ -503,7 +501,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
   async create(
     componentType: string,
     presetProps?: Record<string, any>,
-    parent?: Component,
+    parent?: string,
     index?: number
   ) {
     const state = get(this.store)
@@ -519,7 +517,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
     // Insert in position if specified
     if (parent && index != null) {
       await screenStore.patch((screen: Screen) => {
-        let parentComponent = findComponent(screen.props, parent)
+        let parentComponent = findComponent(screen.props, parent)!
         if (!parentComponent._children?.length) {
           parentComponent._children = [componentInstance]
         } else {
@@ -538,7 +536,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
         }
         const currentComponent = findComponent(
           screen.props,
-          selectedComponentId
+          selectedComponentId!
         )
         if (!currentComponent) {
           return false
@@ -581,7 +579,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
       return state
     })
 
-    componentTreeNodesStore.makeNodeVisible(componentInstance._id)
+    componentTreeNodesStore.makeNodeVisible(componentInstance._id!)
 
     // Log event
     analytics.captureEvent(Events.COMPONENT_CREATED, {
@@ -633,7 +631,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
     // Determine the next component to select, and select it before deletion
     // to avoid an intermediate state of no component selection
     const state = get(this.store)
-    let nextId = ""
+    let nextId: string | null = ""
     if (state.selectedComponentId === component._id) {
       nextId = this.getNext()
       if (!nextId) {
@@ -646,7 +644,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
         nextId = nextId.replace("-navigation", "-screen")
       }
       this.update(state => {
-        state.selectedComponentId = nextId
+        state.selectedComponentId = nextId ?? undefined
         return state
       })
     }
@@ -654,18 +652,18 @@ export class ComponentStore extends BudiStore<ComponentState> {
     // Patch screen
     await screenStore.patch((screen: Screen) => {
       // Check component exists
-      component = findComponent(screen.props, component._id)
+      const updatedComponent = findComponent(screen.props, component._id!)
-      if (!component) {
+      if (!updatedComponent) {
         return false
       }
 
       // Check component has a valid parent
-      const parent = findComponentParent(screen.props, component._id)
+      const parent = findComponentParent(screen.props, updatedComponent._id)
       if (!parent) {
         return false
       }
-      parent._children = parent._children.filter(
+      parent._children = parent._children!.filter(
-        (child: Component) => child._id !== component._id
+        (child: Component) => child._id !== updatedComponent._id
       )
     }, null)
   }
@@ -729,7 +727,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
     // Patch screen
     const patch = (screen: Screen) => {
       // Get up to date ref to target
-      targetComponent = findComponent(screen.props, targetComponent._id)
+      targetComponent = findComponent(screen.props, targetComponent!._id!)!
       if (!targetComponent) {
         return false
       }
@@ -743,7 +741,7 @@ export class ComponentStore extends BudiStore<ComponentState> {
       if (!cut) {
         componentToPaste = makeComponentUnique(componentToPaste)
       }
-      newComponentId = componentToPaste._id
+      newComponentId = componentToPaste._id!
 
       // Strip grid position metadata if pasting into a new screen, but keep
       // alignment metadata
@@ -820,8 +818,8 @@ export class ComponentStore extends BudiStore<ComponentState> {
     if (!screen) {
       throw "A valid screen must be selected"
     }
-    const parent = findComponentParent(screen.props, componentId)
+    const parent = findComponentParent(screen.props, componentId)!
-    const index = parent?._children.findIndex(
+    const index = parent?._children?.findIndex(
       (x: Component) => x._id === componentId
     )
 
@@ -839,29 +837,29 @@ export class ComponentStore extends BudiStore<ComponentState> {
     }
 
     // If we have siblings above us, choose the sibling or a descendant
-    if (index > 0) {
+    if (index !== undefined && index > 0) {
       // If sibling before us accepts children, and is not collapsed, select a descendant
-      const previousSibling = parent._children[index - 1]
+      const previousSibling = parent._children![index - 1]
       if (
         previousSibling._children?.length &&
-        componentTreeNodesStore.isNodeExpanded(previousSibling._id)
+        componentTreeNodesStore.isNodeExpanded(previousSibling._id!)
       ) {
         let target = previousSibling
         while (
           target._children?.length &&
-          componentTreeNodesStore.isNodeExpanded(target._id)
+          componentTreeNodesStore.isNodeExpanded(target._id!)
         ) {
           target = target._children[target._children.length - 1]
         }
-        return target._id
+        return target._id!
       }
 
       // Otherwise just select sibling
-      return previousSibling._id
+      return previousSibling._id!
     }
 
     // If no siblings above us, select the parent
-    return parent._id
+    return parent._id!
   }
 
   getNext() {
@@ -873,9 +871,9 @@ export class ComponentStore extends BudiStore<ComponentState> {
       throw "A valid screen must be selected"
     }
     const parent = findComponentParent(screen.props, componentId)
-    const index = parent?._children.findIndex(
+    const index = parent?._children?.findIndex(
       (x: Component) => x._id === componentId
-    )
+    )!
 
     // Check for screen and navigation component edge cases
     const screenComponentId = `${screen._id}-screen`
@@ -888,37 +886,38 @@ export class ComponentStore extends BudiStore<ComponentState> {
     if (
       component?._children?.length &&
       (state.selectedComponentId === navComponentId ||
-        componentTreeNodesStore.isNodeExpanded(component._id))
+        componentTreeNodesStore.isNodeExpanded(component._id!))
     ) {
-      return component._children[0]._id
+      return component._children[0]._id!
     } else if (!parent) {
       return null
     }
 
     // Otherwise select the next sibling if we have one
-    if (index < parent._children.length - 1) {
+    if (index < parent._children!.length - 1) {
-      const nextSibling = parent._children[index + 1]
+      const nextSibling = parent._children![index + 1]
-      return nextSibling._id
+      return nextSibling._id!
     }
 
     // Last child, select our parents next sibling
     let target = parent
     let targetParent = findComponentParent(screen.props, target._id)
-    let targetIndex = targetParent?._children.findIndex(
+    let targetIndex = targetParent?._children?.findIndex(
       (child: Component) => child._id === target._id
-    )
+    )!
     while (
       targetParent != null &&
-      targetIndex === targetParent._children?.length - 1
+      targetParent._children &&
+      targetIndex === targetParent._children.length - 1
     ) {
       target = targetParent
       targetParent = findComponentParent(screen.props, target._id)
-      targetIndex = targetParent?._children.findIndex(
+      targetIndex = targetParent?._children!.findIndex(
         (child: Component) => child._id === target._id
-      )
+      )!
     }
     if (targetParent) {
-      return targetParent._children[targetIndex + 1]._id
+      return targetParent._children![targetIndex + 1]._id!
     } else {
       return null
     }
@@ -950,16 +949,16 @@ export class ComponentStore extends BudiStore<ComponentState> {
       const parent = findComponentParent(screen.props, componentId)
 
       // Check we aren't right at the top of the tree
-      const index = parent?._children.findIndex(
+      const index = parent?._children?.findIndex(
         (x: Component) => x._id === componentId
-      )
+      )!
       if (!parent || (index === 0 && parent._id === screen.props._id)) {
         return
       }
 
       // Copy original component and remove it from the parent
-      const originalComponent = cloneDeep(parent._children[index])
+      const originalComponent = cloneDeep(parent._children![index])
-      parent._children = parent._children.filter(
+      parent._children = parent._children!.filter(
         (component: Component) => component._id !== componentId
       )
 
@@ -971,9 +970,9 @@ export class ComponentStore extends BudiStore<ComponentState> {
         const definition = this.getDefinition(previousSibling._component)
         if (
           definition?.hasChildren &&
-          componentTreeNodesStore.isNodeExpanded(previousSibling._id)
+          componentTreeNodesStore.isNodeExpanded(previousSibling._id!)
         ) {
-          previousSibling._children.push(originalComponent)
+          previousSibling._children!.push(originalComponent)
         }
 
         // Otherwise just move component above sibling
@@ -985,11 +984,11 @@ export class ComponentStore extends BudiStore<ComponentState> {
       // If no siblings above us, go above the parent as long as it isn't
       // the screen
       else if (parent._id !== screen.props._id) {
-        const grandParent = findComponentParent(screen.props, parent._id)
+        const grandParent = findComponentParent(screen.props, parent._id)!
-        const parentIndex = grandParent._children.findIndex(
+        const parentIndex = grandParent._children!.findIndex(
           (child: Component) => child._id === parent._id
         )
-        grandParent._children.splice(parentIndex, 0, originalComponent)
+        grandParent._children!.splice(parentIndex, 0, originalComponent)
       }
     }, null)
   }
@@ -1028,9 +1027,9 @@ export class ComponentStore extends BudiStore<ComponentState> {
         const definition = this.getDefinition(nextSibling._component)
         if (
           definition?.hasChildren &&
-          componentTreeNodesStore.isNodeExpanded(nextSibling._id)
+          componentTreeNodesStore.isNodeExpanded(nextSibling._id!)
         ) {
-          nextSibling._children.splice(0, 0, originalComponent)
+          nextSibling._children!.splice(0, 0, originalComponent)
         }
 
         // Otherwise move below next sibling
@@ -1041,11 +1040,11 @@ export class ComponentStore extends BudiStore<ComponentState> {
 
       // Last child, so move below our parent
       else {
-        const grandParent = findComponentParent(screen.props, parent._id)
+        const grandParent = findComponentParent(screen.props, parent._id)!
-        const parentIndex = grandParent._children.findIndex(
+        const parentIndex = grandParent._children!.findIndex(
           (child: Component) => child._id === parent._id
         )
-        grandParent._children.splice(parentIndex + 1, 0, originalComponent)
+        grandParent._children!.splice(parentIndex + 1, 0, originalComponent)
       }
     }, null)
   }
@@ -1208,13 +1207,13 @@ export class ComponentStore extends BudiStore<ComponentState> {
       }
 
       // Replace component with parent
-      const index = oldParentDefinition._children.findIndex(
+      const index = oldParentDefinition._children!.findIndex(
         (component: Component) => component._id === componentId
       )
       if (index === -1) {
         return false
       }
-      oldParentDefinition._children[index] = {
+      oldParentDefinition._children![index] = {
         ...newParentDefinition,
         _children: [definition],
       }

packages/builder/src/stores/builder/preview.ts

@@ -1,6 +1,6 @@
 import { get } from "svelte/store"
 import { BudiStore } from "../BudiStore"
-import { PreviewDevice, ComponentContext } from "@budibase/types"
+import { PreviewDevice, ComponentContext, AppContext } from "@budibase/types"
 
 type PreviewEventHandler = (name: string, payload?: any) => void
 
@@ -8,7 +8,7 @@ interface PreviewState {
   previewDevice: PreviewDevice
   previewEventHandler: PreviewEventHandler | null
   showPreview: boolean
-  selectedComponentContext: ComponentContext | null
+  selectedComponentContext: AppContext | null
 }
 
 const INITIAL_PREVIEW_STATE: PreviewState = {

packages/builder/src/stores/builder/screens.ts

@@ -490,7 +490,7 @@ export class ScreenStore extends BudiStore<ScreenState> {
     // Flatten the recursive component tree
     const components = findAllMatchingComponents(
       screen.props,
-      (x: Component) => x
+      (x: Component) => !!x
     )
 
     // Iterate over all components and run checks

packages/frontend-core/src/constants.ts

@@ -112,10 +112,7 @@ export const EventPublishType = {
   ENV_VAR_UPGRADE_PANEL_OPENED: "environment_variable_upgrade_panel_opened",
 }
 
-export const ContextScopes = {
+export { ComponentContextScopes as ContextScopes } from "@budibase/types"
-  Local: "local",
-  Global: "global",
-}
 
 export const TypeIconMap = {
   [FieldType.STRING]: "Text",

packages/server/src/automations/logging/index.ts

@@ -1,7 +1,7 @@
 import env from "../../environment"
 import { AutomationResults, Automation, App } from "@budibase/types"
 import { automations } from "@budibase/pro"
-import { db as dbUtils } from "@budibase/backend-core"
+import { db as dbUtils, logging } from "@budibase/backend-core"
 import sizeof from "object-sizeof"
 
 const MAX_LOG_SIZE_MB = 5
@@ -32,7 +32,16 @@ export async function storeLog(
   if (bytes / MB_IN_BYTES > MAX_LOG_SIZE_MB) {
     sanitiseResults(results)
   }
+  try {
     await automations.logs.storeLog(automation, results)
+  } catch (e: any) {
+    if (e.status === 413 && e.request?.data) {
+      // if content is too large we shouldn't log it
+      delete e.request.data
+      e.request.data = { message: "removed due to large size" }
+    }
+    logging.logAlert("Error writing automation log", e)
+  }
 }
 
 export async function checkAppMetadata(apps: App[]) {

packages/server/src/automations/tests/steps/loop.spec.ts

@@ -21,6 +21,11 @@ describe("Attempt to run a basic loop automation", () => {
   })
 
   beforeEach(async () => {
+    const { automations } = await config.api.automation.fetch()
+    for (const automation of automations) {
+      await config.api.automation.delete(automation)
+    }
+
     table = await config.api.table.save(basicTable())
     await config.api.row.save(table._id!, {})
   })

packages/server/src/automations/tests/triggers/cron.spec.ts

@@ -1,11 +1,15 @@
 import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 import {
-  captureAutomationQueueMessages,
+  captureAutomationMessages,
+  captureAutomationRemovals,
   captureAutomationResults,
+  triggerCron,
 } from "../utilities"
 import { automations } from "@budibase/pro"
-import { AutomationStatus } from "@budibase/types"
+import { AutomationData, AutomationStatus } from "@budibase/types"
+import { MAX_AUTOMATION_RECURRING_ERRORS } from "../../../constants"
+import { queue } from "@budibase/backend-core"
 
 describe("cron trigger", () => {
   const config = new TestConfiguration()
@@ -33,7 +37,7 @@ describe("cron trigger", () => {
       })
       .save()
 
-    const messages = await captureAutomationQueueMessages(automation, () =>
+    const messages = await captureAutomationMessages(automation, () =>
       config.api.application.publish()
     )
     expect(messages).toHaveLength(1)
@@ -63,7 +67,7 @@ describe("cron trigger", () => {
   })
 
   it("should stop if the job fails more than 3 times", async () => {
-    const runner = await createAutomationBuilder(config)
+    const { automation } = await createAutomationBuilder(config)
       .onCron({ cron: "* * * * *" })
       .queryRows({
         // @ts-expect-error intentionally sending invalid data
@@ -71,28 +75,31 @@ describe("cron trigger", () => {
       })
       .save()
 
-    await config.api.application.publish()
+    const [message] = await captureAutomationMessages(automation, () =>
-
+      config.api.application.publish()
-    const results = await captureAutomationResults(
-      runner.automation,
-      async () => {
-        await runner.trigger({ timeout: 1000, fields: {} })
-        await runner.trigger({ timeout: 1000, fields: {} })
-        await runner.trigger({ timeout: 1000, fields: {} })
-        await runner.trigger({ timeout: 1000, fields: {} })
-        await runner.trigger({ timeout: 1000, fields: {} })
-      }
     )
 
+    await config.withProdApp(async () => {
+      let results: queue.TestQueueMessage<AutomationData>[] = []
+      const removed = await captureAutomationRemovals(automation, async () => {
+        results = await captureAutomationResults(automation, async () => {
+          for (let i = 0; i < MAX_AUTOMATION_RECURRING_ERRORS; i++) {
+            triggerCron(message)
+          }
+        })
+      })
+
+      expect(removed).toHaveLength(1)
+      expect(removed[0].id).toEqual(message.id)
+
       expect(results).toHaveLength(5)
 
-    await config.withProdApp(async () => {
+      const search = await automations.logs.logSearch({
-      const {
+        automationId: automation._id,
-        data: [latest, ..._],
+        status: AutomationStatus.STOPPED_ERROR,
-      } = await automations.logs.logSearch({
-        automationId: runner.automation._id,
       })
-      expect(latest.status).toEqual(AutomationStatus.STOPPED_ERROR)
+      expect(search.data).toHaveLength(1)
+      expect(search.data[0].status).toEqual(AutomationStatus.STOPPED_ERROR)
     })
   })
 

packages/server/src/automations/tests/utilities/index.ts

@@ -6,6 +6,7 @@ import { Knex } from "knex"
 import { getQueue } from "../.."
 import { Job } from "bull"
 import { helpers } from "@budibase/shared-core"
+import { queue } from "@budibase/backend-core"
 
 let config: TestConfiguration
 
@@ -20,6 +21,17 @@ export function afterAll() {
   config.end()
 }
 
+export function getTestQueue(): queue.InMemoryQueue<AutomationData> {
+  return getQueue() as unknown as queue.InMemoryQueue<AutomationData>
+}
+
+export function triggerCron(message: Job<AutomationData>) {
+  if (!message.opts?.repeat || !("cron" in message.opts.repeat)) {
+    throw new Error("Expected cron message")
+  }
+  getTestQueue().manualTrigger(message.id)
+}
+
 export async function runInProd(fn: any) {
   env._set("NODE_ENV", "production")
   let error
@@ -34,9 +46,41 @@ export async function runInProd(fn: any) {
   }
 }
 
-export async function captureAllAutomationQueueMessages(
+export async function captureAllAutomationRemovals(f: () => Promise<unknown>) {
+  const messages: Job<AutomationData>[] = []
+  const queue = getQueue()
+
+  const messageListener = async (message: Job<AutomationData>) => {
+    messages.push(message)
+  }
+
+  queue.on("removed", messageListener)
+  try {
+    await f()
+    // Queue messages tend to be send asynchronously in API handlers, so there's
+    // no guarantee that awaiting this function will have queued anything yet.
+    // We wait here to make sure we're queued _after_ any existing async work.
+    await helpers.wait(100)
+  } finally {
+    queue.off("removed", messageListener)
+  }
+
+  return messages
+}
+
+export async function captureAutomationRemovals(
+  automation: Automation | string,
   f: () => Promise<unknown>
 ) {
+  const messages = await captureAllAutomationRemovals(f)
+  return messages.filter(
+    m =>
+      m.data.automation._id ===
+      (typeof automation === "string" ? automation : automation._id)
+  )
+}
+
+export async function captureAllAutomationMessages(f: () => Promise<unknown>) {
   const messages: Job<AutomationData>[] = []
   const queue = getQueue()
 
@@ -58,11 +102,11 @@ export async function captureAllAutomationQueueMessages(
   return messages
 }
 
-export async function captureAutomationQueueMessages(
+export async function captureAutomationMessages(
   automation: Automation | string,
   f: () => Promise<unknown>
 ) {
-  const messages = await captureAllAutomationQueueMessages(f)
+  const messages = await captureAllAutomationMessages(f)
   return messages.filter(
     m =>
       m.data.automation._id ===
@@ -76,18 +120,22 @@ export async function captureAutomationQueueMessages(
  */
 export async function captureAllAutomationResults(
   f: () => Promise<unknown>
-): Promise<Job<AutomationData>[]> {
+): Promise<queue.TestQueueMessage<AutomationData>[]> {
-  const runs: Job<AutomationData>[] = []
+  const runs: queue.TestQueueMessage<AutomationData>[] = []
   const queue = getQueue()
   let messagesOutstanding = 0
 
-  const completedListener = async (job: Job<AutomationData>) => {
+  const completedListener = async (
+    job: queue.TestQueueMessage<AutomationData>
+  ) => {
     runs.push(job)
     messagesOutstanding--
   }
-  const messageListener = async (message: Job<AutomationData>) => {
+  const messageListener = async (
+    message: queue.TestQueueMessage<AutomationData>
+  ) => {
     // Don't count cron messages, as they don't get triggered automatically.
-    if (message.opts?.repeat != null) {
+    if (!message.manualTrigger && message.opts?.repeat != null) {
       return
     }
     messagesOutstanding++

packages/server/src/threads/automation.ts

@@ -181,17 +181,6 @@ class Orchestrator {
     await storeLog(automation, this.executionOutput)
   }
 
-  async checkIfShouldStop(): Promise<boolean> {
-    const metadata = await this.getMetadata()
-    if (!metadata.errorCount || !this.isCron()) {
-      return false
-    }
-    if (metadata.errorCount >= MAX_AUTOMATION_RECURRING_ERRORS) {
-      return true
-    }
-    return false
-  }
-
   async getMetadata(): Promise<AutomationMetadata> {
     const metadataId = generateAutomationMetadataID(this.automation._id!)
     const db = context.getAppDB()
@@ -200,24 +189,29 @@ class Orchestrator {
   }
 
   async incrementErrorCount() {
-    for (let attempt = 0; attempt < 3; attempt++) {
+    const db = context.getAppDB()
+    let err: Error | undefined = undefined
+    for (let attempt = 0; attempt < 10; attempt++) {
       const metadata = await this.getMetadata()
       metadata.errorCount ||= 0
       metadata.errorCount++
 
-      const db = context.getAppDB()
       try {
         await db.put(metadata)
-        return
+        return metadata.errorCount
-      } catch (err) {
+      } catch (error: any) {
+        err = error
+        await helpers.wait(1000 + Math.random() * 1000)
+      }
+    }
+
     logging.logAlertWithInfo(
       "Failed to update error count in automation metadata",
       db.name,
       this.automation._id!,
       err
     )
-      }
+    return undefined
-    }
   }
 
   updateExecutionOutput(id: string, stepId: string, inputs: any, outputs: any) {
@@ -295,28 +289,22 @@ class Orchestrator {
           }
         )
 
-        try {
+        let errorCount = 0
-          await storeLog(this.automation, this.executionOutput)
-        } catch (e: any) {
-          if (e.status === 413 && e.request?.data) {
-            // if content is too large we shouldn't log it
-            delete e.request.data
-            e.request.data = { message: "removed due to large size" }
-          }
-          logging.logAlert("Error writing automation log", e)
-        }
         if (
           isProdAppID(this.appId) &&
           this.isCron() &&
           isErrorInOutput(this.executionOutput)
         ) {
-          await this.incrementErrorCount()
+          errorCount = (await this.incrementErrorCount()) || 0
-          if (await this.checkIfShouldStop()) {
+        }
+
+        if (errorCount >= MAX_AUTOMATION_RECURRING_ERRORS) {
           await this.stopCron("errors")
           span?.addTags({ shouldStop: true })
-            return
+        } else {
-          }
+          await storeLog(this.automation, this.executionOutput)
         }
+
         return this.executionOutput
       }
     )
@@ -743,7 +731,7 @@ export async function executeInThread(
   })) as AutomationResponse
 }
 
-export const removeStalled = async (job: Job) => {
+export const removeStalled = async (job: Job<AutomationData>) => {
   const appId = job.data.event.appId
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")

packages/types/src/ui/components/index.ts

@@ -28,6 +28,8 @@ export interface ComponentDefinition {
     width: number
     height: number
   }
+  context?: ComponentContext | ComponentContext[]
+  actions?: (string | any)[]
 }
 
 export type DependsOnComponentSetting =
@@ -56,3 +58,28 @@ export interface ComponentSetting {
     self: boolean
   }
 }
+interface ComponentAction {
+  type: string
+  suffix?: string
+}
+
+interface ComponentStaticContextValue {
+  label: string
+  key: string
+  type: string // technically this is a long list of options but there are too many to enumerate
+}
+
+export interface ComponentContext {
+  type: ComponentContextType
+  scope?: ComponentContextScopes
+  actions?: ComponentAction[]
+  suffix?: string
+  values?: ComponentStaticContextValue[]
+}
+
+export type ComponentContextType = "action" | "static" | "schema" | "form"
+
+export const enum ComponentContextScopes {
+  Local = "local",
+  Global = "global",
+}

packages/types/src/ui/stores/preview.ts

@@ -1,2 +1,3 @@
 export type PreviewDevice = "desktop" | "tablet" | "mobile"
-export type ComponentContext = Record<string, any>
+
+export type AppContext = Record<string, any>