Merge remote-tracking branch 'origin/execute-script-v2' into execute-script-v2-frontend

.gitattributes

@@ -0,0 +1 @@
+scripts/resources/minio filter=lfs diff=lfs merge=lfs -text

hosting/nginx.dev.conf

@@ -62,7 +62,6 @@ http {
       proxy_connect_timeout 120s;
       proxy_send_timeout 120s;
       proxy_http_version 1.1;
-
       proxy_set_header Host $host;
       proxy_set_header Connection "";
 

hosting/single/Dockerfile

@@ -1,5 +1,5 @@
 ARG BASEIMG=budibase/couchdb:v3.3.3-sqs-v2.1.1
-FROM node:20-slim as build
+FROM node:20-slim AS build
 
 # install node-gyp dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends g++ make python3 jq
@@ -34,13 +34,13 @@ COPY packages/worker/dist packages/worker/dist
 COPY packages/worker/pm2.config.js packages/worker/pm2.config.js
 
 
-FROM $BASEIMG as runner
+FROM $BASEIMG AS runner
 ARG TARGETARCH
-ENV TARGETARCH $TARGETARCH
+ENV TARGETARCH=$TARGETARCH
 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service)
 # e.g. docker build --build-arg TARGETBUILD=aas ....
 ARG TARGETBUILD=single
-ENV TARGETBUILD $TARGETBUILD
+ENV TARGETBUILD=$TARGETBUILD
 
 # install base dependencies
 RUN apt-get update && \
@@ -67,6 +67,12 @@ RUN mkdir -p /var/log/nginx && \
 
 # setup minio
 WORKDIR /minio
+
+# a 2022 version of minio that supports gateway mode
+COPY scripts/resources/minio /minio
+RUN chmod +x minio
+
+# handles the installation of minio in non-aas environments
 COPY scripts/install-minio.sh ./install.sh
 RUN chmod +x install.sh && ./install.sh
 

hosting/single/runner.sh

@@ -1,53 +1,61 @@
 #!/bin/bash
-declare -a ENV_VARS=("COUCHDB_USER" "COUCHDB_PASSWORD" "DATA_DIR" "MINIO_ACCESS_KEY" "MINIO_SECRET_KEY" "INTERNAL_API_KEY" "JWT_SECRET" "REDIS_PASSWORD")
-declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONMENT" "CLUSTER_PORT" "DEPLOYMENT_ENVIRONMENT" "MINIO_URL" "NODE_ENV" "POSTHOG_TOKEN" "REDIS_URL" "SELF_HOSTED" "WORKER_PORT" "WORKER_URL" "TENANT_FEATURE_FLAGS" "ACCOUNT_PORTAL_URL")
-# Check the env vars set in Dockerfile have come through, AAS seems to drop them
-[[ -z "${APP_PORT}" ]] && export APP_PORT=4001
-[[ -z "${ARCHITECTURE}" ]] && export ARCHITECTURE=amd
-[[ -z "${BUDIBASE_ENVIRONMENT}" ]] && export BUDIBASE_ENVIRONMENT=PRODUCTION
-[[ -z "${CLUSTER_PORT}" ]] && export CLUSTER_PORT=80
-[[ -z "${DEPLOYMENT_ENVIRONMENT}" ]] && export DEPLOYMENT_ENVIRONMENT=docker
-[[ -z "${MINIO_URL}" ]] && [[ -z "${USE_S3}" ]] && export MINIO_URL=http://127.0.0.1:9000
-[[ -z "${NODE_ENV}" ]] && export NODE_ENV=production
-[[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
-[[ -z "${ACCOUNT_PORTAL_URL}" ]] && export ACCOUNT_PORTAL_URL=https://account.budibase.app
-[[ -z "${REDIS_URL}" ]] && export REDIS_URL=127.0.0.1:6379
-[[ -z "${SELF_HOSTED}" ]] && export SELF_HOSTED=1
-[[ -z "${WORKER_PORT}" ]] && export WORKER_PORT=4002
-[[ -z "${WORKER_URL}" ]] && export WORKER_URL=http://127.0.0.1:4002
-[[ -z "${APPS_URL}" ]] && export APPS_URL=http://127.0.0.1:4001
-[[ -z "${SERVER_TOP_LEVEL_PATH}" ]] && export SERVER_TOP_LEVEL_PATH=/app
-#  export CUSTOM_DOMAIN=budi001.custom.com
 
-# Azure App Service customisations
+echo "Starting runner.sh..."
-if [[ "${TARGETBUILD}" = "aas" ]]; then
+
-    export DATA_DIR="${DATA_DIR:-/home}"
+# set defaults for Docker-related variables
-    WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
+export APP_PORT="${APP_PORT:-4001}"
-    /etc/init.d/ssh start
+export ARCHITECTURE="${ARCHITECTURE:-amd}"
-else
+export BUDIBASE_ENVIRONMENT="${BUDIBASE_ENVIRONMENT:-PRODUCTION}"
-    export DATA_DIR=${DATA_DIR:-/data}
+export CLUSTER_PORT="${CLUSTER_PORT:-80}"
+export DEPLOYMENT_ENVIRONMENT="${DEPLOYMENT_ENVIRONMENT:-docker}"
+
+# only set MINIO_URL if neither MINIO_URL nor USE_S3 is set
+if [[ -z "${MINIO_URL}" && -z "${USE_S3}" ]]; then
+  export MINIO_URL="http://127.0.0.1:9000"
 fi
-mkdir -p ${DATA_DIR}
+
-# Mount NFS or GCP Filestore if env vars exist for it
+export NODE_ENV="${NODE_ENV:-production}"
-if [[ ! -z ${FILESHARE_IP} && ! -z ${FILESHARE_NAME} ]]; then
+export POSTHOG_TOKEN="${POSTHOG_TOKEN:-phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU}"
+export ACCOUNT_PORTAL_URL="${ACCOUNT_PORTAL_URL:-https://account.budibase.app}"
+export REDIS_URL="${REDIS_URL:-127.0.0.1:6379}"
+export SELF_HOSTED="${SELF_HOSTED:-1}"
+export WORKER_PORT="${WORKER_PORT:-4002}"
+export WORKER_URL="${WORKER_URL:-http://127.0.0.1:4002}"
+export APPS_URL="${APPS_URL:-http://127.0.0.1:4001}"
+export SERVER_TOP_LEVEL_PATH="${SERVER_TOP_LEVEL_PATH:-/app}"
+
+# set DATA_DIR and ensure the directory exists
+if [[ ${TARGETBUILD} == "aas" ]]; then
+    export DATA_DIR="/home"
+else
+    export DATA_DIR="${DATA_DIR:-/data}"
+fi
+mkdir -p "${DATA_DIR}"
+
+# mount NFS or GCP Filestore if FILESHARE_IP and FILESHARE_NAME are set
+if [[ -n "${FILESHARE_IP}" && -n "${FILESHARE_NAME}" ]]; then
     echo "Mounting NFS share"
     apt update && apt install -y nfs-common nfs-kernel-server
     echo "Mount file share ${FILESHARE_IP}:/${FILESHARE_NAME} to ${DATA_DIR}"
-    mount -o nolock ${FILESHARE_IP}:/${FILESHARE_NAME} ${DATA_DIR}
+    mount -o nolock "${FILESHARE_IP}:/${FILESHARE_NAME}" "${DATA_DIR}"
     echo "Mounting result: $?"
 fi
 
-if [ -f "${DATA_DIR}/.env" ]; then
+# source environment variables from a .env file if it exists in DATA_DIR
-    # Read in the .env file and export the variables
+if [[ -f "${DATA_DIR}/.env" ]]; then
-    for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done
+    set -a  # Automatically export all variables loaded from .env
+    source "${DATA_DIR}/.env"
+    set +a
 fi
-# randomise any unset environment variables
+
-for ENV_VAR in "${ENV_VARS[@]}"
+# randomize any unset sensitive environment variables using uuidgen
-do
+env_vars=(COUCHDB_USER COUCHDB_PASSWORD MINIO_ACCESS_KEY MINIO_SECRET_KEY INTERNAL_API_KEY JWT_SECRET REDIS_PASSWORD)
-    if [[ -z "${!ENV_VAR}" ]]; then
+for var in "${env_vars[@]}"; do
-        eval "export $ENV_VAR=$(uuidgen | sed -e 's/-//g')"
+    if [[ -z "${!var}" ]]; then
+        export "$var"="$(uuidgen | tr -d '-')"
     fi
 done
+
 if [[ -z "${COUCH_DB_URL}" ]]; then
     export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@127.0.0.1:5984
 fi
@@ -58,17 +66,15 @@ fi
 
 if [ ! -f "${DATA_DIR}/.env" ]; then
     touch ${DATA_DIR}/.env
-    for ENV_VAR in "${ENV_VARS[@]}"
+    for ENV_VAR in "${ENV_VARS[@]}"; do
-    do
         temp=$(eval "echo \$$ENV_VAR")
-        echo "$ENV_VAR=$temp" >> ${DATA_DIR}/.env
+        echo "$ENV_VAR=$temp" >>${DATA_DIR}/.env
     done
-    for ENV_VAR in "${DOCKER_VARS[@]}"
+    for ENV_VAR in "${DOCKER_VARS[@]}"; do
-    do
         temp=$(eval "echo \$$ENV_VAR")
-        echo "$ENV_VAR=$temp" >> ${DATA_DIR}/.env
+        echo "$ENV_VAR=$temp" >>${DATA_DIR}/.env
     done
-    echo "COUCH_DB_URL=${COUCH_DB_URL}" >> ${DATA_DIR}/.env
+    echo "COUCH_DB_URL=${COUCH_DB_URL}" >>${DATA_DIR}/.env
 fi
 
 # Read in the .env file and export the variables
@@ -79,31 +85,44 @@ ln -s ${DATA_DIR}/.env /worker/.env
 # make these directories in runner, incase of mount
 mkdir -p ${DATA_DIR}/minio
 mkdir -p ${DATA_DIR}/redis
+mkdir -p ${DATA_DIR}/couch
 chown -R couchdb:couchdb ${DATA_DIR}/couch
 
 REDIS_CONFIG="/etc/redis/redis.conf"
 sed -i "s#DATA_DIR#${DATA_DIR}#g" "${REDIS_CONFIG}"
 
 if [[ -n "${USE_DEFAULT_REDIS_CONFIG}" ]]; then
-  REDIS_CONFIG=""
+    REDIS_CONFIG=""
 fi
 
 if [[ -n "${REDIS_PASSWORD}" ]]; then
-  redis-server "${REDIS_CONFIG}" --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
+    redis-server "${REDIS_CONFIG}" --requirepass $REDIS_PASSWORD >/dev/stdout 2>&1 &
 else
-  redis-server "${REDIS_CONFIG}" > /dev/stdout 2>&1 &
+    redis-server "${REDIS_CONFIG}" >/dev/stdout 2>&1 &
 fi
-/bbcouch-runner.sh &
+
+echo "Starting callback CouchDB runner..."
+./bbcouch-runner.sh &
 
 # only start minio if use s3 isn't passed
 if [[ -z "${USE_S3}" ]]; then
-  /minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
+    if [[ ${TARGETBUILD} == aas ]]; then
+        echo "Starting MinIO in Azure Gateway mode"
+        if [[ -z "${AZURE_STORAGE_ACCOUNT}" || -z "${AZURE_STORAGE_KEY}" || -z "${MINIO_ACCESS_KEY}" || -z "${MINIO_SECRET_KEY}" ]]; then
+            echo "The following environment variables must be set: AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_KEY, MINIO_ACCESS_KEY, MINIO_SECRET_KEY"
+            exit 1
+        fi
+        /minio/minio gateway azure --console-address ":9001" >/dev/stdout 2>&1 &
+    else
+        echo "Starting MinIO in standalone mode"
+        /minio/minio server --console-address ":9001" ${DATA_DIR}/minio >/dev/stdout 2>&1 &
+    fi
 fi
 
 /etc/init.d/nginx restart
 if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then
     # Add monthly cron job to renew certbot certificate
-    echo -n "* * 2 * * root exec /app/letsencrypt/certificate-renew.sh ${CUSTOM_DOMAIN}" >> /etc/cron.d/certificate-renew
+    echo -n "* * 2 * * root exec /app/letsencrypt/certificate-renew.sh ${CUSTOM_DOMAIN}" >>/etc/cron.d/certificate-renew
     chmod +x /etc/cron.d/certificate-renew
     # Request the certbot certificate
     /app/letsencrypt/certificate-request.sh ${CUSTOM_DOMAIN}

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.4.12",
+  "version": "3.4.16",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/backend-core/src/context/mainContext.ts

@@ -123,7 +123,7 @@ export async function doInAutomationContext<T>(params: {
   task: () => T
 }): Promise<T> {
   await ensureSnippetContext()
-  return newContext(
+  return await newContext(
     {
       tenantId: getTenantIDFromAppID(params.appId),
       appId: params.appId,
@@ -266,9 +266,9 @@ export const getProdAppId = () => {
   return conversions.getProdAppID(appId)
 }
 
-export function doInEnvironmentContext(
+export function doInEnvironmentContext<T>(
   values: Record<string, string>,
-  task: any
+  task: () => T
 ) {
   if (!values) {
     throw new Error("Must supply environment variables.")

packages/backend-core/src/sql/utils.ts

@@ -5,10 +5,10 @@ import {
   SqlQuery,
   Table,
   TableSourceType,
+  SEPARATOR,
 } from "@budibase/types"
 import { DEFAULT_BB_DATASOURCE_ID } from "../constants"
 import { Knex } from "knex"
-import { SEPARATOR } from "../db"
 import environment from "../environment"
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`

packages/backend-core/src/utils/Duration.ts

@@ -15,23 +15,27 @@ const conversion: Record<DurationType, number> = {
 }
 
 export class Duration {
+  constructor(public ms: number) {}
+
+  to(type: DurationType) {
+    return this.ms / conversion[type]
+  }
+
+  toMs() {
+    return this.ms
+  }
+
+  toSeconds() {
+    return this.to(DurationType.SECONDS)
+  }
+
   static convert(from: DurationType, to: DurationType, duration: number) {
     const milliseconds = duration * conversion[from]
     return milliseconds / conversion[to]
   }
 
   static from(from: DurationType, duration: number) {
-    return {
+    return new Duration(duration * conversion[from])
-      to: (to: DurationType) => {
-        return Duration.convert(from, to, duration)
-      },
-      toMs: () => {
-        return Duration.convert(from, DurationType.MILLISECONDS, duration)
-      },
-      toSeconds: () => {
-        return Duration.convert(from, DurationType.SECONDS, duration)
-      },
-    }
   }
 
   static fromSeconds(duration: number) {

packages/backend-core/src/utils/index.ts

@@ -2,3 +2,4 @@ export * from "./hashing"
 export * from "./utils"
 export * from "./stringUtils"
 export * from "./Duration"
+export * from "./time"

packages/backend-core/src/utils/time.ts

@@ -0,0 +1,7 @@
+import { Duration } from "./Duration"
+
+export async function time<T>(f: () => Promise<T>): Promise<[T, Duration]> {
+  const start = performance.now()
+  const result = await f()
+  return [result, Duration.fromMilliseconds(performance.now() - start)]
+}

packages/builder/src/components/common/CodeEditor/CodeEditor.svelte

@@ -57,6 +57,7 @@
   import { tooltips } from "@codemirror/view"
   import type { BindingCompletion, CodeValidator } from "@/types"
   import { validateHbsTemplate } from "./validator/hbs"
+  import { validateJsTemplate } from "./validator/js"
 
   export let label: string | undefined = undefined
   export let completions: BindingCompletion[] = []
@@ -365,6 +366,9 @@
     if (mode === EditorModes.Handlebars) {
       const diagnostics = validateHbsTemplate(value, validations)
       editor.dispatch(setDiagnostics(editor.state, diagnostics))
+    } else if (mode === EditorModes.JS) {
+      const diagnostics = validateJsTemplate(value, validations)
+      editor.dispatch(setDiagnostics(editor.state, diagnostics))
     }
   }
 

packages/builder/src/components/common/CodeEditor/validator/js.ts

@@ -0,0 +1,101 @@
+import { Parser } from "acorn"
+import * as walk from "acorn-walk"
+
+import type { Diagnostic } from "@codemirror/lint"
+import { CodeValidator } from "@/types"
+
+export function validateJsTemplate(
+  code: string,
+  validations: CodeValidator
+): Diagnostic[] {
+  const diagnostics: Diagnostic[] = []
+
+  try {
+    const ast = Parser.parse(code, {
+      ecmaVersion: "latest",
+      locations: true,
+      allowReturnOutsideFunction: true,
+    })
+
+    const lineOffsets: number[] = []
+    let offset = 0
+    for (const line of code.split("\n")) {
+      lineOffsets.push(offset)
+      offset += line.length + 1 // +1 for newline character
+    }
+
+    let hasReturnStatement = false
+    walk.ancestor(ast, {
+      ReturnStatement(node, _state, ancestors) {
+        if (
+          // it returns a value
+          node.argument &&
+          // and it is top level
+          ancestors.length === 2 &&
+          ancestors[0].type === "Program" &&
+          ancestors[1].type === "ReturnStatement"
+        ) {
+          hasReturnStatement = true
+        }
+      },
+      CallExpression(node) {
+        const callee: any = node.callee
+        if (
+          node.type === "CallExpression" &&
+          callee.object?.name === "helpers" &&
+          node.loc
+        ) {
+          const functionName = callee.property.name
+          const from =
+            lineOffsets[node.loc.start.line - 1] + node.loc.start.column
+          const to = lineOffsets[node.loc.end.line - 1] + node.loc.end.column
+
+          if (!(functionName in validations)) {
+            diagnostics.push({
+              from,
+              to,
+              severity: "warning",
+              message: `"${functionName}" function does not exist.`,
+            })
+            return
+          }
+
+          const { arguments: expectedArguments } = validations[functionName]
+          if (
+            expectedArguments &&
+            node.arguments.length !== expectedArguments.length
+          ) {
+            diagnostics.push({
+              from,
+              to,
+              severity: "error",
+              message: `Function "${functionName}" expects ${
+                expectedArguments.length
+              } parameters (${expectedArguments.join(", ")}), but got ${
+                node.arguments.length
+              }.`,
+            })
+          }
+        }
+      },
+    })
+
+    if (!hasReturnStatement) {
+      diagnostics.push({
+        from: 0,
+        to: code.length,
+        severity: "error",
+        message: "Your code must return a value.",
+      })
+    }
+  } catch (e: any) {
+    diagnostics.push({
+      from: 0,
+      to: code.length,
+      severity: "error",
+      message: `Syntax error: ${e.message}`,
+    })
+  }
+
+  return diagnostics
+}

packages/builder/src/components/common/CodeEditor/validator/tests/js.spec.ts

@@ -0,0 +1,156 @@
+import { validateJsTemplate } from "../js"
+import { CodeValidator } from "@/types"
+
+describe("js validator", () => {
+  it("validates valid code", () => {
+    const text = "return 7"
+    const validators = {}
+
+    const result = validateJsTemplate(text, validators)
+    expect(result).toEqual([])
+  })
+
+  it("does not validate runtime errors", () => {
+    const text = "return a"
+    const validators = {}
+
+    const result = validateJsTemplate(text, validators)
+    expect(result).toEqual([])
+  })
+
+  it("validates multiline code", () => {
+    const text = "const foo='bar'\nreturn 123"
+    const validators = {}
+
+    const result = validateJsTemplate(text, validators)
+    expect(result).toEqual([])
+  })
+
+  it("allows return not being on the last line", () => {
+    const text = "const foo='bar'\nreturn 123\nconsole.log(foo)"
+    const validators = {}
+
+    const result = validateJsTemplate(text, validators)
+    expect(result).toEqual([])
+  })
+
+  it("throws on missing return", () => {
+    const text = "const foo='bar'\nbar='foo'"
+    const validators = {}
+
+    const result = validateJsTemplate(text, validators)
+    expect(result).toEqual([
+      {
+        from: 0,
+        message: "Your code must return a value.",
+        severity: "error",
+        to: 25,
+      },
+    ])
+  })
+
+  it("checks that returns are at top level", () => {
+    const text = `
+    function call(){
+      return 1
+    }`
+    const validators = {}
+
+    const result = validateJsTemplate(text, validators)
+    expect(result).toEqual([
+      {
+        from: 0,
+        message: "Your code must return a value.",
+        severity: "error",
+        to: text.length,
+      },
+    ])
+  })
+
+  describe("helpers", () => {
+    const validators: CodeValidator = {
+      helperFunction: {
+        arguments: ["a", "b", "c"],
+      },
+    }
+
+    it("validates helpers with valid params", () => {
+      const text = "return helpers.helperFunction(1, 99, 'a')"
+
+      const result = validateJsTemplate(text, validators)
+      expect(result).toEqual([])
+    })
+
+    it("throws on too few params", () => {
+      const text = "return helpers.helperFunction(100)"
+
+      const result = validateJsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 7,
+          message: `Function "helperFunction" expects 3 parameters (a, b, c), but got 1.`,
+          severity: "error",
+          to: 34,
+        },
+      ])
+    })
+
+    it("throws on too many params", () => {
+      const text = "return helpers.helperFunction( 1, 99, 'a', 100)"
+
+      const result = validateJsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 7,
+          message: `Function "helperFunction" expects 3 parameters (a, b, c), but got 4.`,
+          severity: "error",
+          to: 47,
+        },
+      ])
+    })
+
+    it("validates helpers on inner functions", () => {
+      const text = `function call(){
+                      return helpers.helperFunction(1, 99)
+                    }
+                    return call()`
+
+      const result = validateJsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 46,
+          message: `Function "helperFunction" expects 3 parameters (a, b, c), but got 2.`,
+          severity: "error",
+          to: 75,
+        },
+      ])
+    })
+
+    it("validates multiple helpers", () => {
+      const text =
+        "return helpers.helperFunction(1, 99, 'a') + helpers.helperFunction(1) + helpers.another(1) + helpers.another()"
+      const validators: CodeValidator = {
+        helperFunction: {
+          arguments: ["a", "b", "c"],
+        },
+        another: { arguments: [] },
+      }
+
+      const result = validateJsTemplate(text, validators)
+      expect(result).toEqual([
+        {
+          from: 44,
+          message: `Function "helperFunction" expects 3 parameters (a, b, c), but got 1.`,
+          severity: "error",
+          to: 69,
+        },
+        {
+          from: 72,
+          message: `Function "another" expects 0 parameters (), but got 1.`,
+          severity: "error",
+          to: 90,
+        },
+      ])
+    })
+  })
+})

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -377,6 +377,7 @@
               value={jsValue ? decodeJSBinding(jsValue) : ""}
               on:change={onChangeJSValue}
               {completions}
+              {validations}
               mode={EditorModes.JS}
               bind:getCaretPosition
               bind:insertAtPos

packages/builder/src/stores/builder/automations.ts

@@ -15,7 +15,6 @@ import {
 import {
   AutomationTriggerStepId,
   AutomationEventType,
-  AutomationStepType,
   AutomationActionStepId,
   Automation,
   AutomationStep,
@@ -26,20 +25,25 @@ import {
   UILogicalOperator,
   EmptyFilterOption,
   AutomationIOType,
-  AutomationStepSchema,
-  AutomationTriggerSchema,
   BranchPath,
   BlockDefinitions,
+  isBranchStep,
+  isTrigger,
+  isRowUpdateTrigger,
+  isRowSaveTrigger,
+  isAppTrigger,
+  BranchStep,
   GetAutomationTriggerDefinitionsResponse,
   GetAutomationActionDefinitionsResponse,
   AppSelfResponse,
   TestAutomationResponse,
   isAutomationResults,
-  AutomationTestTrigger,
-  TriggerTestOutputs,
   RowActionTriggerOutputs,
   WebhookTriggerOutputs,
   AutomationCustomIOType,
+  AutomationTriggerResultOutputs,
+  AutomationTriggerResult,
+  AutomationStepType,
 } from "@budibase/types"
 import { ActionStepID, TriggerStepID } from "@/constants/backend/automations"
 import { FIELDS as COLUMNS } from "@/constants/backend"
@@ -111,7 +115,7 @@ const automationActions = (store: AutomationStore) => ({
         const results: TestAutomationResponse | undefined =
           $selectedAutomation?.testResults
 
-        const testData: TriggerTestOutputs | undefined =
+        const testData: AutomationTriggerResultOutputs | undefined =
           $selectedAutomation.data?.testData
         const triggerDef = $selectedAutomation.data?.definition?.trigger
 
@@ -122,13 +126,13 @@ const automationActions = (store: AutomationStore) => ({
           ? $tables.list.find(table => table._id === rowActionTableId)
           : undefined
 
-        let triggerData: TriggerTestOutputs | undefined
+        let triggerData: AutomationTriggerResultOutputs | undefined
 
         if (results && isAutomationResults(results)) {
-          const automationTrigger: AutomationTestTrigger | undefined =
+          const automationTrigger: AutomationTriggerResult | undefined =
             results?.trigger
 
-          const outputs: TriggerTestOutputs | undefined =
+          const outputs: AutomationTriggerResultOutputs | undefined =
             automationTrigger?.outputs
           triggerData = outputs ? outputs : undefined
 
@@ -417,16 +421,16 @@ const automationActions = (store: AutomationStore) => ({
     let result: (AutomationStep | AutomationTrigger)[] = []
     pathWay.forEach(path => {
       const { stepIdx, branchIdx } = path
-      let last = result.length ? result[result.length - 1] : []
       if (!result.length) {
         // Preceeding steps.
         result = steps.slice(0, stepIdx + 1)
         return
       }
-      if (last && "inputs" in last) {
+      let last = result[result.length - 1]
+      if (isBranchStep(last)) {
         if (Number.isInteger(branchIdx)) {
           const branchId = last.inputs.branches[branchIdx].id
-          const children = last.inputs.children[branchId]
+          const children = last.inputs.children?.[branchId] || []
           const stepChildren = children.slice(0, stepIdx + 1)
           // Preceeding steps.
           result = result.concat(stepChildren)
@@ -599,23 +603,28 @@ const automationActions = (store: AutomationStore) => ({
           id: block.id,
         },
       ]
-      const branches: Branch[] = block.inputs?.branches || []
 
-      branches.forEach((branch, bIdx) => {
+      if (isBranchStep(block)) {
-        block.inputs?.children[branch.id].forEach(
+        const branches = block.inputs?.branches || []
-          (bBlock: AutomationStep, sIdx: number, array: AutomationStep[]) => {
+        const children = block.inputs?.children || {}
-            const ended =
+
-              array.length - 1 === sIdx && !bBlock.inputs?.branches?.length
+        branches.forEach((branch, bIdx) => {
-            treeTraverse(bBlock, pathToCurrentNode, sIdx, bIdx, ended)
+          children[branch.id].forEach(
-          }
+            (bBlock: AutomationStep, sIdx: number, array: AutomationStep[]) => {
-        )
+              const ended = array.length - 1 === sIdx && !branches.length
-      })
+              treeTraverse(bBlock, pathToCurrentNode, sIdx, bIdx, ended)
+            }
+          )
+        })
+
+        terminating = terminating && !branches.length
+      }
 
       store.actions.registerBlock(
         blockRefs,
         block,
         pathToCurrentNode,
-        terminating && !branches.length
+        terminating
       )
     }
 
@@ -732,7 +741,6 @@ const automationActions = (store: AutomationStore) => ({
           pathBlock.stepId === ActionStepID.LOOP &&
           pathBlock.blockToLoop in blocks
       }
-      const isTrigger = pathBlock.type === AutomationStepType.TRIGGER
 
       if (isLoopBlock && loopBlockCount == 0) {
         schema = {
@@ -743,17 +751,14 @@ const automationActions = (store: AutomationStore) => ({
         }
       }
 
-      const icon = isTrigger
+      const icon = isTrigger(pathBlock)
         ? pathBlock.icon
         : isLoopBlock
         ? "Reuse"
         : pathBlock.icon
 
-      if (blockIdx === 0 && isTrigger) {
+      if (blockIdx === 0 && isTrigger(pathBlock)) {
-        if (
+        if (isRowUpdateTrigger(pathBlock) || isRowSaveTrigger(pathBlock)) {
-          pathBlock.event === AutomationEventType.ROW_UPDATE ||
-          pathBlock.event === AutomationEventType.ROW_SAVE
-        ) {
           let table: any = get(tables).list.find(
             (table: Table) => table._id === pathBlock.inputs.tableId
           )
@@ -765,7 +770,7 @@ const automationActions = (store: AutomationStore) => ({
             }
           }
           delete schema.row
-        } else if (pathBlock.event === AutomationEventType.APP_TRIGGER) {
+        } else if (isAppTrigger(pathBlock)) {
           schema = Object.fromEntries(
             Object.keys(pathBlock.inputs.fields || []).map(key => [
               key,
@@ -1081,8 +1086,10 @@ const automationActions = (store: AutomationStore) => ({
     ]
 
     let cache:
-      | AutomationStepSchema<AutomationActionStepId>
+      | AutomationStep
-      | AutomationTriggerSchema<AutomationTriggerStepId>
+      | AutomationTrigger
+      | AutomationStep[]
+      | undefined = undefined
 
     pathWay.forEach((path, pathIdx, array) => {
       const { stepIdx, branchIdx } = path
@@ -1104,9 +1111,13 @@ const automationActions = (store: AutomationStore) => ({
         }
         return
       }
-      if (Number.isInteger(branchIdx)) {
+      if (
+        Number.isInteger(branchIdx) &&
+        !Array.isArray(cache) &&
+        isBranchStep(cache)
+      ) {
         const branchId = cache.inputs.branches[branchIdx].id
-        const children = cache.inputs.children[branchId]
+        const children = cache.inputs.children?.[branchId] || []
 
         if (final) {
           insertBlock(children, stepIdx)
@@ -1256,7 +1267,7 @@ const automationActions = (store: AutomationStore) => ({
   branchLeft: async (
     pathTo: Array<any>,
     automation: Automation,
-    block: AutomationStep
+    block: BranchStep
   ) => {
     const update = store.actions.shiftBranch(pathTo, block)
     if (update) {
@@ -1279,7 +1290,7 @@ const automationActions = (store: AutomationStore) => ({
   branchRight: async (
     pathTo: Array<BranchPath>,
     automation: Automation,
-    block: AutomationStep
+    block: BranchStep
   ) => {
     const update = store.actions.shiftBranch(pathTo, block, 1)
     if (update) {
@@ -1299,7 +1310,7 @@ const automationActions = (store: AutomationStore) => ({
    * @param {Number} direction - the direction of the swap. Defaults to -1 for left, add 1 for right
    * @returns
    */
-  shiftBranch: (pathTo: Array<any>, block: AutomationStep, direction = -1) => {
+  shiftBranch: (pathTo: Array<any>, block: BranchStep, direction = -1) => {
     let newBlock = cloneDeep(block)
     const branchPath = pathTo.at(-1)
     const targetIdx = branchPath.branchIdx
@@ -1595,7 +1606,7 @@ const automationActions = (store: AutomationStore) => ({
 
 export interface AutomationContext {
   user: AppSelfResponse | null
-  trigger?: TriggerTestOutputs
+  trigger?: AutomationTriggerResultOutputs
   steps: Record<string, AutomationStep>
   env: Record<string, any>
   settings: Record<string, any>

packages/builder/src/stores/builder/screenComponent.ts

@@ -8,6 +8,7 @@ import {
   UIComponentError,
   ComponentDefinition,
   DependsOnComponentSetting,
+  Screen,
 } from "@budibase/types"
 import { queries } from "./queries"
 import { views } from "./views"
@@ -66,6 +67,7 @@ export const screenComponentErrorList = derived(
     if (!$selectedScreen) {
       return []
     }
+    const screen = $selectedScreen
 
     const datasources = {
       ...reduceBy("_id", $tables.list),
@@ -79,7 +81,9 @@ export const screenComponentErrorList = derived(
     const errors: UIComponentError[] = []
 
     function checkComponentErrors(component: Component, ancestors: string[]) {
-      errors.push(...getInvalidDatasources(component, datasources, definitions))
+      errors.push(
+        ...getInvalidDatasources(screen, component, datasources, definitions)
+      )
       errors.push(...getMissingRequiredSettings(component, definitions))
       errors.push(...getMissingAncestors(component, definitions, ancestors))
 
@@ -95,6 +99,7 @@ export const screenComponentErrorList = derived(
 )
 
 function getInvalidDatasources(
+  screen: Screen,
   component: Component,
   datasources: Record<string, any>,
   definitions: Record<string, ComponentDefinition>

packages/server/src/api/routes/tests/automation.spec.ts

@@ -14,18 +14,16 @@ import {
   AutomationResults,
   ConfigType,
   FieldType,
+  FilterCondition,
   isDidNotTriggerResponse,
   SettingsConfig,
   Table,
 } from "@budibase/types"
 import { mocks } from "@budibase/backend-core/tests"
 import { createAutomationBuilder } from "../../../automations/tests/utilities/AutomationTestBuilder"
-import { automations } from "@budibase/shared-core"
 import { basicTable } from "../../../tests/utilities/structures"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 
-const FilterConditions = automations.steps.filter.FilterConditions
-
 const MAX_RETRIES = 4
 const {
   basicAutomation,
@@ -483,15 +481,40 @@ describe("/automations", () => {
       expect(events.automation.created).not.toHaveBeenCalled()
       expect(events.automation.triggerUpdated).not.toHaveBeenCalled()
     })
+
+    it("can update an input field", async () => {
+      const { automation } = await createAutomationBuilder(config)
+        .onRowDeleted({ tableId: "tableId" })
+        .serverLog({ text: "test" })
+        .save()
+
+      automation.definition.trigger.inputs.tableId = "newTableId"
+      const { automation: updatedAutomation } =
+        await config.api.automation.update(automation)
+
+      expect(updatedAutomation.definition.trigger.inputs.tableId).toEqual(
+        "newTableId"
+      )
+    })
+
+    it("cannot update a readonly field", async () => {
+      const { automation } = await createAutomationBuilder(config)
+        .onRowAction({ tableId: "tableId" })
+        .serverLog({ text: "test" })
+        .save()
+
+      automation.definition.trigger.inputs.tableId = "newTableId"
+      await config.api.automation.update(automation, {
+        status: 400,
+        body: {
+          message: "Field tableId is readonly and it cannot be modified",
+        },
+      })
+    })
   })
 
   describe("fetch", () => {
     it("return all the automations for an instance", async () => {
-      const fetchResponse = await config.api.automation.fetch()
-      for (const auto of fetchResponse.automations) {
-        await config.api.automation.delete(auto)
-      }
-
       const { automation: automation1 } = await config.api.automation.post(
         newAutomation()
       )
@@ -590,7 +613,7 @@ describe("/automations", () => {
               steps: [
                 {
                   inputs: {
-                    condition: FilterConditions.EQUAL,
+                    condition: FilterCondition.EQUAL,
                     field: "{{ trigger.row.City }}",
                     value: "{{ trigger.oldRow.City }}",
                   },

packages/server/src/automations/actions.ts

@@ -28,6 +28,8 @@ import {
   Hosting,
   ActionImplementation,
   AutomationStepDefinition,
+  AutomationStepInputs,
+  AutomationStepOutputs,
 } from "@budibase/types"
 import sdk from "../sdk"
 import { getAutomationPlugin } from "../utilities/fileSystem"
@@ -123,11 +125,15 @@ export async function getActionDefinitions(): Promise<
 }
 
 /* istanbul ignore next */
-export async function getAction(
+export async function getAction<
-  stepId: AutomationActionStepId
+  TStep extends AutomationActionStepId,
-): Promise<ActionImplementation<any, any> | undefined> {
+  TInputs = AutomationStepInputs<TStep>,
+  TOutputs = AutomationStepOutputs<TStep>
+>(stepId: TStep): Promise<ActionImplementation<TInputs, TOutputs> | undefined> {
   if (ACTION_IMPLS[stepId as keyof ActionImplType] != null) {
-    return ACTION_IMPLS[stepId as keyof ActionImplType]
+    return ACTION_IMPLS[
+      stepId as keyof ActionImplType
+    ] as unknown as ActionImplementation<TInputs, TOutputs>
   }
 
   // must be a plugin

packages/server/src/automations/automationUtils.ts

@@ -6,10 +6,10 @@ import {
 import sdk from "../sdk"
 import {
   AutomationAttachment,
+  BaseIOStructure,
+  FieldSchema,
   FieldType,
   Row,
-  LoopStepType,
-  LoopStepInputs,
 } from "@budibase/types"
 import { objectStore, context } from "@budibase/backend-core"
 import * as uuid from "uuid"
@@ -32,33 +32,34 @@ import path from "path"
  * primitive types.
  */
 export function cleanInputValues<T extends Record<string, any>>(
-  inputs: any,
+  inputs: T,
-  schema?: any
+  schema?: Partial<Record<keyof T, FieldSchema | BaseIOStructure>>
 ): T {
-  if (schema == null) {
+  const keys = Object.keys(inputs) as (keyof T)[]
-    return inputs
+  for (let inputKey of keys) {
-  }
-  for (let inputKey of Object.keys(inputs)) {
     let input = inputs[inputKey]
     if (typeof input !== "string") {
       continue
     }
-    let propSchema = schema.properties[inputKey]
+    let propSchema = schema?.[inputKey]
     if (!propSchema) {
       continue
     }
     if (propSchema.type === "boolean") {
       let lcInput = input.toLowerCase()
       if (lcInput === "true") {
+        // @ts-expect-error - indexing a generic on purpose
         inputs[inputKey] = true
       }
       if (lcInput === "false") {
+        // @ts-expect-error - indexing a generic on purpose
         inputs[inputKey] = false
       }
     }
     if (propSchema.type === "number") {
       let floatInput = parseFloat(input)
       if (!isNaN(floatInput)) {
+        // @ts-expect-error - indexing a generic on purpose
         inputs[inputKey] = floatInput
       }
     }
@@ -93,7 +94,7 @@ export function cleanInputValues<T extends Record<string, any>>(
  */
 export async function cleanUpRow(tableId: string, row: Row) {
   let table = await sdk.tables.getTable(tableId)
-  return cleanInputValues(row, { properties: table.schema })
+  return cleanInputValues(row, table.schema)
 }
 
 export function getError(err: any) {
@@ -271,36 +272,3 @@ export function stringSplit(value: string | string[]) {
   }
   return value.split(",")
 }
-
-export function typecastForLooping(input: LoopStepInputs) {
-  if (!input || !input.binding) {
-    return null
-  }
-  try {
-    switch (input.option) {
-      case LoopStepType.ARRAY:
-        if (typeof input.binding === "string") {
-          return JSON.parse(input.binding)
-        }
-        break
-      case LoopStepType.STRING:
-        if (Array.isArray(input.binding)) {
-          return input.binding.join(",")
-        }
-        break
-    }
-  } catch (err) {
-    throw new Error("Unable to cast to correct type")
-  }
-  return input.binding
-}
-
-export function ensureMaxIterationsAsNumber(
-  value: number | string | undefined
-): number | undefined {
-  if (typeof value === "number") return value
-  if (typeof value === "string") {
-    return parseInt(value)
-  }
-  return undefined
-}

packages/server/src/automations/loopUtils.ts

@@ -1,46 +0,0 @@
-import * as automationUtils from "./automationUtils"
-import { isPlainObject } from "lodash"
-
-type ObjValue = {
-  [key: string]: string | ObjValue
-}
-
-export function replaceFakeBindings<T extends Record<string, any>>(
-  originalStepInput: T,
-  loopStepNumber: number
-): T {
-  const result: Record<string, any> = {}
-  for (const [key, value] of Object.entries(originalStepInput)) {
-    result[key] = replaceBindingsRecursive(value, loopStepNumber)
-  }
-  return result as T
-}
-
-function replaceBindingsRecursive(
-  value: string | ObjValue,
-  loopStepNumber: number
-) {
-  if (value === null || value === undefined) {
-    return value
-  }
-
-  if (typeof value === "object") {
-    for (const [innerKey, innerValue] of Object.entries(value)) {
-      if (typeof innerValue === "string") {
-        value[innerKey] = automationUtils.substituteLoopStep(
-          innerValue,
-          `steps.${loopStepNumber}`
-        )
-      } else if (
-        innerValue &&
-        isPlainObject(innerValue) &&
-        Object.keys(innerValue).length > 0
-      ) {
-        value[innerKey] = replaceBindingsRecursive(innerValue, loopStepNumber)
-      }
-    }
-  } else if (typeof value === "string") {
-    value = automationUtils.substituteLoopStep(value, `steps.${loopStepNumber}`)
-  }
-  return value
-}

packages/server/src/automations/steps/filter.ts

@@ -1,7 +1,8 @@
-import { FilterStepInputs, FilterStepOutputs } from "@budibase/types"
+import {
-import { automations } from "@budibase/shared-core"
+  FilterCondition,
-
+  FilterStepInputs,
-const FilterConditions = automations.steps.filter.FilterConditions
+  FilterStepOutputs,
+} from "@budibase/types"
 
 export async function run({
   inputs,
@@ -26,16 +27,16 @@ export async function run({
     let result = false
     if (typeof field !== "object" && typeof value !== "object") {
       switch (condition) {
-        case FilterConditions.EQUAL:
+        case FilterCondition.EQUAL:
           result = field === value
           break
-        case FilterConditions.NOT_EQUAL:
+        case FilterCondition.NOT_EQUAL:
           result = field !== value
           break
-        case FilterConditions.GREATER_THAN:
+        case FilterCondition.GREATER_THAN:
           result = field > value
           break
-        case FilterConditions.LESS_THAN:
+        case FilterCondition.LESS_THAN:
           result = field < value
           break
       }

packages/server/src/automations/tests/automationUtils.spec.ts

@@ -1,9 +1,5 @@
-import {
+import { AutomationIOType } from "@budibase/types"
-  typecastForLooping,
+import { cleanInputValues, substituteLoopStep } from "../automationUtils"
-  cleanInputValues,
-  substituteLoopStep,
-} from "../automationUtils"
-import { LoopStepType } from "@budibase/types"
 
 describe("automationUtils", () => {
   describe("substituteLoopStep", () => {
@@ -30,29 +26,6 @@ describe("automationUtils", () => {
     })
   })
 
-  describe("typeCastForLooping", () => {
-    it("should parse to correct type", () => {
-      expect(
-        typecastForLooping({ option: LoopStepType.ARRAY, binding: [1, 2, 3] })
-      ).toEqual([1, 2, 3])
-      expect(
-        typecastForLooping({ option: LoopStepType.ARRAY, binding: "[1,2,3]" })
-      ).toEqual([1, 2, 3])
-      expect(
-        typecastForLooping({ option: LoopStepType.STRING, binding: [1, 2, 3] })
-      ).toEqual("1,2,3")
-    })
-    it("should handle null values", () => {
-      // expect it to handle where the binding is null
-      expect(
-        typecastForLooping({ option: LoopStepType.ARRAY, binding: null })
-      ).toEqual(null)
-      expect(() =>
-        typecastForLooping({ option: LoopStepType.ARRAY, binding: "test" })
-      ).toThrow()
-    })
-  })
-
   describe("cleanInputValues", () => {
     it("should handle array relationship fields from read binding", () => {
       const schema = {
@@ -70,15 +43,12 @@ describe("automationUtils", () => {
         },
       }
       expect(
-        cleanInputValues(
+        cleanInputValues({
-          {
+          row: {
-            row: {
+            relationship: `[{"_id": "ro_ta_users_us_3"}]`,
-              relationship: `[{"_id": "ro_ta_users_us_3"}]`,
-            },
-            schema,
           },
-          schema
+          schema,
-        )
+        })
       ).toEqual({
         row: {
           relationship: [{ _id: "ro_ta_users_us_3" }],
@@ -103,15 +73,12 @@ describe("automationUtils", () => {
         },
       }
       expect(
-        cleanInputValues(
+        cleanInputValues({
-          {
+          row: {
-            row: {
+            relationship: `ro_ta_users_us_3`,
-              relationship: `ro_ta_users_us_3`,
-            },
-            schema,
           },
-          schema
+          schema,
-        )
+        })
       ).toEqual({
         row: {
           relationship: "ro_ta_users_us_3",
@@ -122,28 +89,27 @@ describe("automationUtils", () => {
 
     it("should be able to clean inputs with the utilities", () => {
       // can't clean without a schema
-      let output = cleanInputValues({ a: "1" })
+      const one = cleanInputValues({ a: "1" })
-      expect(output.a).toBe("1")
+      expect(one.a).toBe("1")
-      output = cleanInputValues(
+
+      const two = cleanInputValues(
         { a: "1", b: "true", c: "false", d: 1, e: "help" },
         {
-          properties: {
+          a: {
-            a: {
+            type: AutomationIOType.NUMBER,
-              type: "number",
+          },
-            },
+          b: {
-            b: {
+            type: AutomationIOType.BOOLEAN,
-              type: "boolean",
+          },
-            },
+          c: {
-            c: {
+            type: AutomationIOType.BOOLEAN,
-              type: "boolean",
-            },
           },
         }
       )
-      expect(output.a).toBe(1)
+      expect(two.a).toBe(1)
-      expect(output.b).toBe(true)
+      expect(two.b).toBe(true)
-      expect(output.c).toBe(false)
+      expect(two.c).toBe(false)
-      expect(output.d).toBe(1)
+      expect(two.d).toBe(1)
     })
   })
 })

packages/server/src/automations/tests/scenarios.spec.ts

@@ -1,5 +1,11 @@
 import * as automation from "../index"
-import { LoopStepType, FieldType, Table, Datasource } from "@budibase/types"
+import {
+  LoopStepType,
+  FieldType,
+  Table,
+  Datasource,
+  FilterCondition,
+} from "@budibase/types"
 import { createAutomationBuilder } from "./utilities/AutomationTestBuilder"
 import {
   DatabaseName,
@@ -7,12 +13,9 @@ import {
 } from "../../integrations/tests/utils"
 import { Knex } from "knex"
 import { generator } from "@budibase/backend-core/tests"
-import { automations } from "@budibase/shared-core"
 import TestConfiguration from "../../tests/utilities/TestConfiguration"
 import { basicTable } from "../../tests/utilities/structures"
 
-const FilterConditions = automations.steps.filter.FilterConditions
-
 describe("Automation Scenarios", () => {
   const config = new TestConfiguration()
 
@@ -256,7 +259,7 @@ describe("Automation Scenarios", () => {
         })
         .filter({
           field: "{{ steps.2.rows.0.value }}",
-          condition: FilterConditions.EQUAL,
+          condition: FilterCondition.EQUAL,
           value: 20,
         })
         .serverLog({ text: "Equal condition met" })
@@ -282,7 +285,7 @@ describe("Automation Scenarios", () => {
         })
         .filter({
           field: "{{ steps.2.rows.0.value }}",
-          condition: FilterConditions.NOT_EQUAL,
+          condition: FilterCondition.NOT_EQUAL,
           value: 20,
         })
         .serverLog({ text: "Not Equal condition met" })
@@ -295,37 +298,37 @@ describe("Automation Scenarios", () => {
 
     const testCases = [
       {
-        condition: FilterConditions.EQUAL,
+        condition: FilterCondition.EQUAL,
         value: 10,
         rowValue: 10,
         expectPass: true,
       },
       {
-        condition: FilterConditions.NOT_EQUAL,
+        condition: FilterCondition.NOT_EQUAL,
         value: 10,
         rowValue: 20,
         expectPass: true,
       },
       {
-        condition: FilterConditions.GREATER_THAN,
+        condition: FilterCondition.GREATER_THAN,
         value: 10,
         rowValue: 15,
         expectPass: true,
       },
       {
-        condition: FilterConditions.LESS_THAN,
+        condition: FilterCondition.LESS_THAN,
         value: 10,
         rowValue: 5,
         expectPass: true,
       },
       {
-        condition: FilterConditions.GREATER_THAN,
+        condition: FilterCondition.GREATER_THAN,
         value: 10,
         rowValue: 5,
         expectPass: false,
       },
       {
-        condition: FilterConditions.LESS_THAN,
+        condition: FilterCondition.LESS_THAN,
         value: 10,
         rowValue: 15,
         expectPass: false,

packages/server/src/automations/tests/steps/createRow.spec.ts

@@ -4,7 +4,7 @@ import {
 } from "../../../tests/utilities/structures"
 import { objectStore } from "@budibase/backend-core"
 import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
-import { Row, Table } from "@budibase/types"
+import { FilterCondition, Row, Table } from "@budibase/types"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 
 async function uploadTestFile(filename: string) {
@@ -90,7 +90,7 @@ describe("test the create row action", () => {
       .createRow({ row: {} }, { stepName: "CreateRow" })
       .filter({
         field: "{{ stepsByName.CreateRow.success }}",
-        condition: "equal",
+        condition: FilterCondition.EQUAL,
         value: true,
       })
       .serverLog(
@@ -131,7 +131,7 @@ describe("test the create row action", () => {
       .createRow({ row: attachmentRow }, { stepName: "CreateRow" })
       .filter({
         field: "{{ stepsByName.CreateRow.success }}",
-        condition: "equal",
+        condition: FilterCondition.EQUAL,
         value: true,
       })
       .serverLog(

packages/server/src/automations/tests/steps/filter.spec.ts

@@ -1,19 +1,21 @@
-import { automations } from "@budibase/shared-core"
 import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
+import { FilterCondition } from "@budibase/types"
 
-const FilterConditions = automations.steps.filter.FilterConditions
+function stringToFilterCondition(
-
+  condition: "==" | "!=" | ">" | "<"
-function stringToFilterCondition(condition: "==" | "!=" | ">" | "<"): string {
+): FilterCondition {
   switch (condition) {
     case "==":
-      return FilterConditions.EQUAL
+      return FilterCondition.EQUAL
     case "!=":
-      return FilterConditions.NOT_EQUAL
+      return FilterCondition.NOT_EQUAL
     case ">":
-      return FilterConditions.GREATER_THAN
+      return FilterCondition.GREATER_THAN
     case "<":
-      return FilterConditions.LESS_THAN
+      return FilterCondition.LESS_THAN
+    default:
+      throw new Error(`Unsupported condition: ${condition}`)
   }
 }
 

packages/server/src/automations/tests/steps/loop.spec.ts

@@ -6,8 +6,9 @@ import {
   ServerLogStepOutputs,
   CreateRowStepOutputs,
   FieldType,
+  FilterCondition,
+  AutomationStepStatus,
 } from "@budibase/types"
-import * as loopUtils from "../../loopUtils"
 import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 
@@ -30,8 +31,8 @@ describe("Attempt to run a basic loop automation", () => {
     await config.api.row.save(table._id!, {})
   })
 
-  afterAll(() => {
+  afterAll(async () => {
-    automation.shutdown()
+    await automation.shutdown()
     config.end()
   })
 
@@ -535,97 +536,50 @@ describe("Attempt to run a basic loop automation", () => {
     expect(results.steps[2].outputs.rows).toHaveLength(0)
   })
 
-  describe("replaceFakeBindings", () => {
+  describe("loop output", () => {
-    it("should replace loop bindings in nested objects", () => {
+    it("should not output anything if a filter stops the automation", async () => {
-      const originalStepInput = {
+      const results = await createAutomationBuilder(config)
-        schema: {
+        .onAppAction()
-          name: {
+        .filter({
-            type: "string",
+          condition: FilterCondition.EQUAL,
-            constraints: {
+          field: "1",
-              type: "string",
+          value: "2",
-              length: { maximum: null },
+        })
-              presence: false,
+        .loop({
-            },
+          option: LoopStepType.ARRAY,
-            name: "name",
+          binding: [1, 2, 3],
-            display: { type: "Text" },
+        })
-          },
+        .serverLog({ text: "Message {{loop.currentItem}}" })
-        },
+        .test({ fields: {} })
-        row: {
-          tableId: "ta_aaad4296e9f74b12b1b90ef7a84afcad",
-          name: "{{ loop.currentItem.pokemon }}",
-        },
-      }
 
-      const loopStepNumber = 3
+      expect(results.steps.length).toBe(1)
-
+      expect(results.steps[0].outputs).toEqual({
-      const result = loopUtils.replaceFakeBindings(
+        comparisonValue: 2,
-        originalStepInput,
+        refValue: 1,
-        loopStepNumber
+        result: false,
-      )
+        success: true,
-
+        status: "stopped",
-      expect(result).toEqual({
-        schema: {
-          name: {
-            type: "string",
-            constraints: {
-              type: "string",
-              length: { maximum: null },
-              presence: false,
-            },
-            name: "name",
-            display: { type: "Text" },
-          },
-        },
-        row: {
-          tableId: "ta_aaad4296e9f74b12b1b90ef7a84afcad",
-          name: "{{ steps.3.currentItem.pokemon }}",
-        },
       })
     })
 
-    it("should handle null values in nested objects", () => {
+    it("should not fail if queryRows returns nothing", async () => {
-      const originalStepInput = {
+      const table = await config.api.table.save(basicTable())
-        nullValue: null,
+      const results = await createAutomationBuilder(config)
-        nestedNull: {
+        .onAppAction()
-          someKey: null,
+        .queryRows({
-        },
+          tableId: table._id!,
-        validValue: "{{ loop.someValue }}",
+        })
-      }
+        .loop({
+          option: LoopStepType.ARRAY,
+          binding: "{{ steps.1.rows }}",
+        })
+        .serverLog({ text: "Message {{loop.currentItem}}" })
+        .test({ fields: {} })
 
-      const loopStepNumber = 2
+      expect(results.steps[1].outputs.success).toBe(true)
-
+      expect(results.steps[1].outputs.status).toBe(
-      const result = loopUtils.replaceFakeBindings(
+        AutomationStepStatus.NO_ITERATIONS
-        originalStepInput,
-        loopStepNumber
       )
-
-      expect(result).toEqual({
-        nullValue: null,
-        nestedNull: {
-          someKey: null,
-        },
-        validValue: "{{ steps.2.someValue }}",
-      })
-    })
-
-    it("should handle empty objects and arrays", () => {
-      const originalStepInput = {
-        emptyObject: {},
-        emptyArray: [],
-        nestedEmpty: {
-          emptyObj: {},
-          emptyArr: [],
-        },
-      }
-
-      const loopStepNumber = 1
-
-      const result = loopUtils.replaceFakeBindings(
-        originalStepInput,
-        loopStepNumber
-      )
-
-      expect(result).toEqual(originalStepInput)
     })
   })
 })

packages/server/src/automations/tests/triggers/cron.spec.ts

@@ -66,7 +66,7 @@ describe("cron trigger", () => {
     })
   })
 
-  it("should stop if the job fails more than 3 times", async () => {
+  it("should stop if the job fails more than N times", async () => {
     const { automation } = await createAutomationBuilder(config)
       .onCron({ cron: "* * * * *" })
       .queryRows({

packages/server/src/automations/tests/triggers/webhook.spec.ts

@@ -5,7 +5,7 @@ import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 
 mocks.licenses.useSyncAutomations()
 
-describe("Branching automations", () => {
+describe("Webhook trigger test", () => {
   const config = new TestConfiguration()
   let table: Table
   let webhook: Webhook

packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts

@@ -64,6 +64,7 @@ class TriggerBuilder {
   onRowDeleted = this.trigger(AutomationTriggerStepId.ROW_DELETED)
   onWebhook = this.trigger(AutomationTriggerStepId.WEBHOOK)
   onCron = this.trigger(AutomationTriggerStepId.CRON)
+  onRowAction = this.trigger(AutomationTriggerStepId.ROW_ACTION)
 }
 
 class BranchStepBuilder<TStep extends AutomationTriggerStepId> {

packages/server/src/automations/triggers.ts

@@ -184,11 +184,12 @@ export async function externalTrigger(
     // values are likely to be submitted as strings, so we shall convert to correct type
     const coercedFields: any = {}
     const fields = automation.definition.trigger.inputs.fields
-    for (let key of Object.keys(fields || {})) {
+    for (const key of Object.keys(fields || {})) {
       coercedFields[key] = coerce(params.fields[key], fields[key])
     }
     params.fields = coercedFields
   }
+
   // row actions and webhooks flatten the fields down
   else if (
     sdk.automations.isRowAction(automation) ||
@@ -200,6 +201,7 @@ export async function externalTrigger(
       fields: {},
     }
   }
+
   const data: AutomationData = { automation, event: params }
 
   const shouldTrigger = await checkTriggerFilters(automation, {

packages/server/src/automations/utils.ts

@@ -9,6 +9,7 @@ import { Automation, AutomationJob, MetadataType } from "@budibase/types"
 import { automationsEnabled } from "../features"
 import { helpers, REBOOT_CRON } from "@budibase/shared-core"
 import tracer from "dd-trace"
+import { JobId } from "bull"
 
 const CRON_STEP_ID = automations.triggers.definitions.CRON.stepId
 let Runner: Thread
@@ -30,39 +31,35 @@ function loggingArgs(job: AutomationJob) {
 }
 
 export async function processEvent(job: AutomationJob) {
-  return tracer.trace(
+  return tracer.trace("processEvent", async span => {
-    "processEvent",
+    const appId = job.data.event.appId!
-    { resource: "automation" },
+    const automationId = job.data.automation._id!
-    async span => {
-      const appId = job.data.event.appId!
-      const automationId = job.data.automation._id!
 
-      span?.addTags({
+    span.addTags({
-        appId,
+      appId,
-        automationId,
+      automationId,
-        job: {
+      job: {
-          id: job.id,
+        id: job.id,
-          name: job.name,
+        name: job.name,
-          attemptsMade: job.attemptsMade,
+        attemptsMade: job.attemptsMade,
-          opts: {
+        attempts: job.opts.attempts,
-            attempts: job.opts.attempts,
+        priority: job.opts.priority,
-            priority: job.opts.priority,
+        delay: job.opts.delay,
-            delay: job.opts.delay,
+        repeat: job.opts.repeat,
-            repeat: job.opts.repeat,
+        backoff: job.opts.backoff,
-            backoff: job.opts.backoff,
+        lifo: job.opts.lifo,
-            lifo: job.opts.lifo,
+        timeout: job.opts.timeout,
-            timeout: job.opts.timeout,
+        jobId: job.opts.jobId,
-            jobId: job.opts.jobId,
+        removeOnComplete: job.opts.removeOnComplete,
-            removeOnComplete: job.opts.removeOnComplete,
+        removeOnFail: job.opts.removeOnFail,
-            removeOnFail: job.opts.removeOnFail,
+        stackTraceLimit: job.opts.stackTraceLimit,
-            stackTraceLimit: job.opts.stackTraceLimit,
+        preventParsingData: job.opts.preventParsingData,
-            preventParsingData: job.opts.preventParsingData,
+      },
-          },
+    })
-        },
-      })
 
-      const task = async () => {
+    const task = async () => {
-        try {
+      try {
+        return await tracer.trace("task", async () => {
           if (isCronTrigger(job.data.automation) && !job.data.event.timestamp) {
             // Requires the timestamp at run time
             job.data.event.timestamp = Date.now()
@@ -71,25 +68,19 @@ export async function processEvent(job: AutomationJob) {
           console.log("automation running", ...loggingArgs(job))
 
           const runFn = () => Runner.run(job)
-          const result = await quotas.addAutomation(runFn, {
+          const result = await quotas.addAutomation(runFn, { automationId })
-            automationId,
-          })
           console.log("automation completed", ...loggingArgs(job))
           return result
-        } catch (err) {
+        })
-          span?.addTags({ error: true })
+      } catch (err) {
-          console.error(
+        span.addTags({ error: true })
-            `automation was unable to run`,
+        console.error(`automation was unable to run`, err, ...loggingArgs(job))
-            err,
+        return { err }
-            ...loggingArgs(job)
-          )
-          return { err }
-        }
       }
-
-      return await context.doInAutomationContext({ appId, automationId, task })
     }
-  )
+
+    return await context.doInAutomationContext({ appId, automationId, task })
+  })
 }
 
 export async function updateTestHistory(
@@ -129,11 +120,11 @@ export async function disableAllCrons(appId: any) {
   return { count: results.length / 2 }
 }
 
-export async function disableCronById(jobId: number | string) {
+export async function disableCronById(jobId: JobId) {
-  const repeatJobs = await automationQueue.getRepeatableJobs()
+  const jobs = await automationQueue.getRepeatableJobs()
-  for (let repeatJob of repeatJobs) {
+  for (const job of jobs) {
-    if (repeatJob.id === jobId) {
+    if (job.id === jobId) {
-      await automationQueue.removeRepeatableByKey(repeatJob.key)
+      await automationQueue.removeRepeatableByKey(job.key)
     }
   }
   console.log(`jobId=${jobId} disabled`)
@@ -222,33 +213,3 @@ export async function enableCronTrigger(appId: any, automation: Automation) {
 export async function cleanupAutomations(appId: any) {
   await disableAllCrons(appId)
 }
-
-/**
- * Checks if the supplied automation is of a recurring type.
- * @param automation The automation to check.
- * @return if it is recurring (cron).
- */
-export function isRecurring(automation: Automation) {
-  return (
-    automation.definition.trigger.stepId ===
-    automations.triggers.definitions.CRON.stepId
-  )
-}
-
-export function isErrorInOutput(output: {
-  steps: { outputs?: { success: boolean } }[]
-}) {
-  let first = true,
-    error = false
-  for (let step of output.steps) {
-    // skip the trigger, its always successful if automation ran
-    if (first) {
-      first = false
-      continue
-    }
-    if (!step.outputs?.success) {
-      error = true
-    }
-  }
-  return error
-}

packages/server/src/constants/index.ts

@@ -130,11 +130,6 @@ export enum InvalidColumns {
   TABLE_ID = "tableId",
 }
 
-export enum AutomationErrors {
-  INCORRECT_TYPE = "INCORRECT_TYPE",
-  FAILURE_CONDITION = "FAILURE_CONDITION_MET",
-}
-
 // pass through the list from the auth/core lib
 export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets
 export const MAX_AUTOMATION_RECURRING_ERRORS = 5

packages/server/src/definitions/automations.ts

@@ -1,4 +1,9 @@
-import { AutomationResults, LoopStepType, UserBindings } from "@budibase/types"
+import {
+  AutomationStepResultOutputs,
+  AutomationTriggerResultOutputs,
+  LoopStepType,
+  UserBindings,
+} from "@budibase/types"
 
 export interface LoopInput {
   option: LoopStepType
@@ -13,19 +18,17 @@ export interface TriggerOutput {
   timestamp?: number
 }
 
-export interface AutomationContext extends AutomationResults {
+export interface AutomationContext {
-  steps: any[]
+  trigger: AutomationTriggerResultOutputs
-  stepsById: Record<string, any>
+  steps: [AutomationTriggerResultOutputs, ...AutomationStepResultOutputs[]]
-  stepsByName: Record<string, any>
+  stepsById: Record<string, AutomationStepResultOutputs>
+  stepsByName: Record<string, AutomationStepResultOutputs>
   env?: Record<string, string>
   user?: UserBindings
-  trigger: any
   settings?: {
     url?: string
     logo?: string
     company?: string
   }
+  loop?: { currentItem: any }
 }
-
-export interface AutomationResponse
-  extends Omit<AutomationContext, "stepsByName" | "stepsById"> {}

packages/server/src/integrations/couchdb.ts

@@ -62,12 +62,16 @@ const SCHEMA: Integration = {
           type: DatasourceFieldType.STRING,
           required: true,
         },
+        rev: {
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
       },
     },
   },
 }
 
-class CouchDBIntegration implements IntegrationBase {
+export class CouchDBIntegration implements IntegrationBase {
   private readonly client: Database
 
   constructor(config: CouchDBConfig) {
@@ -82,7 +86,8 @@ class CouchDBIntegration implements IntegrationBase {
       connected: false,
     }
     try {
-      response.connected = await this.client.exists()
+      await this.client.allDocs({ limit: 1 })
+      response.connected = true
     } catch (e: any) {
       response.error = e.message as string
     }
@@ -99,13 +104,9 @@ class CouchDBIntegration implements IntegrationBase {
   }
 
   async read(query: { json: string | object }) {
-    const parsed = this.parse(query)
+    const params = { include_docs: true, ...this.parse(query) }
-    const params = {
-      include_docs: true,
-      ...parsed,
-    }
     const result = await this.client.allDocs(params)
-    return result.rows.map(row => row.doc)
+    return result.rows.map(row => row.doc!)
   }
 
   async update(query: { json: string | object }) {
@@ -121,8 +122,8 @@ class CouchDBIntegration implements IntegrationBase {
     return await this.client.get(query.id)
   }
 
-  async delete(query: { id: string }) {
+  async delete(query: { id: string; rev: string }) {
-    return await this.client.remove(query.id)
+    return await this.client.remove(query.id, query.rev)
   }
 }
 

packages/server/src/integrations/tests/couchdb.spec.ts

@@ -1,84 +1,87 @@
-jest.mock("@budibase/backend-core", () => {
+import { env } from "@budibase/backend-core"
-  const core = jest.requireActual("@budibase/backend-core")
+import { CouchDBIntegration } from "../couchdb"
-  return {
+import { generator } from "@budibase/backend-core/tests"
-    ...core,
-    db: {
-      ...core.db,
-      DatabaseWithConnection: function () {
-        return {
-          allDocs: jest.fn().mockReturnValue({ rows: [] }),
-          put: jest.fn(),
-          get: jest.fn().mockReturnValue({ _rev: "a" }),
-          remove: jest.fn(),
-        }
-      },
-    },
-  }
-})
 
-import { default as CouchDBIntegration } from "../couchdb"
+function couchSafeID(): string {
+  // CouchDB IDs must start with a letter, so we prepend an 'a'.
+  return `a${generator.guid()}`
+}
 
-class TestConfiguration {
+function doc(data: Record<string, any>): string {
-  integration: any
+  return JSON.stringify({ _id: couchSafeID(), ...data })
+}
 
-  constructor(
+function query(data?: Record<string, any>): { json: string } {
-    config: any = { url: "http://somewhere", database: "something" }
+  return { json: doc(data || {}) }
-  ) {
-    this.integration = new CouchDBIntegration.integration(config)
-  }
 }
 
 describe("CouchDB Integration", () => {
-  let config: any
+  let couchdb: CouchDBIntegration
 
   beforeEach(() => {
-    config = new TestConfiguration()
+    couchdb = new CouchDBIntegration({
-  })
+      url: env.COUCH_DB_URL,
-
+      database: couchSafeID(),
-  it("calls the create method with the correct params", async () => {
-    const doc = {
-      test: 1,
-    }
-    await config.integration.create({
-      json: JSON.stringify(doc),
-    })
-    expect(config.integration.client.put).toHaveBeenCalledWith(doc)
-  })
-
-  it("calls the read method with the correct params", async () => {
-    const doc = {
-      name: "search",
-    }
-
-    await config.integration.read({
-      json: JSON.stringify(doc),
-    })
-
-    expect(config.integration.client.allDocs).toHaveBeenCalledWith({
-      include_docs: true,
-      name: "search",
     })
   })
 
-  it("calls the update method with the correct params", async () => {
+  it("successfully connects", async () => {
-    const doc = {
+    const { connected } = await couchdb.testConnection()
-      _id: "1234",
+    expect(connected).toBe(true)
-      name: "search",
-    }
-
-    await config.integration.update({
-      json: JSON.stringify(doc),
-    })
-
-    expect(config.integration.client.put).toHaveBeenCalledWith({
-      ...doc,
-      _rev: "a",
-    })
   })
 
-  it("calls the delete method with the correct params", async () => {
+  it("can create documents", async () => {
-    const id = "1234"
+    const { id, ok, rev } = await couchdb.create(query({ test: 1 }))
-    await config.integration.delete({ id })
+    expect(id).toBeDefined()
-    expect(config.integration.client.remove).toHaveBeenCalledWith(id)
+    expect(ok).toBe(true)
+    expect(rev).toBeDefined()
+  })
+
+  it("can read created documents", async () => {
+    const { id, ok, rev } = await couchdb.create(query({ test: 1 }))
+    expect(id).toBeDefined()
+    expect(ok).toBe(true)
+    expect(rev).toBeDefined()
+
+    const docs = await couchdb.read(query())
+    expect(docs).toEqual([
+      {
+        _id: id,
+        _rev: rev,
+        test: 1,
+        createdAt: expect.any(String),
+        updatedAt: expect.any(String),
+      },
+    ])
+  })
+
+  it("can update documents", async () => {
+    const { id, ok, rev } = await couchdb.create(query({ test: 1 }))
+    expect(ok).toBe(true)
+
+    const { id: newId, rev: newRev } = await couchdb.update(
+      query({ _id: id, _rev: rev, test: 2 })
+    )
+    const docs = await couchdb.read(query())
+    expect(docs).toEqual([
+      {
+        _id: newId,
+        _rev: newRev,
+        test: 2,
+        createdAt: expect.any(String),
+        updatedAt: expect.any(String),
+      },
+    ])
+  })
+
+  it("can delete documents", async () => {
+    const { id, ok, rev } = await couchdb.create(query({ test: 1 }))
+    expect(ok).toBe(true)
+
+    const deleteResponse = await couchdb.delete({ id, rev })
+    expect(deleteResponse.ok).toBe(true)
+
+    const docs = await couchdb.read(query())
+    expect(docs).toBeEmpty()
   })
 })

packages/server/src/sdk/app/automations/crud.ts

@@ -40,7 +40,8 @@ function cleanAutomationInputs(automation: Automation) {
     if (step == null) {
       continue
     }
-    for (let inputName of Object.keys(step.inputs)) {
+    for (const key of Object.keys(step.inputs)) {
+      const inputName = key as keyof typeof step.inputs
       if (!step.inputs[inputName] || step.inputs[inputName] === "") {
         delete step.inputs[inputName]
       }
@@ -281,7 +282,8 @@ function guardInvalidUpdatesAndThrow(
     const readonlyFields = Object.keys(
       step.schema.inputs.properties || {}
     ).filter(k => step.schema.inputs.properties[k].readonly)
-    readonlyFields.forEach(readonlyField => {
+    readonlyFields.forEach(key => {
+      const readonlyField = key as keyof typeof step.inputs
       const oldStep = oldStepDefinitions.find(i => i.id === step.id)
       if (step.inputs[readonlyField] !== oldStep?.inputs[readonlyField]) {
         throw new HTTPError(

packages/server/src/sdk/app/automations/tests/index.spec.ts

@@ -1,69 +0,0 @@
-import { sample } from "lodash/fp"
-import { Automation } from "@budibase/types"
-import { generator } from "@budibase/backend-core/tests"
-import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
-import automationSdk from "../"
-import { structures } from "../../../../api/routes/tests/utilities"
-
-describe("automation sdk", () => {
-  const config = new TestConfiguration()
-
-  beforeAll(async () => {
-    await config.init()
-  })
-
-  describe("update", () => {
-    it("can rename existing automations", async () => {
-      await config.doInContext(config.getAppId(), async () => {
-        const automation = structures.newAutomation()
-
-        const response = await automationSdk.create(automation)
-
-        const newName = generator.guid()
-        const update = { ...response, name: newName }
-        const result = await automationSdk.update(update)
-        expect(result.name).toEqual(newName)
-      })
-    })
-
-    it.each([
-      ["trigger", (a: Automation) => a.definition.trigger],
-      ["step", (a: Automation) => a.definition.steps[0]],
-    ])("can update input fields (for a %s)", async (_, getStep) => {
-      await config.doInContext(config.getAppId(), async () => {
-        const automation = structures.newAutomation()
-
-        const keyToUse = sample(Object.keys(getStep(automation).inputs))!
-        getStep(automation).inputs[keyToUse] = "anyValue"
-
-        const response = await automationSdk.create(automation)
-
-        const update = { ...response }
-        getStep(update).inputs[keyToUse] = "anyUpdatedValue"
-        const result = await automationSdk.update(update)
-        expect(getStep(result).inputs[keyToUse]).toEqual("anyUpdatedValue")
-      })
-    })
-
-    it.each([
-      ["trigger", (a: Automation) => a.definition.trigger],
-      ["step", (a: Automation) => a.definition.steps[0]],
-    ])("cannot update readonly fields (for a %s)", async (_, getStep) => {
-      await config.doInContext(config.getAppId(), async () => {
-        const automation = structures.newAutomation()
-        getStep(automation).schema.inputs.properties["readonlyProperty"] = {
-          readonly: true,
-        }
-        getStep(automation).inputs["readonlyProperty"] = "anyValue"
-
-        const response = await automationSdk.create(automation)
-
-        const update = { ...response }
-        getStep(update).inputs["readonlyProperty"] = "anyUpdatedValue"
-        await expect(automationSdk.update(update)).rejects.toThrow(
-          "Field readonlyProperty is readonly and it cannot be modified"
-        )
-      })
-    })
-  })
-})

packages/server/src/tests/utilities/structures.ts

@@ -35,6 +35,8 @@ import {
   WebhookActionType,
   BuiltinPermissionID,
   DeepPartial,
+  FilterCondition,
+  AutomationTriggerResult,
 } from "@budibase/types"
 import { LoopInput } from "../../definitions/automations"
 import { merge } from "lodash"
@@ -372,7 +374,11 @@ export function filterAutomation(opts?: DeepPartial<Automation>): Automation {
           type: AutomationStepType.ACTION,
           internal: true,
           stepId: AutomationActionStepId.FILTER,
-          inputs: { field: "name", value: "test", condition: "EQ" },
+          inputs: {
+            field: "name",
+            value: "test",
+            condition: FilterCondition.EQUAL,
+          },
           schema: BUILTIN_ACTION_DEFINITIONS.EXECUTE_SCRIPT.schema,
         },
       ],
@@ -437,15 +443,24 @@ export function updateRowAutomationWithFilters(
 export function basicAutomationResults(
   automationId: string
 ): AutomationResults {
+  const trigger: AutomationTriggerResult = {
+    id: "trigger",
+    stepId: AutomationTriggerStepId.APP,
+    outputs: {},
+  }
   return {
     automationId,
     status: AutomationStatus.SUCCESS,
-    trigger: "trigger" as any,
+    trigger,
     steps: [
+      trigger,
       {
+        id: "step1",
         stepId: AutomationActionStepId.SERVER_LOG,
         inputs: {},
-        outputs: {},
+        outputs: {
+          success: true,
+        },
       },
     ],
   }

packages/shared-core/src/automations/steps/filter.ts

@@ -3,20 +3,14 @@ import {
   AutomationStepDefinition,
   AutomationStepType,
   AutomationIOType,
+  FilterCondition,
 } from "@budibase/types"
 
-export const FilterConditions = {
-  EQUAL: "EQUAL",
-  NOT_EQUAL: "NOT_EQUAL",
-  GREATER_THAN: "GREATER_THAN",
-  LESS_THAN: "LESS_THAN",
-}
-
 export const PrettyFilterConditions = {
-  [FilterConditions.EQUAL]: "Equals",
+  [FilterCondition.EQUAL]: "Equals",
-  [FilterConditions.NOT_EQUAL]: "Not equals",
+  [FilterCondition.NOT_EQUAL]: "Not equals",
-  [FilterConditions.GREATER_THAN]: "Greater than",
+  [FilterCondition.GREATER_THAN]: "Greater than",
-  [FilterConditions.LESS_THAN]: "Less than",
+  [FilterCondition.LESS_THAN]: "Less than",
 }
 
 export const definition: AutomationStepDefinition = {
@@ -30,7 +24,7 @@ export const definition: AutomationStepDefinition = {
   features: {},
   stepId: AutomationActionStepId.FILTER,
   inputs: {
-    condition: FilterConditions.EQUAL,
+    condition: FilterCondition.EQUAL,
   },
   schema: {
     inputs: {
@@ -42,7 +36,7 @@ export const definition: AutomationStepDefinition = {
         condition: {
           type: AutomationIOType.STRING,
           title: "Condition",
-          enum: Object.values(FilterConditions),
+          enum: Object.values(FilterCondition),
           pretty: Object.values(PrettyFilterConditions),
         },
         value: {

packages/shared-core/src/helpers/helpers.ts

@@ -105,10 +105,10 @@ export function cancelableTimeout(
 
 export async function withTimeout<T>(
   timeout: number,
-  promise: Promise<T>
+  promise: () => Promise<T>
 ): Promise<T> {
   const [timeoutPromise, cancel] = cancelableTimeout(timeout)
-  const result = (await Promise.race([promise, timeoutPromise])) as T
+  const result = (await Promise.race([promise(), timeoutPromise])) as T
   cancel()
   return result
 }

packages/types/src/documents/app/automation/StepInputsOutputs.ts

@@ -8,8 +8,20 @@ import {
 } from "../../../sdk"
 import { HttpMethod } from "../query"
 import { Row } from "../row"
-import { LoopStepType, EmailAttachment, AutomationResults } from "./automation"
+import {
-import { AutomationStep, AutomationStepOutputs } from "./schema"
+  LoopStepType,
+  EmailAttachment,
+  AutomationResults,
+  AutomationStepResult,
+} from "./automation"
+import { AutomationStep } from "./schema"
+
+export enum FilterCondition {
+  EQUAL = "EQUAL",
+  NOT_EQUAL = "NOT_EQUAL",
+  GREATER_THAN = "GREATER_THAN",
+  LESS_THAN = "LESS_THAN",
+}
 
 export type BaseAutomationOutputs = {
   success?: boolean
@@ -93,7 +105,7 @@ export type ExecuteScriptStepOutputs = BaseAutomationOutputs & {
 
 export type FilterStepInputs = {
   field: any
-  condition: string
+  condition: FilterCondition
   value: any
 }
 
@@ -111,7 +123,7 @@ export type LoopStepInputs = {
 }
 
 export type LoopStepOutputs = {
-  items: AutomationStepOutputs[]
+  items: AutomationStepResult[]
   success: boolean
   iterations: number
 }

packages/types/src/documents/app/automation/automation.ts

@@ -146,10 +146,10 @@ export interface Automation extends Document {
   internal?: boolean
   type?: string
   disabled?: boolean
-  testData?: TriggerTestOutputs
+  testData?: AutomationTriggerResultOutputs
 }
 
-interface BaseIOStructure {
+export interface BaseIOStructure {
   type?: AutomationIOType
   subtype?: AutomationIOType
   customType?: AutomationCustomIOType
@@ -179,6 +179,8 @@ export enum AutomationFeature {
 export enum AutomationStepStatus {
   NO_ITERATIONS = "no_iterations",
   MAX_ITERATIONS = "max_iterations_reached",
+  FAILURE_CONDITION = "FAILURE_CONDITION_MET",
+  INCORRECT_TYPE = "INCORRECT_TYPE",
 }
 
 export enum AutomationStatus {
@@ -193,45 +195,37 @@ export enum AutomationStoppedReason {
   TRIGGER_FILTER_NOT_MET = "Automation did not run. Filter conditions in trigger were not met.",
 }
 
-// UI and context fields
+export interface AutomationStepResultOutputs {
-export type AutomationResultFields = {
+  success: boolean
-  meta?: {
+  [key: string]: any
-    [key: string]: unknown
-  }
-  user?: AppSelfResponse
-  automation?: Automation
 }
 
-// Base types for Trigger outputs combined with UI and context fields
+export interface AutomationStepResultInputs {
-export type TriggerTestOutputs =
+  [key: string]: any
-  | (WebhookTriggerOutputs & AutomationResultFields)
+}
-  | (AppActionTriggerOutputs & AutomationResultFields)
-  | (CronTriggerOutputs & AutomationResultFields)
-  | (RowActionTriggerOutputs & AutomationResultFields)
-  | (RowDeletedTriggerInputs & AutomationResultFields)
-  | (RowCreatedTriggerInputs & AutomationResultFields)
-  | (RowUpdatedTriggerOutputs & AutomationResultFields)
 
-export type AutomationTestTrigger = {
+export interface AutomationStepResult {
+  id: string
+  stepId: AutomationActionStepId
+  inputs: AutomationStepResultInputs
+  outputs: AutomationStepResultOutputs
+}
+
+export type AutomationTriggerResultInputs = Record<string, any>
+export type AutomationTriggerResultOutputs = Record<string, any>
+
+export interface AutomationTriggerResult {
   id: string
-  inputs: null
   stepId: AutomationTriggerStepId
-  outputs: TriggerTestOutputs
+  inputs?: AutomationTriggerResultInputs | null
+  outputs: AutomationTriggerResultOutputs
 }
 
 export interface AutomationResults {
   automationId?: string
   status?: AutomationStatus
-  trigger?: AutomationTestTrigger
+  trigger: AutomationTriggerResult
-  steps: {
+  steps: [AutomationTriggerResult, ...AutomationStepResult[]]
-    stepId: AutomationTriggerStepId | AutomationActionStepId
-    inputs: {
-      [key: string]: any
-    }
-    outputs: {
-      [key: string]: any
-    }
-  }[]
 }
 
 export interface DidNotTriggerResponse {
@@ -265,6 +259,7 @@ export type ActionImplementation<TInputs, TOutputs> = (
     inputs: TInputs
   } & AutomationStepInputBase
 ) => Promise<TOutputs>
+
 export interface AutomationMetadata extends Document {
   errorCount?: number
   automationChainCount?: number

packages/types/src/documents/app/automation/schema.ts

@@ -169,24 +169,6 @@ export interface AutomationStepSchemaBase {
   features?: Partial<Record<AutomationFeature, boolean>>
 }
 
-export type AutomationStepOutputs =
-  | CollectStepOutputs
-  | CreateRowStepOutputs
-  | DelayStepOutputs
-  | DeleteRowStepOutputs
-  | ExecuteQueryStepOutputs
-  | ExecuteScriptStepOutputs
-  | FilterStepOutputs
-  | QueryRowsStepOutputs
-  | BaseAutomationOutputs
-  | BashStepOutputs
-  | ExternalAppStepOutputs
-  | OpenAIStepOutputs
-  | ServerLogStepOutputs
-  | TriggerAutomationStepOutputs
-  | UpdateRowStepOutputs
-  | ZapierStepOutputs
-
 export type AutomationStepInputs<T extends AutomationActionStepId> =
   T extends AutomationActionStepId.COLLECT
     ? CollectStepInputs
@@ -236,11 +218,56 @@ export type AutomationStepInputs<T extends AutomationActionStepId> =
     ? BranchStepInputs
     : never
 
+export type AutomationStepOutputs<T extends AutomationActionStepId> =
+  T extends AutomationActionStepId.COLLECT
+    ? CollectStepOutputs
+    : T extends AutomationActionStepId.CREATE_ROW
+    ? CreateRowStepOutputs
+    : T extends AutomationActionStepId.DELAY
+    ? DelayStepOutputs
+    : T extends AutomationActionStepId.DELETE_ROW
+    ? DeleteRowStepOutputs
+    : T extends AutomationActionStepId.EXECUTE_QUERY
+    ? ExecuteQueryStepOutputs
+    : T extends AutomationActionStepId.EXECUTE_SCRIPT
+    ? ExecuteScriptStepOutputs
+    : T extends AutomationActionStepId.FILTER
+    ? FilterStepOutputs
+    : T extends AutomationActionStepId.QUERY_ROWS
+    ? QueryRowsStepOutputs
+    : T extends AutomationActionStepId.SEND_EMAIL_SMTP
+    ? BaseAutomationOutputs
+    : T extends AutomationActionStepId.SERVER_LOG
+    ? ServerLogStepOutputs
+    : T extends AutomationActionStepId.TRIGGER_AUTOMATION_RUN
+    ? TriggerAutomationStepOutputs
+    : T extends AutomationActionStepId.UPDATE_ROW
+    ? UpdateRowStepOutputs
+    : T extends AutomationActionStepId.OUTGOING_WEBHOOK
+    ? ExternalAppStepOutputs
+    : T extends AutomationActionStepId.discord
+    ? ExternalAppStepOutputs
+    : T extends AutomationActionStepId.slack
+    ? ExternalAppStepOutputs
+    : T extends AutomationActionStepId.zapier
+    ? ZapierStepOutputs
+    : T extends AutomationActionStepId.integromat
+    ? ExternalAppStepOutputs
+    : T extends AutomationActionStepId.n8n
+    ? ExternalAppStepOutputs
+    : T extends AutomationActionStepId.EXECUTE_BASH
+    ? BashStepOutputs
+    : T extends AutomationActionStepId.OPENAI
+    ? OpenAIStepOutputs
+    : T extends AutomationActionStepId.LOOP
+    ? BaseAutomationOutputs
+    : never
+
 export interface AutomationStepSchema<TStep extends AutomationActionStepId>
   extends AutomationStepSchemaBase {
   id: string
   stepId: TStep
-  inputs: AutomationStepInputs<TStep> & Record<string, any> // The record union to be removed once the types are fixed
+  inputs: AutomationStepInputs<TStep>
 }
 
 export type CollectStep = AutomationStepSchema<AutomationActionStepId.COLLECT>
@@ -326,6 +353,36 @@ export type AutomationStep =
   | OpenAIStep
   | BranchStep
 
+export function isBranchStep(
+  step: AutomationStep | AutomationTrigger
+): step is BranchStep {
+  return step.stepId === AutomationActionStepId.BRANCH
+}
+
+export function isTrigger(
+  step: AutomationStep | AutomationTrigger
+): step is AutomationTrigger {
+  return step.type === AutomationStepType.TRIGGER
+}
+
+export function isRowUpdateTrigger(
+  step: AutomationStep | AutomationTrigger
+): step is RowUpdatedTrigger {
+  return step.stepId === AutomationTriggerStepId.ROW_UPDATED
+}
+
+export function isRowSaveTrigger(
+  step: AutomationStep | AutomationTrigger
+): step is RowSavedTrigger {
+  return step.stepId === AutomationTriggerStepId.ROW_SAVED
+}
+
+export function isAppTrigger(
+  step: AutomationStep | AutomationTrigger
+): step is AppActionTrigger {
+  return step.stepId === AutomationTriggerStepId.APP
+}
+
 type EmptyInputs = {}
 export type AutomationStepDefinition = Omit<AutomationStep, "id" | "inputs"> & {
   inputs: EmptyInputs

packages/types/src/sdk/search.ts

@@ -82,6 +82,10 @@ type RangeFilter = Record<
 
 type LogicalFilter = { conditions: SearchFilters[] }
 
+export function isLogicalFilter(filter: any): filter is LogicalFilter {
+  return "conditions" in filter
+}
+
 export type AnySearchFilter = BasicFilter | ArrayFilter | RangeFilter
 
 export interface SearchFilters {

packages/worker/package.json

@@ -82,7 +82,6 @@
     "@swc/jest": "0.2.27",
     "@types/jest": "29.5.5",
     "@types/jsonwebtoken": "9.0.3",
-    "@types/koa": "2.13.4",
     "@types/koa__router": "12.0.4",
     "@types/lodash": "4.14.200",
     "@types/node-fetch": "2.6.4",

packages/worker/src/api/routes/global/tests/realEmail.spec.ts

@@ -31,8 +31,8 @@ describe("/api/global/email", () => {
   ) {
     let response, text
     try {
-      await helpers.withTimeout(20000, config.saveEtherealSmtpConfig())
+      await helpers.withTimeout(20000, () => config.saveEtherealSmtpConfig())
-      await helpers.withTimeout(20000, config.saveSettingsConfig())
+      await helpers.withTimeout(20000, () => config.saveSettingsConfig())
       let res
       if (attachments) {
         res = await config.api.emails

scripts/install-minio.sh

@@ -1,10 +1,18 @@
 #!/bin/bash
-if [[ $TARGETARCH == arm* ]] ;
+
-then
+if [[ $TARGETBUILD == "aas" ]]; then
+  echo "A aas-compatible version of Minio is already installed."
+  exit 0
+fi
+
+if [[ $TARGETARCH == arm* ]]; then
   echo "INSTALLING ARM64 MINIO"
+  rm -f minio
   wget https://dl.min.io/server/minio/release/linux-arm64/minio
 else
   echo "INSTALLING AMD64 MINIO"
+  rm -f minio
   wget https://dl.min.io/server/minio/release/linux-amd64/minio
 fi
-chmod +x minio
+
+chmod +x minio

scripts/resources/minio

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63db3aa3c2299ebaf13b46c64523a589bd5bf272f9e971d17f1eaa55f6f1fd79
+size 118595584

yarn.lock

@@ -2778,9 +2778,9 @@
     through2 "^2.0.0"
 
 "@budibase/pro@npm:@budibase/pro@latest":
-  version "3.4.6"
+  version "3.4.12"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.6.tgz#62b6ee13a015b98d4768dc7821f468f8177da3e9"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.12.tgz#60e630944de4e2de970a04179d8f0f57d48ce75e"
-  integrity sha512-MC3P5SMokmqbjejZMlNM6z7NB9o5H6hZ++yVvbyThniBPYfuDc2ssa1HNwwcuNE3uRLhcxcKe8CY/0SbFgn51g==
+  integrity sha512-msUBmcWxRDg+ugjZvd27XudERQqtQRdiARsO8MaDVTcp5ejIXgshEIVVshHOCj3hcbRblw9pXvBIMI53iTMUsA==
   dependencies:
     "@anthropic-ai/sdk" "^0.27.3"
     "@budibase/backend-core" "*"