Automation trigger filtering (#14123)

* backend for triggering automation based on filters

* frontend for handling triggering automations on filter / old row

* lint and bug fix

* fix issue with test header

* make test data optional

* improve safety on trigger gate

* add support for running trigger with filter if no change happened but filter matches

* update var naming to actually make sense

* tests

* fix lint

* improve gating for shouldTrigger check

* remove unecessary cast

* unecessary tableId check

* frontend text updates

* resolving comments

* pro

* Update packages/types/src/documents/app/automation.ts

Co-authored-by: Sam Rose <hello@samwho.dev>

* link out to docs for trigger filtering

* fix pro

* more pr comments

* use getAppId

---------

Co-authored-by: Sam Rose <hello@samwho.dev>

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte

@@ -16,13 +16,12 @@
   export let enableNaming = true
   let validRegex = /^[A-Za-z0-9_\s]+$/
   let typing = false
-
   const dispatch = createEventDispatcher()
 
   $: stepNames = $selectedAutomation?.definition.stepNames
   $: automationName = stepNames?.[block.id] || block?.name || ""
   $: automationNameError = getAutomationNameError(automationName)
-  $: status = updateStatus(testResult, isTrigger)
+  $: status = updateStatus(testResult)
   $: isHeaderTrigger = isTrigger || block.type === "TRIGGER"
 
   $: {
@@ -43,7 +42,7 @@
     })
   }
 
-  function updateStatus(results, isTrigger) {
+  function updateStatus(results) {
     if (!results) {
       return {}
     }
@@ -56,7 +55,6 @@
       return { negative: true, message: "Error" }
     }
   }
-
   const getAutomationNameError = name => {
     if (stepNames) {
       for (const [key, value] of Object.entries(stepNames)) {

packages/builder/src/components/automation/AutomationBuilder/TestDisplay.svelte

@@ -12,14 +12,31 @@
   let blocks
 
   function prepTestResults(results) {
-    return results?.steps.filter(x => x.stepId !== ActionStepID.LOOP || [])
+    if (results.message) {
+      return [
+        {
+          inputs: {},
+          outputs: {
+            success: results.outputs?.success || false,
+            status: results.outputs?.status || "unknown",
+            message: results.message,
+          },
+        },
+      ]
+    } else {
+      return results?.steps?.filter(x => x.stepId !== ActionStepID.LOOP) || []
+    }
   }
 
   $: filteredResults = prepTestResults(testResults)
 
   $: {
+    if (testResults.message) {
+      blocks = automation?.definition?.trigger
+        ? [automation.definition.trigger]
+        : []
+    } else if (automation) {
       blocks = []
-    if (automation) {
       if (automation.definition.trigger) {
         blocks.push(automation.definition.trigger)
       }
@@ -46,7 +63,9 @@
           open={!!openBlocks[block.id]}
           on:toggle={() => (openBlocks[block.id] = !openBlocks[block.id])}
           isTrigger={idx === 0}
-          testResult={filteredResults?.[idx]}
+          testResult={testResults.message
+            ? testResults
+            : filteredResults?.[idx]}
           showTestStatus
           {block}
           {idx}
@@ -68,7 +87,9 @@
             <Tabs quiet noHorizPadding selected="Input">
               <Tab title="Input">
                 <div class="wrap">
-                  {#if filteredResults?.[idx]?.inputs}
+                  {#if testResults.message}
+                    No input
+                  {:else if filteredResults?.[idx]?.inputs}
                     <JsonView depth={2} json={filteredResults?.[idx]?.inputs} />
                   {:else}
                     No input
@@ -77,13 +98,22 @@
               </Tab>
               <Tab title="Output">
                 <div class="wrap">
-                  {#if filteredResults?.[idx]?.outputs}
+                  {#if testResults.message}
+                    <JsonView
+                      depth={2}
+                      json={{
+                        success: testResults.outputs?.success || false,
+                        status: testResults.outputs?.status || "unknown",
+                        message: testResults.message,
+                      }}
+                    />
+                  {:else if filteredResults?.[idx]?.outputs}
                     <JsonView
                       depth={2}
                       json={filteredResults?.[idx]?.outputs}
                     />
                   {:else}
-                    No input
+                    No output
                   {/if}
                 </div>
               </Tab>

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -17,7 +17,9 @@
     Helpers,
     Toggle,
     Divider,
+    Icon,
   } from "@budibase/bbui"
+
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
   import { automationStore, selectedAutomation, tables } from "stores/builder"
   import { environment, licensing } from "stores/portal"
@@ -365,41 +367,74 @@
 
   /**
    * Handler for row trigger automation updates.
-    @param {object} update - An automation block.inputs update object
+   * @param {object} update - An automation block.inputs update object
-    @example
+   * @param {string} [update.tableId] - The ID of the table
-    onRowTriggerUpdate({ 
+   * @param {object} [update.filters] - Filter configuration for the row trigger
-      "tableId" : "ta_bb_employee"
+   * @param {object} [update.filters-def] - Filter definitions for the row trigger
-    })
+   * @example
+   * // Example with tableId
+   * onRowTriggerUpdate({
+   *   "tableId" : "ta_bb_employee"
+   * })
+   * @example
+   * // Example with filters
+   * onRowTriggerUpdate({
+   *   filters: {
+   *     equal: { "1:Approved": "true" }
+   *   },
+   *   "filters-def": [{
+   *     id: "oH1T4S49n",
+   *     field: "1:Approved",
+   *     operator: "equal",
+   *     value: "true",
+   *     valueType: "Value",
+   *     type: "string"
+   *   }]
+   * })
    */
   const onRowTriggerUpdate = async update => {
+    if (
+      ["tableId", "filters", "meta"].some(key => Object.hasOwn(update, key))
+    ) {
+      try {
+        let updatedAutomation
+
         if (
           Object.hasOwn(update, "tableId") &&
           $selectedAutomation.testData?.row?.tableId !== update.tableId
         ) {
-      try {
           const reqSchema = getSchemaForDatasourcePlus(update.tableId, {
             searchableSchema: true,
           }).schema
 
-        // Parse the block inputs as usual
+          updatedAutomation = await automationStore.actions.processBlockInputs(
-        const updatedAutomation =
+            block,
-          await automationStore.actions.processBlockInputs(block, {
+            {
               schema: reqSchema,
               ...update,
-          })
+            }
+          )
 
-        // Save the entire automation and reset the testData
+          // Reset testData when tableId changes
-        await automationStore.actions.save({
+          updatedAutomation = {
             ...updatedAutomation,
             testData: {
-            // Reset Core fields
               row: { tableId: update.tableId },
               oldRow: { tableId: update.tableId },
               meta: {},
               id: "",
               revision: "",
             },
-        })
+          }
+        } else {
+          // For filters update, just process block inputs without resetting testData
+          updatedAutomation = await automationStore.actions.processBlockInputs(
+            block,
+            update
+          )
+        }
+
+        await automationStore.actions.save(updatedAutomation)
 
         return
       } catch (e) {
@@ -408,7 +443,6 @@
       }
     }
   }
-
   /**
    * Handler for App trigger automation updates.
    * Ensure updates to the field list are reflected in testData
@@ -743,6 +777,7 @@
       value.customType !== "triggerSchema" &&
       value.customType !== "automationFields" &&
       value.customType !== "fields" &&
+      value.customType !== "trigger_filter_setting" &&
       value.type !== "signature_single" &&
       value.type !== "attachment" &&
       value.type !== "attachment_single"
@@ -807,13 +842,23 @@
         {@const label = getFieldLabel(key, value)}
         <div class:block-field={shouldRenderField(value)}>
           {#if key !== "fields" && value.type !== "boolean" && shouldRenderField(value)}
-            <Label
+            <div class="label-container">
-              tooltip={value.title === "Binding / Value"
+              <Label>
-                ? "If using the String input type, please use a comma or newline separated string"
-                : null}
-            >
                 {label}
               </Label>
+              {#if value.customType === "trigger_filter"}
+                <Icon
+                  hoverable
+                  on:click={() =>
+                    window.open(
+                      "https://docs.budibase.com/docs/row-trigger-filters",
+                      "_blank"
+                    )}
+                  size="XS"
+                  name="InfoOutline"
+                />
+              {/if}
+            </div>
           {/if}
           <div class:field-width={shouldRenderField(value)}>
             {#if value.type === "string" && value.enum && canShowField(key, value)}
@@ -932,8 +977,12 @@
                   {/if}
                 </div>
               </div>
-            {:else if value.customType === "filters"}
+            {:else if value.customType === "filters" || value.customType === "trigger_filter"}
-              <ActionButton on:click={drawer.show}>Define filters</ActionButton>
+              <ActionButton fullWidth on:click={drawer.show}
+                >{filters.length > 0
+                  ? "Update Filter"
+                  : "No Filter set"}</ActionButton
+              >
               <Drawer bind:this={drawer} title="Filtering">
                 <Button cta slot="buttons" on:click={() => saveFilters(key)}>
                   Save
@@ -945,6 +994,7 @@
                     {schemaFields}
                     datasource={{ type: "table", tableId }}
                     panel={AutomationBindingPanel}
+                    showFilterEmptyDropdown={!rowTriggers.includes(stepId)}
                     on:change={e => (tempFilters = e.detail)}
                   />
                 </DrawerContent>
@@ -1085,6 +1135,11 @@
 {/if}
 
 <style>
+  .label-container {
+    display: flex;
+    align-items: center;
+    gap: var(--spacing-s);
+  }
   .field-width {
     width: 320px;
   }

packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte

@@ -14,7 +14,7 @@
   export let panel = ClientBindingPanel
   export let allowBindings = true
   export let datasource
-
+  export let showFilterEmptyDropdown
   const dispatch = createEventDispatcher()
 
   let rawFilters
@@ -63,6 +63,7 @@
   {schemaFields}
   {datasource}
   {allowBindings}
+  {showFilterEmptyDropdown}
 >
   <div slot="filtering-hero-content" />
 

packages/builder/src/stores/builder/automations.js

@@ -208,7 +208,7 @@ const automationActions = store => ({
       const message = err.message || err.status || JSON.stringify(err)
       throw `Automation test failed - ${message}`
     }
-    if (!result?.trigger && !result?.steps?.length) {
+    if (!result?.trigger && !result?.steps?.length && !result?.message) {
       if (result?.err?.code === "usage_limit_exceeded") {
         throw "You have exceeded your automation quota"
       }

packages/frontend-core/src/components/FilterBuilder.svelte

@@ -26,7 +26,7 @@
   export let behaviourFilters = false
   export let allowBindings = false
   export let filtersLabel = "Filters"
-
+  export let showFilterEmptyDropdown = true
   $: {
     if (
       tables.find(
@@ -218,7 +218,7 @@
                 on:change={e => handleAllOr(e.detail)}
                 placeholder={null}
               />
-              {#if datasource?.type === "table"}
+              {#if datasource?.type === "table" && showFilterEmptyDropdown}
                 <Select
                   label="When filter empty"
                   value={onEmptyFilter}

packages/server/src/api/routes/tests/automation.spec.ts

@@ -11,7 +11,7 @@ import {
 } from "../../../automations"
 import { events } from "@budibase/backend-core"
 import sdk from "../../../sdk"
-import { Automation } from "@budibase/types"
+import { Automation, FieldType, Table } from "@budibase/types"
 import { mocks } from "@budibase/backend-core/tests"
 import { FilterConditions } from "../../../automations/steps/filter"
 
@@ -23,6 +23,7 @@ let {
   automationStep,
   collectAutomation,
   filterAutomation,
+  updateRowAutomationWithFilters,
 } = setup.structures
 
 describe("/automations", () => {
@@ -452,14 +453,13 @@ describe("/automations", () => {
 
         let table = await config.createTable()
 
-        let automation = await filterAutomation()
+        let automation = await filterAutomation(config.getAppId())
         automation.definition.trigger.inputs.tableId = table._id
         automation.definition.steps[0].inputs = {
           condition: FilterConditions.EQUAL,
           field: "{{ trigger.row.City }}",
           value: "{{ trigger.oldRow.City }}",
         }
-        automation.appId = config.appId!
         automation = await config.createAutomation(automation)
         let triggerInputs = {
           oldRow: {
@@ -474,4 +474,91 @@ describe("/automations", () => {
       }
     )
   })
+  describe("Automation Update / Creator row trigger filtering", () => {
+    let table: Table
+
+    beforeAll(async () => {
+      table = await config.createTable({
+        name: "table",
+        type: "table",
+        schema: {
+          Approved: {
+            name: "Approved",
+            type: FieldType.BOOLEAN,
+          },
+        },
+      })
+    })
+
+    const testCases = [
+      {
+        description: "should run when Approved changes from false to true",
+        filters: {
+          equal: { "1:Approved": true },
+        },
+        row: { Approved: "true" },
+        oldRow: { Approved: "false" },
+        expectToRun: true,
+      },
+      {
+        description: "should run when Approved is true in both old and new row",
+        filters: { equal: { "1:Approved": true } },
+        row: { Approved: "true" },
+        oldRow: { Approved: "true" },
+        expectToRun: true,
+      },
+
+      {
+        description:
+          "should run when a contains filter matches the correct options",
+        filters: {
+          contains: { "1:opts": ["Option 1", "Option 3"] },
+        },
+        row: { opts: ["Option 1", "Option 3"] },
+        oldRow: { opts: ["Option 3"] },
+        expectToRun: true,
+      },
+      {
+        description:
+          "should not run when opts doesn't contain any specified option",
+        filters: {
+          contains: { "1:opts": ["Option 1", "Option 2"] },
+        },
+        row: { opts: ["Option 3", "Option 4"] },
+        oldRow: { opts: ["Option 3", "Option 4"] },
+        expectToRun: false,
+      },
+    ]
+
+    it.each(testCases)(
+      "$description",
+      async ({ filters, row, oldRow, expectToRun }) => {
+        let automation = await updateRowAutomationWithFilters(config.getAppId())
+        automation.definition.trigger.inputs = {
+          tableId: table._id,
+          filters,
+        }
+        automation = await config.createAutomation(automation)
+
+        const inputs = {
+          row: {
+            tableId: table._id,
+            ...row,
+          },
+          oldRow: {
+            tableId: table._id,
+            ...oldRow,
+          },
+        }
+
+        const res = await testAutomation(config, automation, inputs)
+
+        if (expectToRun) {
+          expect(res.body.steps[1].outputs.success).toEqual(true)
+        } else {
+          expect(res.body.outputs.success).toEqual(false)
+        }
+      }
+    )
+  })
 })

packages/server/src/automations/triggerInfo/rowSaved.ts

@@ -23,6 +23,11 @@ export const definition: AutomationTriggerSchema = {
           customType: AutomationCustomIOType.TABLE,
           title: "Table",
         },
+        filters: {
+          type: AutomationIOType.OBJECT,
+          customType: AutomationCustomIOType.FILTERS,
+          title: "Filtering",
+        },
       },
       required: ["tableId"],
     },

packages/server/src/automations/triggerInfo/rowUpdated.ts

@@ -23,6 +23,11 @@ export const definition: AutomationTriggerSchema = {
           customType: AutomationCustomIOType.TABLE,
           title: "Table",
         },
+        filters: {
+          type: AutomationIOType.OBJECT,
+          customType: AutomationCustomIOType.TRIGGER_FILTER,
+          title: "Filtering",
+        },
       },
       required: ["tableId"],
     },

packages/server/src/automations/triggers.ts

@@ -15,14 +15,19 @@ import {
   AutomationJob,
   AutomationEventType,
   UpdatedRowEventEmitter,
+  SearchFilters,
+  AutomationStoppedReason,
+  AutomationStatus,
 } from "@budibase/types"
 import { executeInThread } from "../threads/automation"
+import { dataFilters } from "@budibase/shared-core"
 
 export const TRIGGER_DEFINITIONS = definitions
 const JOB_OPTS = {
   removeOnComplete: true,
   removeOnFail: true,
 }
+import * as automationUtils from "../automations/automationUtils"
 
 async function getAllAutomations() {
   const db = context.getAppDB()
@@ -33,7 +38,7 @@ async function getAllAutomations() {
 }
 
 async function queueRelevantRowAutomations(
-  event: { appId: string; row: Row },
+  event: { appId: string; row: Row; oldRow: Row },
   eventType: string
 ) {
   if (event.appId == null) {
@@ -62,9 +67,15 @@ async function queueRelevantRowAutomations(
       ) {
         continue
       }
+
+      const shouldTrigger = await checkTriggerFilters(automation, {
+        row: event.row,
+        oldRow: event.oldRow,
+      })
       if (
         automationTrigger?.inputs &&
-        automationTrigger.inputs.tableId === event.row.tableId
+        automationTrigger.inputs.tableId === event.row.tableId &&
+        shouldTrigger
       ) {
         try {
           await automationQueue.add({ automation, event }, JOB_OPTS)
@@ -103,6 +114,11 @@ emitter.on(AutomationEventType.ROW_DELETE, async function (event) {
   await queueRelevantRowAutomations(event, AutomationEventType.ROW_DELETE)
 })
 
+function rowPassesFilters(row: Row, filters: SearchFilters) {
+  const filteredRows = dataFilters.runQuery([row], filters)
+  return filteredRows.length > 0
+}
+
 export async function externalTrigger(
   automation: Automation,
   params: { fields: Record<string, any>; timeout?: number },
@@ -126,7 +142,23 @@ export async function externalTrigger(
     }
     params.fields = coercedFields
   }
-  const data: AutomationData = { automation, event: params as any }
+  const data: AutomationData = { automation, event: params }
+
+  const shouldTrigger = await checkTriggerFilters(automation, {
+    row: data.event?.row ?? {},
+    oldRow: data.event?.oldRow ?? {},
+  })
+
+  if (!shouldTrigger) {
+    return {
+      outputs: {
+        success: false,
+        status: AutomationStatus.STOPPED,
+      },
+      message: AutomationStoppedReason.TRIGGER_FILTER_NOT_MET,
+    }
+  }
+
   if (getResponses) {
     data.event = {
       ...data.event,
@@ -171,3 +203,25 @@ export async function rebootTrigger() {
     })
   }
 }
+
+async function checkTriggerFilters(
+  automation: Automation,
+  event: { row: Row; oldRow: Row }
+): Promise<boolean> {
+  const trigger = automation.definition.trigger
+  const filters = trigger?.inputs?.filters
+  const tableId = trigger?.inputs?.tableId
+
+  if (!filters) {
+    return true
+  }
+
+  if (
+    trigger.stepId === definitions.ROW_UPDATED.stepId ||
+    trigger.stepId === definitions.ROW_SAVED.stepId
+  ) {
+    const newRow = await automationUtils.cleanUpRow(tableId, event.row)
+    return rowPassesFilters(newRow, filters)
+  }
+  return true
+}

packages/server/src/tests/utilities/structures.ts

@@ -357,18 +357,23 @@ export function collectAutomation(tableId?: string): Automation {
       },
     },
   }
-  return automation as Automation
+  return automation
 }
 
-export function filterAutomation(tableId?: string): Automation {
+export function filterAutomation(appId: string, tableId?: string): Automation {
-  const automation: any = {
+  const automation: Automation = {
     name: "looping",
     type: "automation",
+    appId,
     definition: {
       steps: [
         {
+          name: "Filter Step",
+          tagline: "An automation filter step",
+          description: "A filter automation",
           id: "b",
-          type: "ACTION",
+          icon: "Icon",
+          type: AutomationStepType.ACTION,
           internal: true,
           stepId: AutomationActionStepId.FILTER,
           inputs: {},
@@ -376,8 +381,12 @@ export function filterAutomation(tableId?: string): Automation {
         },
       ],
       trigger: {
+        name: "trigger Step",
+        tagline: "An automation trigger",
+        description: "A trigger",
+        icon: "Icon",
         id: "a",
-        type: "TRIGGER",
+        type: AutomationStepType.TRIGGER,
         event: "row:save",
         stepId: AutomationTriggerStepId.ROW_SAVED,
         inputs: {
@@ -387,7 +396,45 @@ export function filterAutomation(tableId?: string): Automation {
       },
     },
   }
-  return automation as Automation
+  return automation
+}
+
+export function updateRowAutomationWithFilters(appId: string): Automation {
+  const automation: Automation = {
+    name: "updateRowWithFilters",
+    type: "automation",
+    appId,
+    definition: {
+      steps: [
+        {
+          name: "Filter Step",
+          tagline: "An automation filter step",
+          description: "A filter automation",
+          icon: "Icon",
+          id: "b",
+          type: AutomationStepType.ACTION,
+          internal: true,
+          stepId: AutomationActionStepId.SERVER_LOG,
+          inputs: {},
+          schema: BUILTIN_ACTION_DEFINITIONS.SERVER_LOG.schema,
+        },
+      ],
+      trigger: {
+        name: "trigger Step",
+        tagline: "An automation trigger",
+        description: "A trigger",
+        icon: "Icon",
+
+        id: "a",
+        type: AutomationStepType.TRIGGER,
+        event: "row:update",
+        stepId: AutomationTriggerStepId.ROW_UPDATED,
+        inputs: {},
+        schema: TRIGGER_DEFINITIONS.ROW_UPDATED.schema,
+      },
+    },
+  }
+  return automation
 }
 
 export function basicAutomationResults(

packages/types/src/documents/app/automation.ts

@@ -35,6 +35,7 @@ export enum AutomationCustomIOType {
   AUTOMATION = "automation",
   AUTOMATION_FIELDS = "automationFields",
   MULTI_ATTACHMENTS = "multi_attachments",
+  TRIGGER_FILTER = "trigger_filter",
 }
 
 export enum AutomationTriggerStepId {
@@ -128,6 +129,15 @@ export interface Automation extends Document {
   internal?: boolean
   type?: string
   disabled?: boolean
+  testData?: {
+    row?: Row
+    meta: {
+      [key: string]: unknown
+    }
+    id: string
+    revision: string
+    oldRow?: Row
+  }
 }
 
 interface BaseIOStructure {
@@ -201,6 +211,10 @@ export enum AutomationStatus {
   STOPPED_ERROR = "stopped_error",
 }
 
+export enum AutomationStoppedReason {
+  TRIGGER_FILTER_NOT_MET = "Automation did not run. Filter conditions in trigger were not met.",
+}
+
 export interface AutomationResults {
   automationId?: string
   status?: AutomationStatus

packages/types/src/sdk/automations/index.ts

@@ -1,4 +1,4 @@
-import { Automation, AutomationMetadata } from "../../documents"
+import { Automation, AutomationMetadata, Row } from "../../documents"
 import { Job } from "bull"
 
 export interface AutomationDataEvent {
@@ -6,6 +6,8 @@ export interface AutomationDataEvent {
   metadata?: AutomationMetadata
   automation?: Automation
   timeout?: number
+  row?: Row
+  oldRow?: Row
 }
 
 export interface AutomationData {