diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index e5f20882d3..6a00c125ad 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -266,9 +266,9 @@ export const getProdAppId = () => { return conversions.getProdAppID(appId) } -export function doInEnvironmentContext( +export function doInEnvironmentContext( values: Record, - task: any + task: () => T ) { if (!values) { throw new Error("Must supply environment variables.") diff --git a/packages/backend-core/src/utils/Duration.ts b/packages/backend-core/src/utils/Duration.ts index 730b59d1dc..f1cefa5a1f 100644 --- a/packages/backend-core/src/utils/Duration.ts +++ b/packages/backend-core/src/utils/Duration.ts @@ -15,23 +15,27 @@ const conversion: Record = { } export class Duration { + constructor(public ms: number) {} + + to(type: DurationType) { + return this.ms / conversion[type] + } + + toMs() { + return this.ms + } + + toSeconds() { + return this.to(DurationType.SECONDS) + } + static convert(from: DurationType, to: DurationType, duration: number) { const milliseconds = duration * conversion[from] return milliseconds / conversion[to] } static from(from: DurationType, duration: number) { - return { - to: (to: DurationType) => { - return Duration.convert(from, to, duration) - }, - toMs: () => { - return Duration.convert(from, DurationType.MILLISECONDS, duration) - }, - toSeconds: () => { - return Duration.convert(from, DurationType.SECONDS, duration) - }, - } + return new Duration(duration * conversion[from]) } static fromSeconds(duration: number) { diff --git a/packages/backend-core/src/utils/index.ts b/packages/backend-core/src/utils/index.ts index ac17227459..14bc4ca231 100644 --- a/packages/backend-core/src/utils/index.ts +++ b/packages/backend-core/src/utils/index.ts @@ -2,3 +2,4 @@ export * from "./hashing" export * from "./utils" export * from "./stringUtils" export * from "./Duration" +export * from "./time" diff --git a/packages/backend-core/src/utils/time.ts b/packages/backend-core/src/utils/time.ts new file mode 100644 index 0000000000..8ee40dd29f --- /dev/null +++ b/packages/backend-core/src/utils/time.ts @@ -0,0 +1,7 @@ +import { Duration } from "./Duration" + +export async function time(f: () => Promise): Promise<[T, Duration]> { + const start = performance.now() + const result = await f() + return [result, Duration.fromMilliseconds(performance.now() - start)] +} diff --git a/packages/builder/src/stores/builder/automations.ts b/packages/builder/src/stores/builder/automations.ts index ddb8706482..039a057a1b 100644 --- a/packages/builder/src/stores/builder/automations.ts +++ b/packages/builder/src/stores/builder/automations.ts @@ -15,7 +15,6 @@ import { import { AutomationTriggerStepId, AutomationEventType, - AutomationStepType, AutomationActionStepId, Automation, AutomationStep, @@ -26,10 +25,14 @@ import { UILogicalOperator, EmptyFilterOption, AutomationIOType, - AutomationStepSchema, - AutomationTriggerSchema, BranchPath, BlockDefinitions, + isBranchStep, + isTrigger, + isRowUpdateTrigger, + isRowSaveTrigger, + isAppTrigger, + BranchStep, } from "@budibase/types" import { ActionStepID } from "@/constants/backend/automations" import { FIELDS } from "@/constants/backend" @@ -291,16 +294,16 @@ const automationActions = (store: AutomationStore) => ({ let result: (AutomationStep | AutomationTrigger)[] = [] pathWay.forEach(path => { const { stepIdx, branchIdx } = path - let last = result.length ? result[result.length - 1] : [] if (!result.length) { // Preceeding steps. result = steps.slice(0, stepIdx + 1) return } - if (last && "inputs" in last) { + let last = result[result.length - 1] + if (isBranchStep(last)) { if (Number.isInteger(branchIdx)) { const branchId = last.inputs.branches[branchIdx].id - const children = last.inputs.children[branchId] + const children = last.inputs.children?.[branchId] || [] const stepChildren = children.slice(0, stepIdx + 1) // Preceeding steps. result = result.concat(stepChildren) @@ -473,23 +476,28 @@ const automationActions = (store: AutomationStore) => ({ id: block.id, }, ] - const branches: Branch[] = block.inputs?.branches || [] - branches.forEach((branch, bIdx) => { - block.inputs?.children[branch.id].forEach( - (bBlock: AutomationStep, sIdx: number, array: AutomationStep[]) => { - const ended = - array.length - 1 === sIdx && !bBlock.inputs?.branches?.length - treeTraverse(bBlock, pathToCurrentNode, sIdx, bIdx, ended) - } - ) - }) + if (isBranchStep(block)) { + const branches = block.inputs?.branches || [] + const children = block.inputs?.children || {} + + branches.forEach((branch, bIdx) => { + children[branch.id].forEach( + (bBlock: AutomationStep, sIdx: number, array: AutomationStep[]) => { + const ended = array.length - 1 === sIdx && !branches.length + treeTraverse(bBlock, pathToCurrentNode, sIdx, bIdx, ended) + } + ) + }) + + terminating = terminating && !branches.length + } store.actions.registerBlock( blockRefs, block, pathToCurrentNode, - terminating && !branches.length + terminating ) } @@ -575,7 +583,6 @@ const automationActions = (store: AutomationStore) => ({ pathBlock.stepId === ActionStepID.LOOP && pathBlock.blockToLoop in blocks } - const isTrigger = pathBlock.type === AutomationStepType.TRIGGER if (isLoopBlock && loopBlockCount == 0) { schema = { @@ -586,17 +593,14 @@ const automationActions = (store: AutomationStore) => ({ } } - const icon = isTrigger + const icon = isTrigger(pathBlock) ? pathBlock.icon : isLoopBlock ? "Reuse" : pathBlock.icon - if (blockIdx === 0 && isTrigger) { - if ( - pathBlock.event === AutomationEventType.ROW_UPDATE || - pathBlock.event === AutomationEventType.ROW_SAVE - ) { + if (blockIdx === 0 && isTrigger(pathBlock)) { + if (isRowUpdateTrigger(pathBlock) || isRowSaveTrigger(pathBlock)) { let table: any = get(tables).list.find( (table: Table) => table._id === pathBlock.inputs.tableId ) @@ -608,7 +612,7 @@ const automationActions = (store: AutomationStore) => ({ } } delete schema.row - } else if (pathBlock.event === AutomationEventType.APP_TRIGGER) { + } else if (isAppTrigger(pathBlock)) { schema = Object.fromEntries( Object.keys(pathBlock.inputs.fields || []).map(key => [ key, @@ -915,8 +919,10 @@ const automationActions = (store: AutomationStore) => ({ ] let cache: - | AutomationStepSchema - | AutomationTriggerSchema + | AutomationStep + | AutomationTrigger + | AutomationStep[] + | undefined = undefined pathWay.forEach((path, pathIdx, array) => { const { stepIdx, branchIdx } = path @@ -938,9 +944,13 @@ const automationActions = (store: AutomationStore) => ({ } return } - if (Number.isInteger(branchIdx)) { + if ( + Number.isInteger(branchIdx) && + !Array.isArray(cache) && + isBranchStep(cache) + ) { const branchId = cache.inputs.branches[branchIdx].id - const children = cache.inputs.children[branchId] + const children = cache.inputs.children?.[branchId] || [] if (final) { insertBlock(children, stepIdx) @@ -1090,7 +1100,7 @@ const automationActions = (store: AutomationStore) => ({ branchLeft: async ( pathTo: Array, automation: Automation, - block: AutomationStep + block: BranchStep ) => { const update = store.actions.shiftBranch(pathTo, block) if (update) { @@ -1113,7 +1123,7 @@ const automationActions = (store: AutomationStore) => ({ branchRight: async ( pathTo: Array, automation: Automation, - block: AutomationStep + block: BranchStep ) => { const update = store.actions.shiftBranch(pathTo, block, 1) if (update) { @@ -1133,7 +1143,7 @@ const automationActions = (store: AutomationStore) => ({ * @param {Number} direction - the direction of the swap. Defaults to -1 for left, add 1 for right * @returns */ - shiftBranch: (pathTo: Array, block: AutomationStep, direction = -1) => { + shiftBranch: (pathTo: Array, block: BranchStep, direction = -1) => { let newBlock = cloneDeep(block) const branchPath = pathTo.at(-1) const targetIdx = branchPath.branchIdx diff --git a/packages/pro b/packages/pro index eb96d8b2f2..45f5673d5e 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit eb96d8b2f2029033b0f758078ed30c888e8fb249 +Subproject commit 45f5673d5e5ab3c22deb6663cea2e31a628aa133 diff --git a/packages/server/src/api/routes/tests/automation.spec.ts b/packages/server/src/api/routes/tests/automation.spec.ts index 1d7b9cd6ed..1591412735 100644 --- a/packages/server/src/api/routes/tests/automation.spec.ts +++ b/packages/server/src/api/routes/tests/automation.spec.ts @@ -13,6 +13,7 @@ import sdk from "../../../sdk" import { ConfigType, FieldType, + FilterCondition, isDidNotTriggerResponse, SettingsConfig, Table, @@ -20,12 +21,9 @@ import { import { mocks } from "@budibase/backend-core/tests" import { removeDeprecated } from "../../../automations/utils" import { createAutomationBuilder } from "../../../automations/tests/utilities/AutomationTestBuilder" -import { automations } from "@budibase/shared-core" import { basicTable } from "../../../tests/utilities/structures" import TestConfiguration from "../../../tests/utilities/TestConfiguration" -const FilterConditions = automations.steps.filter.FilterConditions - const MAX_RETRIES = 4 const { basicAutomation, @@ -487,15 +485,40 @@ describe("/automations", () => { expect(events.automation.created).not.toHaveBeenCalled() expect(events.automation.triggerUpdated).not.toHaveBeenCalled() }) + + it("can update an input field", async () => { + const { automation } = await createAutomationBuilder(config) + .onRowDeleted({ tableId: "tableId" }) + .serverLog({ text: "test" }) + .save() + + automation.definition.trigger.inputs.tableId = "newTableId" + const { automation: updatedAutomation } = + await config.api.automation.update(automation) + + expect(updatedAutomation.definition.trigger.inputs.tableId).toEqual( + "newTableId" + ) + }) + + it("cannot update a readonly field", async () => { + const { automation } = await createAutomationBuilder(config) + .onRowAction({ tableId: "tableId" }) + .serverLog({ text: "test" }) + .save() + + automation.definition.trigger.inputs.tableId = "newTableId" + await config.api.automation.update(automation, { + status: 400, + body: { + message: "Field tableId is readonly and it cannot be modified", + }, + }) + }) }) describe("fetch", () => { it("return all the automations for an instance", async () => { - const fetchResponse = await config.api.automation.fetch() - for (const auto of fetchResponse.automations) { - await config.api.automation.delete(auto) - } - const { automation: automation1 } = await config.api.automation.post( newAutomation() ) @@ -594,7 +617,7 @@ describe("/automations", () => { steps: [ { inputs: { - condition: FilterConditions.EQUAL, + condition: FilterCondition.EQUAL, field: "{{ trigger.row.City }}", value: "{{ trigger.oldRow.City }}", }, diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index 65a57c2586..89a940ae7b 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -27,6 +27,8 @@ import { Hosting, ActionImplementation, AutomationStepDefinition, + AutomationStepInputs, + AutomationStepOutputs, } from "@budibase/types" import sdk from "../sdk" import { getAutomationPlugin } from "../utilities/fileSystem" @@ -120,11 +122,15 @@ export async function getActionDefinitions(): Promise< } /* istanbul ignore next */ -export async function getAction( - stepId: AutomationActionStepId -): Promise | undefined> { +export async function getAction< + TStep extends AutomationActionStepId, + TInputs = AutomationStepInputs, + TOutputs = AutomationStepOutputs +>(stepId: TStep): Promise | undefined> { if (ACTION_IMPLS[stepId as keyof ActionImplType] != null) { - return ACTION_IMPLS[stepId as keyof ActionImplType] + return ACTION_IMPLS[ + stepId as keyof ActionImplType + ] as unknown as ActionImplementation } // must be a plugin diff --git a/packages/server/src/automations/automationUtils.ts b/packages/server/src/automations/automationUtils.ts index eacf81ef92..4c03f0f994 100644 --- a/packages/server/src/automations/automationUtils.ts +++ b/packages/server/src/automations/automationUtils.ts @@ -6,10 +6,10 @@ import { import sdk from "../sdk" import { AutomationAttachment, + BaseIOStructure, + FieldSchema, FieldType, Row, - LoopStepType, - LoopStepInputs, } from "@budibase/types" import { objectStore, context } from "@budibase/backend-core" import * as uuid from "uuid" @@ -32,33 +32,34 @@ import path from "path" * primitive types. */ export function cleanInputValues>( - inputs: any, - schema?: any + inputs: T, + schema?: Partial> ): T { - if (schema == null) { - return inputs - } - for (let inputKey of Object.keys(inputs)) { + const keys = Object.keys(inputs) as (keyof T)[] + for (let inputKey of keys) { let input = inputs[inputKey] if (typeof input !== "string") { continue } - let propSchema = schema.properties[inputKey] + let propSchema = schema?.[inputKey] if (!propSchema) { continue } if (propSchema.type === "boolean") { let lcInput = input.toLowerCase() if (lcInput === "true") { + // @ts-expect-error - indexing a generic on purpose inputs[inputKey] = true } if (lcInput === "false") { + // @ts-expect-error - indexing a generic on purpose inputs[inputKey] = false } } if (propSchema.type === "number") { let floatInput = parseFloat(input) if (!isNaN(floatInput)) { + // @ts-expect-error - indexing a generic on purpose inputs[inputKey] = floatInput } } @@ -93,7 +94,7 @@ export function cleanInputValues>( */ export async function cleanUpRow(tableId: string, row: Row) { let table = await sdk.tables.getTable(tableId) - return cleanInputValues(row, { properties: table.schema }) + return cleanInputValues(row, table.schema) } export function getError(err: any) { @@ -271,36 +272,3 @@ export function stringSplit(value: string | string[]) { } return value.split(",") } - -export function typecastForLooping(input: LoopStepInputs) { - if (!input || !input.binding) { - return null - } - try { - switch (input.option) { - case LoopStepType.ARRAY: - if (typeof input.binding === "string") { - return JSON.parse(input.binding) - } - break - case LoopStepType.STRING: - if (Array.isArray(input.binding)) { - return input.binding.join(",") - } - break - } - } catch (err) { - throw new Error("Unable to cast to correct type") - } - return input.binding -} - -export function ensureMaxIterationsAsNumber( - value: number | string | undefined -): number | undefined { - if (typeof value === "number") return value - if (typeof value === "string") { - return parseInt(value) - } - return undefined -} diff --git a/packages/server/src/automations/loopUtils.ts b/packages/server/src/automations/loopUtils.ts deleted file mode 100644 index 2596fb796d..0000000000 --- a/packages/server/src/automations/loopUtils.ts +++ /dev/null @@ -1,46 +0,0 @@ -import * as automationUtils from "./automationUtils" -import { isPlainObject } from "lodash" - -type ObjValue = { - [key: string]: string | ObjValue -} - -export function replaceFakeBindings>( - originalStepInput: T, - loopStepNumber: number -): T { - const result: Record = {} - for (const [key, value] of Object.entries(originalStepInput)) { - result[key] = replaceBindingsRecursive(value, loopStepNumber) - } - return result as T -} - -function replaceBindingsRecursive( - value: string | ObjValue, - loopStepNumber: number -) { - if (value === null || value === undefined) { - return value - } - - if (typeof value === "object") { - for (const [innerKey, innerValue] of Object.entries(value)) { - if (typeof innerValue === "string") { - value[innerKey] = automationUtils.substituteLoopStep( - innerValue, - `steps.${loopStepNumber}` - ) - } else if ( - innerValue && - isPlainObject(innerValue) && - Object.keys(innerValue).length > 0 - ) { - value[innerKey] = replaceBindingsRecursive(innerValue, loopStepNumber) - } - } - } else if (typeof value === "string") { - value = automationUtils.substituteLoopStep(value, `steps.${loopStepNumber}`) - } - return value -} diff --git a/packages/server/src/automations/steps/filter.ts b/packages/server/src/automations/steps/filter.ts index 9b7e347034..92a76692cc 100644 --- a/packages/server/src/automations/steps/filter.ts +++ b/packages/server/src/automations/steps/filter.ts @@ -1,7 +1,8 @@ -import { FilterStepInputs, FilterStepOutputs } from "@budibase/types" -import { automations } from "@budibase/shared-core" - -const FilterConditions = automations.steps.filter.FilterConditions +import { + FilterCondition, + FilterStepInputs, + FilterStepOutputs, +} from "@budibase/types" export async function run({ inputs, @@ -26,16 +27,16 @@ export async function run({ let result = false if (typeof field !== "object" && typeof value !== "object") { switch (condition) { - case FilterConditions.EQUAL: + case FilterCondition.EQUAL: result = field === value break - case FilterConditions.NOT_EQUAL: + case FilterCondition.NOT_EQUAL: result = field !== value break - case FilterConditions.GREATER_THAN: + case FilterCondition.GREATER_THAN: result = field > value break - case FilterConditions.LESS_THAN: + case FilterCondition.LESS_THAN: result = field < value break } diff --git a/packages/server/src/automations/tests/automationUtils.spec.ts b/packages/server/src/automations/tests/automationUtils.spec.ts index 456feb6e7a..05dd7483e9 100644 --- a/packages/server/src/automations/tests/automationUtils.spec.ts +++ b/packages/server/src/automations/tests/automationUtils.spec.ts @@ -1,9 +1,5 @@ -import { - typecastForLooping, - cleanInputValues, - substituteLoopStep, -} from "../automationUtils" -import { LoopStepType } from "@budibase/types" +import { AutomationIOType } from "@budibase/types" +import { cleanInputValues, substituteLoopStep } from "../automationUtils" describe("automationUtils", () => { describe("substituteLoopStep", () => { @@ -30,29 +26,6 @@ describe("automationUtils", () => { }) }) - describe("typeCastForLooping", () => { - it("should parse to correct type", () => { - expect( - typecastForLooping({ option: LoopStepType.ARRAY, binding: [1, 2, 3] }) - ).toEqual([1, 2, 3]) - expect( - typecastForLooping({ option: LoopStepType.ARRAY, binding: "[1,2,3]" }) - ).toEqual([1, 2, 3]) - expect( - typecastForLooping({ option: LoopStepType.STRING, binding: [1, 2, 3] }) - ).toEqual("1,2,3") - }) - it("should handle null values", () => { - // expect it to handle where the binding is null - expect( - typecastForLooping({ option: LoopStepType.ARRAY, binding: null }) - ).toEqual(null) - expect(() => - typecastForLooping({ option: LoopStepType.ARRAY, binding: "test" }) - ).toThrow() - }) - }) - describe("cleanInputValues", () => { it("should handle array relationship fields from read binding", () => { const schema = { @@ -70,15 +43,12 @@ describe("automationUtils", () => { }, } expect( - cleanInputValues( - { - row: { - relationship: `[{"_id": "ro_ta_users_us_3"}]`, - }, - schema, + cleanInputValues({ + row: { + relationship: `[{"_id": "ro_ta_users_us_3"}]`, }, - schema - ) + schema, + }) ).toEqual({ row: { relationship: [{ _id: "ro_ta_users_us_3" }], @@ -103,15 +73,12 @@ describe("automationUtils", () => { }, } expect( - cleanInputValues( - { - row: { - relationship: `ro_ta_users_us_3`, - }, - schema, + cleanInputValues({ + row: { + relationship: `ro_ta_users_us_3`, }, - schema - ) + schema, + }) ).toEqual({ row: { relationship: "ro_ta_users_us_3", @@ -122,28 +89,27 @@ describe("automationUtils", () => { it("should be able to clean inputs with the utilities", () => { // can't clean without a schema - let output = cleanInputValues({ a: "1" }) - expect(output.a).toBe("1") - output = cleanInputValues( + const one = cleanInputValues({ a: "1" }) + expect(one.a).toBe("1") + + const two = cleanInputValues( { a: "1", b: "true", c: "false", d: 1, e: "help" }, { - properties: { - a: { - type: "number", - }, - b: { - type: "boolean", - }, - c: { - type: "boolean", - }, + a: { + type: AutomationIOType.NUMBER, + }, + b: { + type: AutomationIOType.BOOLEAN, + }, + c: { + type: AutomationIOType.BOOLEAN, }, } ) - expect(output.a).toBe(1) - expect(output.b).toBe(true) - expect(output.c).toBe(false) - expect(output.d).toBe(1) + expect(two.a).toBe(1) + expect(two.b).toBe(true) + expect(two.c).toBe(false) + expect(two.d).toBe(1) }) }) }) diff --git a/packages/server/src/automations/tests/scenarios.spec.ts b/packages/server/src/automations/tests/scenarios.spec.ts index 3015e75018..91934a9e22 100644 --- a/packages/server/src/automations/tests/scenarios.spec.ts +++ b/packages/server/src/automations/tests/scenarios.spec.ts @@ -1,5 +1,11 @@ import * as automation from "../index" -import { LoopStepType, FieldType, Table, Datasource } from "@budibase/types" +import { + LoopStepType, + FieldType, + Table, + Datasource, + FilterCondition, +} from "@budibase/types" import { createAutomationBuilder } from "./utilities/AutomationTestBuilder" import { DatabaseName, @@ -7,12 +13,9 @@ import { } from "../../integrations/tests/utils" import { Knex } from "knex" import { generator } from "@budibase/backend-core/tests" -import { automations } from "@budibase/shared-core" import TestConfiguration from "../../tests/utilities/TestConfiguration" import { basicTable } from "../../tests/utilities/structures" -const FilterConditions = automations.steps.filter.FilterConditions - describe("Automation Scenarios", () => { const config = new TestConfiguration() @@ -256,7 +259,7 @@ describe("Automation Scenarios", () => { }) .filter({ field: "{{ steps.2.rows.0.value }}", - condition: FilterConditions.EQUAL, + condition: FilterCondition.EQUAL, value: 20, }) .serverLog({ text: "Equal condition met" }) @@ -282,7 +285,7 @@ describe("Automation Scenarios", () => { }) .filter({ field: "{{ steps.2.rows.0.value }}", - condition: FilterConditions.NOT_EQUAL, + condition: FilterCondition.NOT_EQUAL, value: 20, }) .serverLog({ text: "Not Equal condition met" }) @@ -295,37 +298,37 @@ describe("Automation Scenarios", () => { const testCases = [ { - condition: FilterConditions.EQUAL, + condition: FilterCondition.EQUAL, value: 10, rowValue: 10, expectPass: true, }, { - condition: FilterConditions.NOT_EQUAL, + condition: FilterCondition.NOT_EQUAL, value: 10, rowValue: 20, expectPass: true, }, { - condition: FilterConditions.GREATER_THAN, + condition: FilterCondition.GREATER_THAN, value: 10, rowValue: 15, expectPass: true, }, { - condition: FilterConditions.LESS_THAN, + condition: FilterCondition.LESS_THAN, value: 10, rowValue: 5, expectPass: true, }, { - condition: FilterConditions.GREATER_THAN, + condition: FilterCondition.GREATER_THAN, value: 10, rowValue: 5, expectPass: false, }, { - condition: FilterConditions.LESS_THAN, + condition: FilterCondition.LESS_THAN, value: 10, rowValue: 15, expectPass: false, diff --git a/packages/server/src/automations/tests/steps/createRow.spec.ts b/packages/server/src/automations/tests/steps/createRow.spec.ts index 0a3913cd25..01ce227f36 100644 --- a/packages/server/src/automations/tests/steps/createRow.spec.ts +++ b/packages/server/src/automations/tests/steps/createRow.spec.ts @@ -4,7 +4,7 @@ import { } from "../../../tests/utilities/structures" import { objectStore } from "@budibase/backend-core" import { createAutomationBuilder } from "../utilities/AutomationTestBuilder" -import { Row, Table } from "@budibase/types" +import { FilterCondition, Row, Table } from "@budibase/types" import TestConfiguration from "../../../tests/utilities/TestConfiguration" async function uploadTestFile(filename: string) { @@ -90,7 +90,7 @@ describe("test the create row action", () => { .createRow({ row: {} }, { stepName: "CreateRow" }) .filter({ field: "{{ stepsByName.CreateRow.success }}", - condition: "equal", + condition: FilterCondition.EQUAL, value: true, }) .serverLog( @@ -131,7 +131,7 @@ describe("test the create row action", () => { .createRow({ row: attachmentRow }, { stepName: "CreateRow" }) .filter({ field: "{{ stepsByName.CreateRow.success }}", - condition: "equal", + condition: FilterCondition.EQUAL, value: true, }) .serverLog( diff --git a/packages/server/src/automations/tests/steps/filter.spec.ts b/packages/server/src/automations/tests/steps/filter.spec.ts index 23c191b38d..da1f6e4702 100644 --- a/packages/server/src/automations/tests/steps/filter.spec.ts +++ b/packages/server/src/automations/tests/steps/filter.spec.ts @@ -1,19 +1,21 @@ -import { automations } from "@budibase/shared-core" import { createAutomationBuilder } from "../utilities/AutomationTestBuilder" import TestConfiguration from "../../../tests/utilities/TestConfiguration" +import { FilterCondition } from "@budibase/types" -const FilterConditions = automations.steps.filter.FilterConditions - -function stringToFilterCondition(condition: "==" | "!=" | ">" | "<"): string { +function stringToFilterCondition( + condition: "==" | "!=" | ">" | "<" +): FilterCondition { switch (condition) { case "==": - return FilterConditions.EQUAL + return FilterCondition.EQUAL case "!=": - return FilterConditions.NOT_EQUAL + return FilterCondition.NOT_EQUAL case ">": - return FilterConditions.GREATER_THAN + return FilterCondition.GREATER_THAN case "<": - return FilterConditions.LESS_THAN + return FilterCondition.LESS_THAN + default: + throw new Error(`Unsupported condition: ${condition}`) } } diff --git a/packages/server/src/automations/tests/steps/loop.spec.ts b/packages/server/src/automations/tests/steps/loop.spec.ts index 88e641f5ff..883732330f 100644 --- a/packages/server/src/automations/tests/steps/loop.spec.ts +++ b/packages/server/src/automations/tests/steps/loop.spec.ts @@ -6,8 +6,8 @@ import { ServerLogStepOutputs, CreateRowStepOutputs, FieldType, + FilterCondition, } from "@budibase/types" -import * as loopUtils from "../../loopUtils" import { createAutomationBuilder } from "../utilities/AutomationTestBuilder" import TestConfiguration from "../../../tests/utilities/TestConfiguration" @@ -30,8 +30,8 @@ describe("Attempt to run a basic loop automation", () => { await config.api.row.save(table._id!, {}) }) - afterAll(() => { - automation.shutdown() + afterAll(async () => { + await automation.shutdown() config.end() }) @@ -535,97 +535,30 @@ describe("Attempt to run a basic loop automation", () => { expect(results.steps[2].outputs.rows).toHaveLength(0) }) - describe("replaceFakeBindings", () => { - it("should replace loop bindings in nested objects", () => { - const originalStepInput = { - schema: { - name: { - type: "string", - constraints: { - type: "string", - length: { maximum: null }, - presence: false, - }, - name: "name", - display: { type: "Text" }, - }, - }, - row: { - tableId: "ta_aaad4296e9f74b12b1b90ef7a84afcad", - name: "{{ loop.currentItem.pokemon }}", - }, - } + describe("loop output", () => { + it("should not output anything if a filter stops the automation", async () => { + const results = await createAutomationBuilder(config) + .onAppAction() + .filter({ + condition: FilterCondition.EQUAL, + field: "1", + value: "2", + }) + .loop({ + option: LoopStepType.ARRAY, + binding: [1, 2, 3], + }) + .serverLog({ text: "Message {{loop.currentItem}}" }) + .test({ fields: {} }) - const loopStepNumber = 3 - - const result = loopUtils.replaceFakeBindings( - originalStepInput, - loopStepNumber - ) - - expect(result).toEqual({ - schema: { - name: { - type: "string", - constraints: { - type: "string", - length: { maximum: null }, - presence: false, - }, - name: "name", - display: { type: "Text" }, - }, - }, - row: { - tableId: "ta_aaad4296e9f74b12b1b90ef7a84afcad", - name: "{{ steps.3.currentItem.pokemon }}", - }, + expect(results.steps.length).toBe(1) + expect(results.steps[0].outputs).toEqual({ + comparisonValue: 2, + refValue: 1, + result: false, + success: true, + status: "stopped", }) }) - - it("should handle null values in nested objects", () => { - const originalStepInput = { - nullValue: null, - nestedNull: { - someKey: null, - }, - validValue: "{{ loop.someValue }}", - } - - const loopStepNumber = 2 - - const result = loopUtils.replaceFakeBindings( - originalStepInput, - loopStepNumber - ) - - expect(result).toEqual({ - nullValue: null, - nestedNull: { - someKey: null, - }, - validValue: "{{ steps.2.someValue }}", - }) - }) - - it("should handle empty objects and arrays", () => { - const originalStepInput = { - emptyObject: {}, - emptyArray: [], - nestedEmpty: { - emptyObj: {}, - emptyArr: [], - }, - } - - const loopStepNumber = 1 - - const result = loopUtils.replaceFakeBindings( - originalStepInput, - loopStepNumber - ) - - expect(result).toEqual(originalStepInput) - }) }) }) diff --git a/packages/server/src/automations/tests/triggers/cron.spec.ts b/packages/server/src/automations/tests/triggers/cron.spec.ts index 90d29a60c1..8db9cb425e 100644 --- a/packages/server/src/automations/tests/triggers/cron.spec.ts +++ b/packages/server/src/automations/tests/triggers/cron.spec.ts @@ -66,7 +66,7 @@ describe("cron trigger", () => { }) }) - it("should stop if the job fails more than 3 times", async () => { + it("should stop if the job fails more than N times", async () => { const { automation } = await createAutomationBuilder(config) .onCron({ cron: "* * * * *" }) .queryRows({ diff --git a/packages/server/src/automations/tests/triggers/webhook.spec.ts b/packages/server/src/automations/tests/triggers/webhook.spec.ts index 664812f860..9649846830 100644 --- a/packages/server/src/automations/tests/triggers/webhook.spec.ts +++ b/packages/server/src/automations/tests/triggers/webhook.spec.ts @@ -5,7 +5,7 @@ import TestConfiguration from "../../../tests/utilities/TestConfiguration" mocks.licenses.useSyncAutomations() -describe("Branching automations", () => { +describe("Webhook trigger test", () => { const config = new TestConfiguration() let table: Table let webhook: Webhook diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index a95e58f69d..9e9fe38ca5 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -63,6 +63,7 @@ class TriggerBuilder { onRowDeleted = this.trigger(AutomationTriggerStepId.ROW_DELETED) onWebhook = this.trigger(AutomationTriggerStepId.WEBHOOK) onCron = this.trigger(AutomationTriggerStepId.CRON) + onRowAction = this.trigger(AutomationTriggerStepId.ROW_ACTION) } class BranchStepBuilder { diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index 2ac90f3f9c..16d5246a91 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -182,11 +182,12 @@ export async function externalTrigger( // values are likely to be submitted as strings, so we shall convert to correct type const coercedFields: any = {} const fields = automation.definition.trigger.inputs.fields - for (let key of Object.keys(fields || {})) { + for (const key of Object.keys(fields || {})) { coercedFields[key] = coerce(params.fields[key], fields[key]) } params.fields = coercedFields } + // row actions and webhooks flatten the fields down else if ( sdk.automations.isRowAction(automation) || @@ -198,6 +199,7 @@ export async function externalTrigger( fields: {}, } } + const data: AutomationData = { automation, event: params } const shouldTrigger = await checkTriggerFilters(automation, { diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index 9af166bcf9..3ec8f41621 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -18,6 +18,7 @@ import { import { automationsEnabled } from "../features" import { helpers, REBOOT_CRON } from "@budibase/shared-core" import tracer from "dd-trace" +import { JobId } from "bull" const CRON_STEP_ID = automations.triggers.definitions.CRON.stepId let Runner: Thread @@ -155,11 +156,11 @@ export async function disableAllCrons(appId: any) { return { count: results.length / 2 } } -export async function disableCronById(jobId: number | string) { - const repeatJobs = await automationQueue.getRepeatableJobs() - for (let repeatJob of repeatJobs) { - if (repeatJob.id === jobId) { - await automationQueue.removeRepeatableByKey(repeatJob.key) +export async function disableCronById(jobId: JobId) { + const jobs = await automationQueue.getRepeatableJobs() + for (const job of jobs) { + if (job.id === jobId) { + await automationQueue.removeRepeatableByKey(job.key) } } console.log(`jobId=${jobId} disabled`) @@ -248,33 +249,3 @@ export async function enableCronTrigger(appId: any, automation: Automation) { export async function cleanupAutomations(appId: any) { await disableAllCrons(appId) } - -/** - * Checks if the supplied automation is of a recurring type. - * @param automation The automation to check. - * @return if it is recurring (cron). - */ -export function isRecurring(automation: Automation) { - return ( - automation.definition.trigger.stepId === - automations.triggers.definitions.CRON.stepId - ) -} - -export function isErrorInOutput(output: { - steps: { outputs?: { success: boolean } }[] -}) { - let first = true, - error = false - for (let step of output.steps) { - // skip the trigger, its always successful if automation ran - if (first) { - first = false - continue - } - if (!step.outputs?.success) { - error = true - } - } - return error -} diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts index fde1efd1b9..d511365dca 100644 --- a/packages/server/src/constants/index.ts +++ b/packages/server/src/constants/index.ts @@ -130,11 +130,6 @@ export enum InvalidColumns { TABLE_ID = "tableId", } -export enum AutomationErrors { - INCORRECT_TYPE = "INCORRECT_TYPE", - FAILURE_CONDITION = "FAILURE_CONDITION_MET", -} - // pass through the list from the auth/core lib export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets export const MAX_AUTOMATION_RECURRING_ERRORS = 5 diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts index 67d6e04e9d..a04b960ca5 100644 --- a/packages/server/src/definitions/automations.ts +++ b/packages/server/src/definitions/automations.ts @@ -1,4 +1,9 @@ -import { AutomationResults, LoopStepType, UserBindings } from "@budibase/types" +import { + AutomationStepResultOutputs, + AutomationTriggerResultOutputs, + LoopStepType, + UserBindings, +} from "@budibase/types" export interface LoopInput { option: LoopStepType @@ -13,19 +18,17 @@ export interface TriggerOutput { timestamp?: number } -export interface AutomationContext extends AutomationResults { - steps: any[] - stepsById: Record - stepsByName: Record +export interface AutomationContext { + trigger: AutomationTriggerResultOutputs + steps: [AutomationTriggerResultOutputs, ...AutomationStepResultOutputs[]] + stepsById: Record + stepsByName: Record env?: Record user?: UserBindings - trigger: any settings?: { url?: string logo?: string company?: string } + loop?: { currentItem: any } } - -export interface AutomationResponse - extends Omit {} diff --git a/packages/server/src/sdk/app/automations/crud.ts b/packages/server/src/sdk/app/automations/crud.ts index cd8af1e548..764b1df784 100644 --- a/packages/server/src/sdk/app/automations/crud.ts +++ b/packages/server/src/sdk/app/automations/crud.ts @@ -40,7 +40,8 @@ function cleanAutomationInputs(automation: Automation) { if (step == null) { continue } - for (let inputName of Object.keys(step.inputs)) { + for (const key of Object.keys(step.inputs)) { + const inputName = key as keyof typeof step.inputs if (!step.inputs[inputName] || step.inputs[inputName] === "") { delete step.inputs[inputName] } @@ -281,7 +282,8 @@ function guardInvalidUpdatesAndThrow( const readonlyFields = Object.keys( step.schema.inputs.properties || {} ).filter(k => step.schema.inputs.properties[k].readonly) - readonlyFields.forEach(readonlyField => { + readonlyFields.forEach(key => { + const readonlyField = key as keyof typeof step.inputs const oldStep = oldStepDefinitions.find(i => i.id === step.id) if (step.inputs[readonlyField] !== oldStep?.inputs[readonlyField]) { throw new HTTPError( diff --git a/packages/server/src/sdk/app/automations/tests/index.spec.ts b/packages/server/src/sdk/app/automations/tests/index.spec.ts deleted file mode 100644 index 6c70392300..0000000000 --- a/packages/server/src/sdk/app/automations/tests/index.spec.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { sample } from "lodash/fp" -import { Automation } from "@budibase/types" -import { generator } from "@budibase/backend-core/tests" -import TestConfiguration from "../../../../tests/utilities/TestConfiguration" -import automationSdk from "../" -import { structures } from "../../../../api/routes/tests/utilities" - -describe("automation sdk", () => { - const config = new TestConfiguration() - - beforeAll(async () => { - await config.init() - }) - - describe("update", () => { - it("can rename existing automations", async () => { - await config.doInContext(config.getAppId(), async () => { - const automation = structures.newAutomation() - - const response = await automationSdk.create(automation) - - const newName = generator.guid() - const update = { ...response, name: newName } - const result = await automationSdk.update(update) - expect(result.name).toEqual(newName) - }) - }) - - it.each([ - ["trigger", (a: Automation) => a.definition.trigger], - ["step", (a: Automation) => a.definition.steps[0]], - ])("can update input fields (for a %s)", async (_, getStep) => { - await config.doInContext(config.getAppId(), async () => { - const automation = structures.newAutomation() - - const keyToUse = sample(Object.keys(getStep(automation).inputs))! - getStep(automation).inputs[keyToUse] = "anyValue" - - const response = await automationSdk.create(automation) - - const update = { ...response } - getStep(update).inputs[keyToUse] = "anyUpdatedValue" - const result = await automationSdk.update(update) - expect(getStep(result).inputs[keyToUse]).toEqual("anyUpdatedValue") - }) - }) - - it.each([ - ["trigger", (a: Automation) => a.definition.trigger], - ["step", (a: Automation) => a.definition.steps[0]], - ])("cannot update readonly fields (for a %s)", async (_, getStep) => { - await config.doInContext(config.getAppId(), async () => { - const automation = structures.newAutomation() - getStep(automation).schema.inputs.properties["readonlyProperty"] = { - readonly: true, - } - getStep(automation).inputs["readonlyProperty"] = "anyValue" - - const response = await automationSdk.create(automation) - - const update = { ...response } - getStep(update).inputs["readonlyProperty"] = "anyUpdatedValue" - await expect(automationSdk.update(update)).rejects.toThrow( - "Field readonlyProperty is readonly and it cannot be modified" - ) - }) - }) - }) -}) diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts index a78a2c6c9e..38d60e1c11 100644 --- a/packages/server/src/tests/utilities/structures.ts +++ b/packages/server/src/tests/utilities/structures.ts @@ -35,6 +35,8 @@ import { WebhookActionType, BuiltinPermissionID, DeepPartial, + FilterCondition, + AutomationTriggerResult, } from "@budibase/types" import { LoopInput } from "../../definitions/automations" import { merge } from "lodash" @@ -372,7 +374,11 @@ export function filterAutomation(opts?: DeepPartial): Automation { type: AutomationStepType.ACTION, internal: true, stepId: AutomationActionStepId.FILTER, - inputs: { field: "name", value: "test", condition: "EQ" }, + inputs: { + field: "name", + value: "test", + condition: FilterCondition.EQUAL, + }, schema: BUILTIN_ACTION_DEFINITIONS.EXECUTE_SCRIPT.schema, }, ], @@ -437,15 +443,24 @@ export function updateRowAutomationWithFilters( export function basicAutomationResults( automationId: string ): AutomationResults { + const trigger: AutomationTriggerResult = { + id: "trigger", + stepId: AutomationTriggerStepId.APP, + outputs: {}, + } return { automationId, status: AutomationStatus.SUCCESS, - trigger: "trigger" as any, + trigger, steps: [ + trigger, { + id: "step1", stepId: AutomationActionStepId.SERVER_LOG, inputs: {}, - outputs: {}, + outputs: { + success: true, + }, }, ], } diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index f854635559..6ee467023f 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -1,18 +1,13 @@ import { default as threadUtils } from "./utils" import { Job } from "bull" -import { - disableCronById, - isErrorInOutput, - isRecurring, -} from "../automations/utils" +import { disableCronById } from "../automations/utils" import * as actions from "../automations/actions" import * as automationUtils from "../automations/automationUtils" -import { replaceFakeBindings } from "../automations/loopUtils" -import { dataFilters, helpers, utils } from "@budibase/shared-core" +import { dataFilters, helpers } from "@budibase/shared-core" import { default as AutomationEmitter } from "../events/AutomationEmitter" import { generateAutomationMetadataID, isProdAppID } from "../db/utils" import { automations } from "@budibase/shared-core" -import { AutomationErrors, MAX_AUTOMATION_RECURRING_ERRORS } from "../constants" +import { MAX_AUTOMATION_RECURRING_ERRORS } from "../constants" import { storeLog } from "../automations/logging" import { Automation, @@ -25,54 +20,172 @@ import { AutomationStepStatus, BranchSearchFilters, BranchStep, - isLogicalSearchOperator, LoopStep, - UserBindings, - isBasicSearchOperator, ContextEmitter, + LoopStepType, + AutomationTriggerResult, + AutomationResults, + AutomationStepResult, + isLogicalFilter, + Branch, } from "@budibase/types" -import { - AutomationContext, - AutomationResponse, - TriggerOutput, -} from "../definitions/automations" +import { AutomationContext } from "../definitions/automations" import { WorkerCallback } from "./definitions" -import { context, logging, configs } from "@budibase/backend-core" +import { context, logging, configs, utils } from "@budibase/backend-core" import { findHBSBlocks, processObject, processStringSync, } from "@budibase/string-templates" import { cloneDeep } from "lodash/fp" -import { performance } from "perf_hooks" import * as sdkUtils from "../sdk/utils" import env from "../environment" import tracer from "dd-trace" +import { isPlainObject } from "lodash" threadUtils.threadSetup() const CRON_STEP_ID = automations.triggers.definitions.CRON.stepId const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED } -function getLoopIterations(loopStep: LoopStep) { - const binding = loopStep.inputs.binding - if (!binding) { - return 0 +function matchesLoopFailureCondition(step: LoopStep, currentItem: any) { + const { failure } = step.inputs + if (!failure) { + return false } - try { - const json = typeof binding === "string" ? JSON.parse(binding) : binding - if (Array.isArray(json)) { - return json.length - } - } catch (err) { - // ignore error - wasn't able to parse + + if (isPlainObject(currentItem)) { + return Object.values(currentItem).some(e => e === failure) } - if (typeof binding === "string") { - return automationUtils.stringSplit(binding).length - } - return 0 + + return currentItem === failure } -export async function enrichBaseContext(context: Record) { +// Returns an array of the things to loop over for a given LoopStep. This +// function handles the various ways that a LoopStep can be configured, parsing +// the input and returning an array of items to loop over. +function getLoopIterable(step: LoopStep): any[] { + const option = step.inputs.option + let input = step.inputs.binding + + if (option === LoopStepType.ARRAY && typeof input === "string") { + input = JSON.parse(input) + } + + if (option === LoopStepType.STRING && Array.isArray(input)) { + input = input.join(",") + } + + if (option === LoopStepType.STRING && typeof input === "string") { + input = automationUtils.stringSplit(input) + } + + return Array.isArray(input) ? input : [input] +} + +function getLoopMaxIterations(loopStep: LoopStep): number { + const loopMaxIterations = + typeof loopStep.inputs.iterations === "string" + ? parseInt(loopStep.inputs.iterations) + : loopStep.inputs.iterations + return Math.min( + loopMaxIterations || env.AUTOMATION_MAX_ITERATIONS, + env.AUTOMATION_MAX_ITERATIONS + ) +} + +function stepSuccess( + step: Readonly, + outputs: Readonly>, + inputs?: Readonly> +): AutomationStepResult { + return { + id: step.id, + stepId: step.stepId, + inputs: inputs || step.inputs, + outputs: { + success: true, + ...outputs, + }, + } +} + +function stepFailure( + step: Readonly, + outputs: Readonly>, + inputs?: Readonly> +): AutomationStepResult { + return { + id: step.id, + stepId: step.stepId, + inputs: inputs || step.inputs, + outputs: { + success: false, + ...outputs, + }, + } +} + +function stepStopped(step: AutomationStep): AutomationStepResult { + return { + id: step.id, + stepId: step.stepId, + inputs: step.inputs, + outputs: STOPPED_STATUS, + } +} + +async function branchMatches( + ctx: AutomationContext, + branch: Readonly +): Promise { + const toFilter: Record = {} + const preparedCtx = prepareContext(ctx) + + // Because we allow bindings on both the left and right of each condition in + // automation branches, we can't pass the BranchSearchFilters directly to + // dataFilters.runQuery as-is. We first need to walk the filter tree and + // evaluate all of the bindings. + const evaluateBindings = (fs: Readonly) => { + const filters = cloneDeep(fs) + for (const filter of Object.values(filters)) { + if (!filter) { + continue + } + + if (isLogicalFilter(filter)) { + filter.conditions = filter.conditions.map(evaluateBindings) + } else { + for (const [field, value] of Object.entries(filter)) { + toFilter[field] = processStringSync(field, preparedCtx) + if (typeof value === "string" && findHBSBlocks(value).length > 0) { + filter[field] = processStringSync(value, preparedCtx) + } + } + } + } + + return filters + } + + const result = dataFilters.runQuery( + [toFilter], + evaluateBindings(branch.condition) + ) + return result.length > 0 +} + +function prepareContext(context: AutomationContext) { + return { + ...context, + steps: { + ...context.steps, + ...context.stepsById, + ...context.stepsByName, + }, + } +} + +async function enrichBaseContext(context: AutomationContext) { context.env = await sdkUtils.getEnvironmentVariables() try { @@ -83,102 +196,70 @@ export async function enrichBaseContext(context: Record) { company: config.company, } } catch (e) { - // if settings doc doesn't exist, make the settings blank context.settings = {} } - - return context } -/** - * The automation orchestrator is a class responsible for executing automations. - * It handles the context of the automation and makes sure each step gets the correct - * inputs and handles any outputs. - */ +// Because the trigger appears twice in an AutomationResult, once as .trigger +// and again as .steps[0], this function makes sure that the two are kept in +// sync when setting trigger output. +function setTriggerOutput(result: AutomationResults, outputs: any) { + result.trigger.outputs = { + ...result.trigger.outputs, + ...outputs, + } + result.steps[0] = result.trigger +} + class Orchestrator { - private chainCount: number - private appId: string - private automation: Automation + private readonly job: AutomationJob private emitter: ContextEmitter - private context: AutomationContext - private job: Job - private loopStepOutputs: LoopStep[] private stopped: boolean - private executionOutput: AutomationResponse - private currentUser: UserBindings | undefined - constructor(job: AutomationJob) { - let automation = job.data.automation - let triggerOutput = job.data.event - const metadata = triggerOutput.metadata - this.chainCount = metadata ? metadata.automationChainCount! : 0 - this.appId = triggerOutput.appId as string + constructor(job: Readonly) { this.job = job - const triggerStepId = automation.definition.trigger.stepId - triggerOutput = this.cleanupTriggerOutputs(triggerStepId, triggerOutput) - // remove from context - delete triggerOutput.appId - delete triggerOutput.metadata - // step zero is never used as the template string is zero indexed for customer facing - this.context = { - steps: [{}], - stepsById: {}, - stepsByName: {}, - trigger: triggerOutput, - } - - this.automation = automation - // create an emitter which has the chain count for this automation run in it, so it can block - // excessive chaining if required - this.emitter = new AutomationEmitter(this.chainCount + 1) - this.executionOutput = { trigger: {}, steps: [] } - // setup the execution output - const triggerId = automation.definition.trigger.id - this.updateExecutionOutput(triggerId, triggerStepId, null, triggerOutput) - this.loopStepOutputs = [] this.stopped = false - this.currentUser = triggerOutput.user + + // create an emitter which has the chain count for this automation run in + // it, so it can block excessive chaining if required + const chainCount = job.data.event.metadata?.automationChainCount || 0 + this.emitter = new AutomationEmitter(chainCount + 1) } - cleanupTriggerOutputs(stepId: string, triggerOutput: TriggerOutput) { - if (stepId === CRON_STEP_ID && !triggerOutput.timestamp) { - triggerOutput.timestamp = Date.now() - } - return triggerOutput + get automation(): Automation { + return this.job.data.automation } - async getStepFunctionality(stepId: AutomationActionStepId) { - let step = await actions.getAction(stepId) - if (step == null) { - throw `Cannot find automation step by name ${stepId}` - } - return step + get appId(): string { + return this.job.data.event.appId! } isCron(): boolean { - return isRecurring(this.automation) + return this.automation.definition.trigger.stepId === CRON_STEP_ID } - async stopCron(reason: string) { + async stopCron(reason: string, opts?: { result: AutomationResults }) { if (!this.isCron()) { return } - logging.logWarn( - `CRON disabled reason=${reason} - ${this.appId}/${this.automation._id}` - ) - const automation = this.automation - const trigger = automation.definition.trigger + + const msg = `CRON disabled reason=${reason} - ${this.appId}/${this.automation._id}` + logging.logWarn(msg) + await disableCronById(this.job.id) - this.updateExecutionOutput( - trigger.id, - trigger.stepId, - {}, - { - status: AutomationStatus.STOPPED_ERROR, + + const { result } = opts || {} + if (result) { + setTriggerOutput(result, { success: false, - } - ) - await storeLog(automation, this.executionOutput) + status: AutomationStatus.STOPPED_ERROR, + }) + await this.logResult(result) + } + } + + private async logResult(result: AutomationResults) { + await storeLog(this.automation, result) } async getMetadata(): Promise { @@ -214,487 +295,293 @@ class Orchestrator { return undefined } - updateExecutionOutput(id: string, stepId: string, inputs: any, outputs: any) { - const stepObj = { id, stepId, inputs, outputs } - // replacing trigger when disabling CRON - if ( - stepId === CRON_STEP_ID && - outputs.status === AutomationStatus.STOPPED_ERROR - ) { - this.executionOutput.trigger = stepObj - this.executionOutput.steps = [stepObj] - return - } - // first entry is always the trigger (constructor) - if ( - this.executionOutput.steps.length === 0 || - this.executionOutput.trigger.id === id - ) { - this.executionOutput.trigger = stepObj - } - this.executionOutput.steps.push(stepObj) + private isProdApp(): boolean { + return isProdAppID(this.appId) } - updateContextAndOutput( - currentLoopStepIndex: number | undefined, - step: AutomationStep, - output: any, - result: { success: boolean; status: string } - ) { - if (currentLoopStepIndex === undefined) { - throw new Error("No loop step number provided.") + hasErrored(context: AutomationContext): boolean { + const [_trigger, ...steps] = context.steps + for (const step of steps) { + if (step.success === false) { + return true + } } - this.executionOutput.steps.splice(currentLoopStepIndex, 0, { - id: step.id, - stepId: step.stepId, - outputs: { - ...output, - success: result.success, - status: result.status, - }, - inputs: step.inputs, - }) - this.context.steps.splice(currentLoopStepIndex, 0, { - ...output, - success: result.success, - status: result.status, - }) + return false } - async execute(): Promise { + async execute(): Promise { return tracer.trace( "Orchestrator.execute", { resource: "automation" }, async span => { - span?.addTags({ - appId: this.appId, - automationId: this.automation._id, - }) + span?.addTags({ appId: this.appId, automationId: this.automation._id }) - await enrichBaseContext(this.context) - this.context.user = this.currentUser + const job = cloneDeep(this.job) + delete job.data.event.appId + delete job.data.event.metadata - const start = performance.now() + if (this.isCron() && !job.data.event.timestamp) { + job.data.event.timestamp = Date.now() + } - await this.executeSteps(this.automation.definition.steps) + const trigger: AutomationTriggerResult = { + id: job.data.automation.definition.trigger.id, + stepId: job.data.automation.definition.trigger.stepId, + inputs: null, + outputs: job.data.event, + } + const result: AutomationResults = { trigger, steps: [trigger] } - const end = performance.now() - const executionTime = end - start + const ctx: AutomationContext = { + trigger: trigger.outputs, + steps: [trigger.outputs], + stepsById: {}, + stepsByName: {}, + user: trigger.outputs.user, + } + await enrichBaseContext(ctx) - console.info( - `Automation ID: ${this.automation._id} Execution time: ${executionTime} milliseconds`, - { - _logKey: "automation", - executionTime, + const timeout = + this.job.data.event.timeout || env.AUTOMATION_THREAD_TIMEOUT + + try { + await helpers.withTimeout(timeout, async () => { + const [stepOutputs, executionTime] = await utils.time(() => + this.executeSteps(ctx, job.data.automation.definition.steps) + ) + + result.steps.push(...stepOutputs) + + console.info( + `Automation ID: ${ + this.automation._id + } Execution time: ${executionTime.toMs()} milliseconds`, + { + _logKey: "automation", + executionTime, + } + ) + }) + } catch (e: any) { + if (e.errno === "ETIME") { + span?.addTags({ timedOut: true }) + console.warn(`Automation execution timed out after ${timeout}ms`) } - ) + } let errorCount = 0 - if ( - isProdAppID(this.appId) && - this.isCron() && - isErrorInOutput(this.executionOutput) - ) { + if (this.isProdApp() && this.isCron() && this.hasErrored(ctx)) { errorCount = (await this.incrementErrorCount()) || 0 } if (errorCount >= MAX_AUTOMATION_RECURRING_ERRORS) { - await this.stopCron("errors") + await this.stopCron("errors", { result }) span?.addTags({ shouldStop: true }) } else { - await storeLog(this.automation, this.executionOutput) + await this.logResult(result) } - return this.executionOutput + return result } ) } private async executeSteps( - steps: AutomationStep[], - pathIdx?: number - ): Promise { - return tracer.trace( - "Orchestrator.executeSteps", - { resource: "automation" }, - async span => { - let stepIndex = 0 - const timeout = - this.job.data.event.timeout || env.AUTOMATION_THREAD_TIMEOUT + ctx: AutomationContext, + steps: AutomationStep[] + ): Promise { + return tracer.trace("Orchestrator.executeSteps", async () => { + let stepIndex = 0 + const results: AutomationStepResult[] = [] - try { - await helpers.withTimeout( - timeout, - (async () => { - while (stepIndex < steps.length) { - const step = steps[stepIndex] - if (step.stepId === AutomationActionStepId.BRANCH) { - // stepIndex for current step context offset - // pathIdx relating to the full list of steps in the run - await this.executeBranchStep(step, stepIndex + (pathIdx || 0)) - stepIndex++ - } else if (step.stepId === AutomationActionStepId.LOOP) { - stepIndex = await this.executeLoopStep( - step, - steps, - stepIndex, - pathIdx - ) - } else { - if (!this.stopped) { - await this.executeStep(step) - } - stepIndex++ - } - } - })() - ) - } catch (error: any) { - if (error.errno === "ETIME") { - span?.addTags({ timedOut: true }) - console.warn(`Automation execution timed out after ${timeout}ms`) + function addToContext( + step: AutomationStep, + result: AutomationStepResult + ) { + ctx.steps.push(result.outputs) + ctx.stepsById[step.id] = result.outputs + ctx.stepsByName[step.name || step.id] = result.outputs + results.push(result) + } + + while (stepIndex < steps.length) { + if (this.stopped) { + break + } + + const step = steps[stepIndex] + switch (step.stepId) { + case AutomationActionStepId.BRANCH: { + results.push(...(await this.executeBranchStep(ctx, step))) + stepIndex++ + break + } + case AutomationActionStepId.LOOP: { + const stepToLoop = steps[stepIndex + 1] + addToContext( + stepToLoop, + await this.executeLoopStep(ctx, step, stepToLoop) + ) + // We increment by 2 here because the way loops work is that the + // step immediately following the loop step is what gets looped. + // So when we're done looping, to advance correctly we need to + // skip the step that was looped. + stepIndex += 2 + break + } + default: { + addToContext(step, await this.executeStep(ctx, step)) + stepIndex++ + break } } } - ) + + return results + }) } private async executeLoopStep( - loopStep: LoopStep, - steps: AutomationStep[], - stepIdx: number, - pathIdx?: number - ): Promise { - await processObject(loopStep.inputs, this.mergeContexts(this.context)) - const iterations = getLoopIterations(loopStep) - let stepToLoopIndex = stepIdx + 1 - let pathStepIdx = (pathIdx || stepIdx) + 1 + ctx: AutomationContext, + step: LoopStep, + stepToLoop: AutomationStep + ): Promise { + await processObject(step.inputs, prepareContext(ctx)) - let iterationCount = 0 - let shouldCleanup = true - let reachedMaxIterations = false - - for (let loopStepIndex = 0; loopStepIndex < iterations; loopStepIndex++) { - try { - loopStep.inputs.binding = automationUtils.typecastForLooping( - loopStep.inputs - ) - } catch (err) { - this.updateContextAndOutput( - pathStepIdx + 1, - steps[stepToLoopIndex], - {}, - { - status: AutomationErrors.INCORRECT_TYPE, - success: false, - } - ) - shouldCleanup = false - break - } - const maxIterations = automationUtils.ensureMaxIterationsAsNumber( - loopStep.inputs.iterations - ) - - if ( - loopStepIndex === env.AUTOMATION_MAX_ITERATIONS || - (loopStep.inputs.iterations && loopStepIndex === maxIterations) - ) { - reachedMaxIterations = true - shouldCleanup = true - break - } - - let isFailure = false - const currentItem = this.getCurrentLoopItem(loopStep, loopStepIndex) - if (currentItem && typeof currentItem === "object") { - isFailure = Object.keys(currentItem).some(value => { - return currentItem[value] === loopStep?.inputs.failure - }) - } else { - isFailure = currentItem && currentItem === loopStep.inputs.failure - } - - if (isFailure) { - this.updateContextAndOutput( - pathStepIdx + 1, - steps[stepToLoopIndex], - { - items: this.loopStepOutputs, - iterations: loopStepIndex, - }, - { - status: AutomationErrors.FAILURE_CONDITION, - success: false, - } - ) - shouldCleanup = false - break - } - - this.context.steps[pathStepIdx] = { - currentItem: this.getCurrentLoopItem(loopStep, loopStepIndex), - } - - stepToLoopIndex = stepIdx + 1 - - await this.executeStep(steps[stepToLoopIndex], stepToLoopIndex) - iterationCount++ - } - - if (shouldCleanup) { - let tempOutput = - iterations === 0 - ? { - status: AutomationStepStatus.NO_ITERATIONS, - success: true, - } - : { - success: true, - items: this.loopStepOutputs, - iterations: iterationCount, - } - - if (reachedMaxIterations && iterations !== 0) { - tempOutput.status = AutomationStepStatus.MAX_ITERATIONS - } - - // Loop Step clean up - this.executionOutput.steps.splice(pathStepIdx, 0, { - id: steps[stepToLoopIndex].id, - stepId: steps[stepToLoopIndex].stepId, - outputs: tempOutput, - inputs: steps[stepToLoopIndex].inputs, + const maxIterations = getLoopMaxIterations(step) + const items: Record[] = [] + let iterations = 0 + let iterable: any[] = [] + try { + iterable = getLoopIterable(step) + } catch (err) { + return stepFailure(stepToLoop, { + status: AutomationStepStatus.INCORRECT_TYPE, }) - - this.context.stepsById[steps[stepToLoopIndex].id] = tempOutput - const stepName = steps[stepToLoopIndex].name || steps[stepToLoopIndex].id - this.context.stepsByName[stepName] = tempOutput - this.context.steps[this.context.steps.length] = tempOutput - this.context.steps = this.context.steps.filter( - item => !item.hasOwnProperty.call(item, "currentItem") - ) - - this.loopStepOutputs = [] } - return stepToLoopIndex + 1 + for (; iterations < iterable.length; iterations++) { + const currentItem = iterable[iterations] + + if (iterations === maxIterations) { + return stepFailure(stepToLoop, { + status: AutomationStepStatus.MAX_ITERATIONS, + iterations, + }) + } + + if (matchesLoopFailureCondition(step, currentItem)) { + return stepFailure(stepToLoop, { + status: AutomationStepStatus.FAILURE_CONDITION, + }) + } + + ctx.loop = { currentItem } + const result = await this.executeStep(ctx, stepToLoop) + items.push(result.outputs) + ctx.loop = undefined + } + + const status = + iterations === 0 ? AutomationStatus.NO_CONDITION_MET : undefined + return stepSuccess(stepToLoop, { status, iterations, items }) } + private async executeBranchStep( - branchStep: BranchStep, - pathIdx?: number - ): Promise { - const { branches, children } = branchStep.inputs + ctx: AutomationContext, + step: BranchStep + ): Promise { + const { branches, children } = step.inputs for (const branch of branches) { - const condition = await this.evaluateBranchCondition(branch.condition) - if (condition) { - const branchStatus = { - branchName: branch.name, - status: `${branch.name} branch taken`, - branchId: `${branch.id}`, - success: true, - } - - this.updateExecutionOutput( - branchStep.id, - branchStep.stepId, - branchStep.inputs, - branchStatus - ) - this.context.steps[this.context.steps.length] = branchStatus - this.context.stepsById[branchStep.id] = branchStatus - - const branchSteps = children?.[branch.id] || [] - // A final +1 to accomodate the branch step itself - await this.executeSteps(branchSteps, (pathIdx || 0) + 1) - return + if (await branchMatches(ctx, branch)) { + return [ + stepSuccess(step, { + branchName: branch.name, + status: `${branch.name} branch taken`, + branchId: `${branch.id}`, + }), + ...(await this.executeSteps(ctx, children?.[branch.id] || [])), + ] } } - this.stopped = true - this.updateExecutionOutput( - branchStep.id, - branchStep.stepId, - branchStep.inputs, - { - success: false, - status: AutomationStatus.NO_CONDITION_MET, - } - ) + return [stepFailure(step, { status: AutomationStatus.NO_CONDITION_MET })] } - private async evaluateBranchCondition( - conditions: BranchSearchFilters - ): Promise { - const toFilter: Record = {} - - const recurseSearchFilters = ( - filters: BranchSearchFilters - ): BranchSearchFilters => { - for (const filterKey of Object.keys( - filters - ) as (keyof typeof filters)[]) { - if (!filters[filterKey]) { - continue - } - - if (isLogicalSearchOperator(filterKey)) { - filters[filterKey].conditions = filters[filterKey].conditions.map( - condition => recurseSearchFilters(condition) - ) - } else if (isBasicSearchOperator(filterKey)) { - for (const [field, value] of Object.entries(filters[filterKey])) { - const fromContext = processStringSync( - field, - this.mergeContexts(this.context) - ) - toFilter[field] = fromContext - - if (typeof value === "string" && findHBSBlocks(value).length > 0) { - const processedVal = processStringSync( - value, - this.mergeContexts(this.context) - ) - - filters[filterKey][field] = processedVal - } - } - } else { - // We want to types to complain if we extend BranchSearchFilters, but not to throw if the request comes with some extra data. It will just be ignored - utils.unreachable(filterKey, { doNotThrow: true }) - } - } - - return filters - } - - const processedConditions = recurseSearchFilters(conditions) - - const result = dataFilters.runQuery([toFilter], processedConditions) - return result.length > 0 - } private async executeStep( - step: AutomationStep, - loopIteration?: number - ): Promise { - return tracer.trace( - "Orchestrator.execute.step", - { resource: "automation" }, - async span => { - span?.addTags({ - resource: "automation", - step: { - stepId: step.stepId, - id: step.id, - name: step.name, - type: step.type, - title: step.stepTitle, - internal: step.internal, - deprecated: step.deprecated, - }, - }) - - if (this.stopped) { - this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS) - return - } - - let originalStepInput = cloneDeep(step.inputs) - if (loopIteration !== undefined) { - originalStepInput = replaceFakeBindings( - originalStepInput, - loopIteration - ) - } - - const stepFn = await this.getStepFunctionality(step.stepId) - let inputs = await processObject( - originalStepInput, - this.mergeContexts(this.context) - ) - inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs) - - const outputs = await stepFn({ - inputs: inputs, - appId: this.appId, - emitter: this.emitter, - context: this.mergeContexts(this.context), - }) - this.handleStepOutput(step, outputs, loopIteration) - } - ) - } - - private getCurrentLoopItem(loopStep: LoopStep, index: number): any { - if (!loopStep) return null - if ( - typeof loopStep.inputs.binding === "string" && - loopStep.inputs.option === "String" - ) { - return automationUtils.stringSplit(loopStep.inputs.binding)[index] - } else if (Array.isArray(loopStep.inputs.binding)) { - return loopStep.inputs.binding[index] - } - return null - } - - private mergeContexts(context: AutomationContext) { - const mergeContexts = { - ...context, - steps: { - ...context.steps, - ...context.stepsById, - ...context.stepsByName, - }, - } - return mergeContexts - } - - private handleStepOutput( - step: AutomationStep, - outputs: any, - loopIteration: number | undefined - ): void { - if (step.stepId === AutomationActionStepId.FILTER && !outputs.result) { - this.stopped = true - this.updateExecutionOutput(step.id, step.stepId, step.inputs, { - ...outputs, - ...STOPPED_STATUS, + ctx: AutomationContext, + step: Readonly + ): Promise { + return tracer.trace("Orchestrator.executeStep", async span => { + span.addTags({ + step: { + stepId: step.stepId, + id: step.id, + name: step.name, + type: step.type, + title: step.stepTitle, + internal: step.internal, + deprecated: step.deprecated, + }, }) - } else if (loopIteration !== undefined) { - this.loopStepOutputs = this.loopStepOutputs || [] - this.loopStepOutputs.push(outputs) - } else { - this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs) - this.context.steps[this.context.steps.length] = outputs - this.context.stepsById![step.id] = outputs - const stepName = step.name || step.id - this.context.stepsByName![stepName] = outputs - } + + if (this.stopped) { + span.addTags({ stopped: true }) + return stepStopped(step) + } + + const fn = await actions.getAction(step.stepId) + if (fn == null) { + throw new Error(`Cannot find automation step by name ${step.stepId}`) + } + + const inputs = automationUtils.cleanInputValues( + await processObject(cloneDeep(step.inputs), prepareContext(ctx)), + step.schema.inputs.properties + ) + + const outputs = await fn({ + inputs, + appId: this.appId, + emitter: this.emitter, + context: prepareContext(ctx), + }) + + if ( + step.stepId === AutomationActionStepId.FILTER && + "result" in outputs && + outputs.result === false + ) { + this.stopped = true + ;(outputs as any).status = AutomationStatus.STOPPED + } + + return stepSuccess(step, outputs, inputs) + }) } } export function execute(job: Job, callback: WorkerCallback) { const appId = job.data.event.appId - const automationId = job.data.automation._id if (!appId) { throw new Error("Unable to execute, event doesn't contain app ID.") } + + const automationId = job.data.automation._id if (!automationId) { throw new Error("Unable to execute, event doesn't contain automation ID.") } + return context.doInAutomationContext({ appId, automationId, task: async () => { const envVars = await sdkUtils.getEnvironmentVariables() - // put into automation thread for whole context await context.doInEnvironmentContext(envVars, async () => { - const automationOrchestrator = new Orchestrator(job) + const orchestrator = new Orchestrator(job) try { - const response = await automationOrchestrator.execute() - callback(null, response) + callback(null, await orchestrator.execute()) } catch (err) { callback(err) } @@ -705,30 +592,20 @@ export function execute(job: Job, callback: WorkerCallback) { export async function executeInThread( job: Job -): Promise { +): Promise { const appId = job.data.event.appId if (!appId) { throw new Error("Unable to execute, event doesn't contain app ID.") } - const timeoutPromise = new Promise((resolve, reject) => { - setTimeout(() => { - reject(new Error("Timeout exceeded")) - }, job.data.event.timeout || env.AUTOMATION_THREAD_TIMEOUT) - }) - - return (await context.doInAppContext(appId, async () => { + return await context.doInAppContext(appId, async () => { await context.ensureSnippetContext() const envVars = await sdkUtils.getEnvironmentVariables() - // put into automation thread for whole context return await context.doInEnvironmentContext(envVars, async () => { - const automationOrchestrator = new Orchestrator(job) - return await Promise.race([ - automationOrchestrator.execute(), - timeoutPromise, - ]) + const orchestrator = new Orchestrator(job) + return orchestrator.execute() }) - })) as AutomationResponse + }) } export const removeStalled = async (job: Job) => { @@ -737,7 +614,7 @@ export const removeStalled = async (job: Job) => { throw new Error("Unable to execute, event doesn't contain app ID.") } await context.doInAppContext(appId, async () => { - const automationOrchestrator = new Orchestrator(job) - await automationOrchestrator.stopCron("stalled") + const orchestrator = new Orchestrator(job) + await orchestrator.stopCron("stalled") }) } diff --git a/packages/shared-core/src/automations/steps/filter.ts b/packages/shared-core/src/automations/steps/filter.ts index 70dcb6f66e..6305e667c7 100644 --- a/packages/shared-core/src/automations/steps/filter.ts +++ b/packages/shared-core/src/automations/steps/filter.ts @@ -3,20 +3,14 @@ import { AutomationStepDefinition, AutomationStepType, AutomationIOType, + FilterCondition, } from "@budibase/types" -export const FilterConditions = { - EQUAL: "EQUAL", - NOT_EQUAL: "NOT_EQUAL", - GREATER_THAN: "GREATER_THAN", - LESS_THAN: "LESS_THAN", -} - export const PrettyFilterConditions = { - [FilterConditions.EQUAL]: "Equals", - [FilterConditions.NOT_EQUAL]: "Not equals", - [FilterConditions.GREATER_THAN]: "Greater than", - [FilterConditions.LESS_THAN]: "Less than", + [FilterCondition.EQUAL]: "Equals", + [FilterCondition.NOT_EQUAL]: "Not equals", + [FilterCondition.GREATER_THAN]: "Greater than", + [FilterCondition.LESS_THAN]: "Less than", } export const definition: AutomationStepDefinition = { @@ -30,7 +24,7 @@ export const definition: AutomationStepDefinition = { features: {}, stepId: AutomationActionStepId.FILTER, inputs: { - condition: FilterConditions.EQUAL, + condition: FilterCondition.EQUAL, }, schema: { inputs: { @@ -42,7 +36,7 @@ export const definition: AutomationStepDefinition = { condition: { type: AutomationIOType.STRING, title: "Condition", - enum: Object.values(FilterConditions), + enum: Object.values(FilterCondition), pretty: Object.values(PrettyFilterConditions), }, value: { diff --git a/packages/shared-core/src/helpers/helpers.ts b/packages/shared-core/src/helpers/helpers.ts index 8dbdb7bbfd..10d625be28 100644 --- a/packages/shared-core/src/helpers/helpers.ts +++ b/packages/shared-core/src/helpers/helpers.ts @@ -105,10 +105,10 @@ export function cancelableTimeout( export async function withTimeout( timeout: number, - promise: Promise + promise: () => Promise ): Promise { const [timeoutPromise, cancel] = cancelableTimeout(timeout) - const result = (await Promise.race([promise, timeoutPromise])) as T + const result = (await Promise.race([promise(), timeoutPromise])) as T cancel() return result } diff --git a/packages/types/src/documents/app/automation/StepInputsOutputs.ts b/packages/types/src/documents/app/automation/StepInputsOutputs.ts index 18a6f86284..52b07ae17f 100644 --- a/packages/types/src/documents/app/automation/StepInputsOutputs.ts +++ b/packages/types/src/documents/app/automation/StepInputsOutputs.ts @@ -7,8 +7,20 @@ import { } from "../../../sdk" import { HttpMethod } from "../query" import { Row } from "../row" -import { LoopStepType, EmailAttachment, AutomationResults } from "./automation" -import { AutomationStep, AutomationStepOutputs } from "./schema" +import { + LoopStepType, + EmailAttachment, + AutomationResults, + AutomationStepResult, +} from "./automation" +import { AutomationStep } from "./schema" + +export enum FilterCondition { + EQUAL = "EQUAL", + NOT_EQUAL = "NOT_EQUAL", + GREATER_THAN = "GREATER_THAN", + LESS_THAN = "LESS_THAN", +} export type BaseAutomationOutputs = { success?: boolean @@ -92,7 +104,7 @@ export type ExecuteScriptStepOutputs = BaseAutomationOutputs & { export type FilterStepInputs = { field: any - condition: string + condition: FilterCondition value: any } @@ -110,7 +122,7 @@ export type LoopStepInputs = { } export type LoopStepOutputs = { - items: AutomationStepOutputs[] + items: AutomationStepResult[] success: boolean iterations: number } diff --git a/packages/types/src/documents/app/automation/automation.ts b/packages/types/src/documents/app/automation/automation.ts index 0314701d72..d5ef35d059 100644 --- a/packages/types/src/documents/app/automation/automation.ts +++ b/packages/types/src/documents/app/automation/automation.ts @@ -146,7 +146,7 @@ export interface Automation extends Document { } } -interface BaseIOStructure { +export interface BaseIOStructure { type?: AutomationIOType subtype?: AutomationIOType customType?: AutomationCustomIOType @@ -176,6 +176,8 @@ export enum AutomationFeature { export enum AutomationStepStatus { NO_ITERATIONS = "no_iterations", MAX_ITERATIONS = "max_iterations_reached", + FAILURE_CONDITION = "FAILURE_CONDITION_MET", + INCORRECT_TYPE = "INCORRECT_TYPE", } export enum AutomationStatus { @@ -190,19 +192,37 @@ export enum AutomationStoppedReason { TRIGGER_FILTER_NOT_MET = "Automation did not run. Filter conditions in trigger were not met.", } +export interface AutomationStepResultOutputs { + success: boolean + [key: string]: any +} + +export interface AutomationStepResultInputs { + [key: string]: any +} + +export interface AutomationStepResult { + id: string + stepId: AutomationActionStepId + inputs: AutomationStepResultInputs + outputs: AutomationStepResultOutputs +} + +export type AutomationTriggerResultInputs = Record +export type AutomationTriggerResultOutputs = Record + +export interface AutomationTriggerResult { + id: string + stepId: AutomationTriggerStepId + inputs?: AutomationTriggerResultInputs | null + outputs: AutomationTriggerResultOutputs +} + export interface AutomationResults { automationId?: string status?: AutomationStatus - trigger?: AutomationTrigger - steps: { - stepId: AutomationTriggerStepId | AutomationActionStepId - inputs: { - [key: string]: any - } - outputs: { - [key: string]: any - } - }[] + trigger: AutomationTriggerResult + steps: [AutomationTriggerResult, ...AutomationStepResult[]] } export interface DidNotTriggerResponse { @@ -236,6 +256,7 @@ export type ActionImplementation = ( inputs: TInputs } & AutomationStepInputBase ) => Promise + export interface AutomationMetadata extends Document { errorCount?: number automationChainCount?: number diff --git a/packages/types/src/documents/app/automation/schema.ts b/packages/types/src/documents/app/automation/schema.ts index 820858b48c..745737e2a6 100644 --- a/packages/types/src/documents/app/automation/schema.ts +++ b/packages/types/src/documents/app/automation/schema.ts @@ -164,24 +164,6 @@ export interface AutomationStepSchemaBase { features?: Partial> } -export type AutomationStepOutputs = - | CollectStepOutputs - | CreateRowStepOutputs - | DelayStepOutputs - | DeleteRowStepOutputs - | ExecuteQueryStepOutputs - | ExecuteScriptStepOutputs - | FilterStepOutputs - | QueryRowsStepOutputs - | BaseAutomationOutputs - | BashStepOutputs - | ExternalAppStepOutputs - | OpenAIStepOutputs - | ServerLogStepOutputs - | TriggerAutomationStepOutputs - | UpdateRowStepOutputs - | ZapierStepOutputs - export type AutomationStepInputs = T extends AutomationActionStepId.COLLECT ? CollectStepInputs @@ -229,11 +211,56 @@ export type AutomationStepInputs = ? BranchStepInputs : never +export type AutomationStepOutputs = + T extends AutomationActionStepId.COLLECT + ? CollectStepOutputs + : T extends AutomationActionStepId.CREATE_ROW + ? CreateRowStepOutputs + : T extends AutomationActionStepId.DELAY + ? DelayStepOutputs + : T extends AutomationActionStepId.DELETE_ROW + ? DeleteRowStepOutputs + : T extends AutomationActionStepId.EXECUTE_QUERY + ? ExecuteQueryStepOutputs + : T extends AutomationActionStepId.EXECUTE_SCRIPT + ? ExecuteScriptStepOutputs + : T extends AutomationActionStepId.FILTER + ? FilterStepOutputs + : T extends AutomationActionStepId.QUERY_ROWS + ? QueryRowsStepOutputs + : T extends AutomationActionStepId.SEND_EMAIL_SMTP + ? BaseAutomationOutputs + : T extends AutomationActionStepId.SERVER_LOG + ? ServerLogStepOutputs + : T extends AutomationActionStepId.TRIGGER_AUTOMATION_RUN + ? TriggerAutomationStepOutputs + : T extends AutomationActionStepId.UPDATE_ROW + ? UpdateRowStepOutputs + : T extends AutomationActionStepId.OUTGOING_WEBHOOK + ? ExternalAppStepOutputs + : T extends AutomationActionStepId.discord + ? ExternalAppStepOutputs + : T extends AutomationActionStepId.slack + ? ExternalAppStepOutputs + : T extends AutomationActionStepId.zapier + ? ZapierStepOutputs + : T extends AutomationActionStepId.integromat + ? ExternalAppStepOutputs + : T extends AutomationActionStepId.n8n + ? ExternalAppStepOutputs + : T extends AutomationActionStepId.EXECUTE_BASH + ? BashStepOutputs + : T extends AutomationActionStepId.OPENAI + ? OpenAIStepOutputs + : T extends AutomationActionStepId.LOOP + ? BaseAutomationOutputs + : never + export interface AutomationStepSchema extends AutomationStepSchemaBase { id: string stepId: TStep - inputs: AutomationStepInputs & Record // The record union to be removed once the types are fixed + inputs: AutomationStepInputs } export type CollectStep = AutomationStepSchema @@ -315,6 +342,36 @@ export type AutomationStep = | OpenAIStep | BranchStep +export function isBranchStep( + step: AutomationStep | AutomationTrigger +): step is BranchStep { + return step.stepId === AutomationActionStepId.BRANCH +} + +export function isTrigger( + step: AutomationStep | AutomationTrigger +): step is AutomationTrigger { + return step.type === AutomationStepType.TRIGGER +} + +export function isRowUpdateTrigger( + step: AutomationStep | AutomationTrigger +): step is RowUpdatedTrigger { + return step.stepId === AutomationTriggerStepId.ROW_UPDATED +} + +export function isRowSaveTrigger( + step: AutomationStep | AutomationTrigger +): step is RowSavedTrigger { + return step.stepId === AutomationTriggerStepId.ROW_SAVED +} + +export function isAppTrigger( + step: AutomationStep | AutomationTrigger +): step is AppActionTrigger { + return step.stepId === AutomationTriggerStepId.APP +} + type EmptyInputs = {} export type AutomationStepDefinition = Omit & { inputs: EmptyInputs diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index fdc2fafe57..992e9961d4 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -82,6 +82,10 @@ type RangeFilter = Record< type LogicalFilter = { conditions: SearchFilters[] } +export function isLogicalFilter(filter: any): filter is LogicalFilter { + return "conditions" in filter +} + export type AnySearchFilter = BasicFilter | ArrayFilter | RangeFilter export interface SearchFilters { diff --git a/packages/worker/src/api/routes/global/tests/realEmail.spec.ts b/packages/worker/src/api/routes/global/tests/realEmail.spec.ts index bf5ed7b4ee..479a1f0476 100644 --- a/packages/worker/src/api/routes/global/tests/realEmail.spec.ts +++ b/packages/worker/src/api/routes/global/tests/realEmail.spec.ts @@ -31,8 +31,8 @@ describe("/api/global/email", () => { ) { let response, text try { - await helpers.withTimeout(20000, config.saveEtherealSmtpConfig()) - await helpers.withTimeout(20000, config.saveSettingsConfig()) + await helpers.withTimeout(20000, () => config.saveEtherealSmtpConfig()) + await helpers.withTimeout(20000, () => config.saveSettingsConfig()) let res if (attachments) { res = await config.api.emails