Merge pull request #5145 from Budibase/feature/foreach-block

Foreach Automation Block
This commit is contained in:
Peter Clement 2022-04-19 12:58:00 +01:00 committed by GitHub
commit bb00d69183
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 490 additions and 110 deletions

View File

@ -114,6 +114,9 @@ spec:
value: {{ .Values.globals.google.clientId | quote }}
- name: GOOGLE_CLIENT_SECRET
value: {{ .Values.globals.google.secret | quote }}
- name: AUTOMATION_MAX_ITERATIONS
value: {{ .Values.globals.automationMaxIterations | quote }}
image: budibase/apps:{{ .Values.globals.appVersion }}
imagePullPolicy: Always
name: bbapps

View File

@ -103,6 +103,7 @@ globals:
google:
clientId: ""
secret: ""
automationMaxIterations: "500"
createSecrets: true # creates an internal API key, JWT secrets and redis password for you

View File

@ -20,7 +20,6 @@ filterTests(['smoke', 'all'], () => {
})
// Setup trigger
cy.contains("Setup").click()
cy.get(".spectrum-Picker-label").click()
cy.wait(500)
cy.contains("dog").click()
@ -32,7 +31,6 @@ filterTests(['smoke', 'all'], () => {
cy.contains("Create Row").trigger('mouseover').click().click()
cy.get(".spectrum-Button--cta").click()
})
cy.contains("Setup").click()
cy.get(".spectrum-Picker-label").eq(1).click()
cy.contains("dog").click()
cy.get(".spectrum-Textfield-input")

View File

@ -39,6 +39,7 @@
if (v.internal) {
acc[k] = v
}
delete acc.LOOP
return acc
}, {})

View File

@ -72,7 +72,9 @@
animate:flip={{ duration: 500 }}
in:fly|local={{ x: 500, duration: 1500 }}
>
{#if block.stepId !== "LOOP"}
<FlowItem {testDataModal} {block} />
{/if}
</div>
{/each}
</div>

View File

@ -9,8 +9,8 @@
Modal,
Button,
StatusLight,
ActionButton,
Select,
ActionButton,
notifications,
} from "@budibase/bbui"
import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
@ -25,8 +25,8 @@
let webhookModal
let actionModal
let resultsModal
let setupToggled
let blockComplete
let showLooping = false
$: rowControl = $automationStore.selectedAutomation.automation.rowControl
$: showBindingPicker =
@ -52,8 +52,21 @@
block.schema?.inputs?.properties || {}
).every(x => block?.inputs[x])
$: loopingSelected =
$automationStore.selectedAutomation?.automation.definition.steps.find(
x => x.blockToLoop === block.id
)
async function deleteStep() {
let loopBlock =
$automationStore.selectedAutomation?.automation.definition.steps.find(
x => x.blockToLoop === block.id
)
try {
if (loopBlock) {
automationStore.actions.deleteAutomationBlock(loopBlock)
}
automationStore.actions.deleteAutomationBlock(block)
await automationStore.actions.save(
$automationStore.selectedAutomation?.automation
@ -76,6 +89,23 @@
)
}
async function addLooping() {
loopingSelected = true
const loopDefinition = $automationStore.blockDefinitions.ACTION.LOOP
const loopBlock = $automationStore.selectedAutomation.constructBlock(
"ACTION",
"LOOP",
loopDefinition
)
loopBlock.blockToLoop = block.id
block.loopBlock = loopBlock.id
automationStore.actions.addBlockToAutomation(loopBlock, blockIdx)
await automationStore.actions.save(
$automationStore.selectedAutomation?.automation
)
}
async function onSelect(block) {
await automationStore.update(state => {
state.selectedBlock = block
@ -84,13 +114,68 @@
}
</script>
<div
class={`block ${block.type} hoverable`}
class:selected
<div class={`block ${block.type} hoverable`} class:selected on:click={() => {}}>
{#if loopingSelected}
<div class="blockSection">
<div
on:click={() => {
showLooping = !showLooping
}}
class="splitHeader"
>
<div class="center-items">
<svg
width="28px"
height="28px"
class="spectrum-Icon"
style="color:grey;"
focusable="false"
>
<use xlink:href="#spectrum-icon-18-Reuse" />
</svg>
<div class="iconAlign">
<Detail size="S">Looping</Detail>
</div>
</div>
<div class="blockTitle">
<div
style="margin-left: 10px;"
on:click={() => {
onSelect(block)
}}
>
>
<Icon name={showLooping ? "ChevronDown" : "ChevronUp"} />
</div>
</div>
</div>
</div>
<Divider noMargin />
{#if !showLooping}
<div class="blockSection">
<div class="block-options">
<div class="delete-padding" on:click={() => deleteStep()}>
<Icon name="DeleteOutline" />
</div>
</div>
<Layout noPadding gap="S">
<AutomationBlockSetup
schemaProperties={Object.entries(
$automationStore.blockDefinitions.ACTION.LOOP.schema.inputs
.properties
)}
block={$automationStore.selectedAutomation?.automation.definition.steps.find(
x => x.blockToLoop === block.id
)}
{webhookModal}
/>
</Layout>
</div>
<Divider noMargin />
{/if}
{/if}
<div class="blockSection">
<div
on:click={() => {
@ -127,55 +212,57 @@
<Detail size="S">{block?.name?.toUpperCase() || ""}</Detail>
</div>
</div>
<div class="blockTitle">
{#if testResult && testResult[0]}
<span on:click={() => resultsModal.show()}>
<div style="float: right;" on:click={() => resultsModal.show()}>
<StatusLight
positive={isTrigger || testResult[0].outputs?.success}
negative={!testResult[0].outputs?.success}
><Body size="XS">View response</Body></StatusLight
>
</span>
</div>
{/if}
<div
style="margin-left: 10px;"
on:click={() => {
onSelect(block)
}}
>
<Icon name={blockComplete ? "ChevronDown" : "ChevronUp"} />
</div>
</div>
</div>
</div>
{#if !blockComplete}
<Divider noMargin />
<div class="blockSection">
<Layout noPadding gap="S">
<div class="splitHeader">
<ActionButton
on:click={() => {
onSelect(block)
setupToggled = !setupToggled
}}
quiet
icon={setupToggled ? "ChevronDown" : "ChevronRight"}
>
<Detail size="S">Setup</Detail>
</ActionButton>
{#if !isTrigger}
<div class="block-options">
{#if showBindingPicker}
<div>
<div class="block-options">
{#if !loopingSelected}
<ActionButton on:click={() => addLooping()} icon="Reuse"
>Add Looping</ActionButton
>
{/if}
{#if showBindingPicker}
<Select
on:change={toggleFieldControl}
quiet
defaultValue="Use values"
autoWidth
value={rowControl ? "Use bindings" : "Use values"}
options={["Use values", "Use bindings"]}
placeholder={null}
/>
</div>
{/if}
<div class="delete-padding" on:click={() => deleteStep()}>
<Icon name="DeleteOutline" />
<ActionButton
on:click={() => deleteStep()}
icon="DeleteOutline"
/>
</div>
</div>
{/if}
</div>
{#if setupToggled}
<AutomationBlockSetup
schemaProperties={Object.entries(block.schema.inputs.properties)}
{block}
@ -186,7 +273,6 @@
>Finish and test automation</Button
>
{/if}
{/if}
</Layout>
</div>
{/if}
@ -220,8 +306,10 @@
padding-left: 30px;
}
.block-options {
display: flex;
justify-content: flex-end;
align-items: center;
display: flex;
gap: var(--spacing-m);
}
.center-items {
display: flex;
@ -256,4 +344,9 @@
/* center horizontally */
align-self: center;
}
.blockTitle {
display: flex;
align-items: center;
}
</style>

View File

@ -1,5 +1,5 @@
<script>
import { ModalContent, Icon, Detail, TextArea } from "@budibase/bbui"
import { ModalContent, Icon, Detail, TextArea, Label } from "@budibase/bbui"
export let testResult
export let isTrigger
@ -10,7 +10,7 @@
<ModalContent
showCloseIcon={false}
showConfirmButton={false}
title="Test Automation"
title="Test Results"
cancelText="Close"
>
<div slot="header">
@ -26,7 +26,18 @@
{/if}
</div>
</div>
<span>
{#if testResult[0].outputs.iterations}
<div style="display: flex;">
<Icon name="Reuse" />
<div style="margin-left: 10px;">
<Label>
This loop ran {testResult[0].outputs.iterations} times.</Label
>
</div>
</div>
{/if}
</span>
<div
on:click={() => {
inputToggled = !inputToggled

View File

@ -88,36 +88,65 @@
if (!block || !automation) {
return []
}
// Find previous steps to the selected one
let allSteps = [...automation.steps]
if (automation.trigger) {
allSteps = [automation.trigger, ...allSteps]
}
const blockIdx = allSteps.findIndex(step => step.id === block.id)
let blockIdx = allSteps.findIndex(step => step.id === block.id)
// Extract all outputs from all previous steps as available bindings
// Extract all outputs from all previous steps as available bindins
let bindings = []
for (let idx = 0; idx < blockIdx; idx++) {
const outputs = Object.entries(
allSteps[idx].schema?.outputs?.properties ?? {}
)
let wasLoopBlock = allSteps[idx]?.stepId === "LOOP"
let isLoopBlock =
allSteps[idx]?.stepId === "LOOP" &&
allSteps.find(x => x.blockToLoop === block.id)
// If the previous block was a loop block, decerement the index so the following
// steps are in the correct order
if (wasLoopBlock) {
blockIdx--
}
let schema = allSteps[idx]?.schema?.outputs?.properties ?? {}
// If its a Loop Block, we need to add this custom schema
if (isLoopBlock) {
schema = {
currentItem: {
type: "string",
description: "the item currently being executed",
},
}
}
const outputs = Object.entries(schema)
bindings = bindings.concat(
outputs.map(([name, value]) => {
const stepsLabel = block.name.startsWith("JS")
let runtimeName = isLoopBlock
? `loop.${name}`
: block.name.startsWith("JS")
? `steps[${idx}].${name}`
: `steps.${idx}.${name}`
const runtime = idx === 0 ? `trigger.${name}` : stepsLabel
const runtime = idx === 0 ? `trigger.${name}` : runtimeName
return {
label: runtime,
type: value.type,
description: value.description,
category: idx === 0 ? "Trigger outputs" : `Step ${idx} outputs`,
category:
idx === 0
? "Trigger outputs"
: isLoopBlock
? "Loop Outputs"
: `Step ${idx} outputs`,
path: runtime,
}
})
)
}
return bindings
}
@ -264,6 +293,14 @@
value={inputData[key]}
/>
</CodeEditorModal>
{:else if value.customType === "loopOption"}
<Select
on:change={e => onChange(e, key)}
autoWidth
value={inputData[key]}
options={["Array", "String"]}
defaultValue={"Array"}
/>
{:else if value.type === "string" || value.type === "number" || value.type === "integer"}
{#if isTestModal}
<ModalBindableInput

View File

@ -13,6 +13,7 @@ const integromat = require("./steps/integromat")
let filter = require("./steps/filter")
let delay = require("./steps/delay")
let queryRow = require("./steps/queryRows")
let loop = require("./steps/loop")
const env = require("../environment")
const ACTION_IMPLS = {
@ -27,6 +28,7 @@ const ACTION_IMPLS = {
DELAY: delay.run,
FILTER: filter.run,
QUERY_ROWS: queryRow.run,
LOOP: loop.run,
// these used to be lowercase step IDs, maintain for backwards compat
discord: discord.run,
slack: slack.run,
@ -45,6 +47,7 @@ const ACTION_DEFINITIONS = {
DELAY: delay.definition,
FILTER: filter.definition,
QUERY_ROWS: queryRow.definition,
LOOP: loop.definition,
// these used to be lowercase step IDs, maintain for backwards compat
discord: discord.definition,
slack: slack.definition,

View File

@ -1,4 +1,5 @@
const { getTable } = require("../api/controllers/table/utils")
const { findHBSBlocks } = require("@budibase/string-templates")
/**
* When values are input to the system generally they will be of type string as this is required for template strings.
@ -74,3 +75,14 @@ exports.getError = err => {
}
return typeof err !== "string" ? err.toString() : err
}
exports.substituteLoopStep = (hbsString, substitute) => {
let blocks = findHBSBlocks(hbsString)
for (let block of blocks) {
let oldBlock = block
block = block.replace(/loop/, substitute)
hbsString = hbsString.replace(new RegExp(oldBlock, "g"), block)
}
return hbsString
}

View File

@ -0,0 +1,50 @@
exports.definition = {
name: "Looping",
icon: "Reuse",
tagline: "Loop the block",
description: "Loop",
stepId: "LOOP",
internal: true,
inputs: {},
schema: {
inputs: {
properties: {
option: {
customType: "loopOption",
title: "Input type",
},
binding: {
type: "string",
title: "Binding / Value",
},
iterations: {
type: "number",
title: "Max loop iterations",
},
failure: {
type: "string",
title: "Failure Condition",
},
},
required: ["type", "value", "iterations", "failure"],
},
outputs: {
properties: {
items: {
customType: "item",
description: "The item currently being executed",
},
success: {
type: "boolean",
description: "Whether the message loop was successfully",
},
iterations: {
type: "number",
descriptions: "The amount of times the block ran",
},
},
required: ["success", "items", "iterations"],
},
},
type: "LOGIC",
}

View File

@ -190,5 +190,11 @@ exports.WebhookType = {
AUTOMATION: "automation",
}
exports.AutomationErrors = {
INCORRECT_TYPE: "INCORRECT_TYPE",
MAX_ITERATIONS: "MAX_ITERATIONS_REACHED",
FAILURE_CONDITION: "FAILURE_CONDITION_MET",
}
// pass through the list from the auth/core lib
exports.ObjectStoreBuckets = ObjectStoreBuckets

View File

@ -59,6 +59,7 @@ module.exports = {
LOG_LEVEL: process.env.LOG_LEVEL,
AUTOMATION_DIRECTORY: process.env.AUTOMATION_DIRECTORY,
AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET,
AUTOMATION_MAX_ITERATIONS: process.env.AUTOMATION_MAX_ITERATIONS,
SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,

View File

@ -8,10 +8,14 @@ const { DocumentTypes } = require("../db/utils")
const { doInTenant } = require("@budibase/backend-core/tenancy")
const { definitions: triggerDefs } = require("../automations/triggerInfo")
const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
const { AutomationErrors } = require("../constants")
const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
const CRON_STEP_ID = triggerDefs.CRON.stepId
const STOPPED_STATUS = { success: false, status: "STOPPED" }
const { cloneDeep } = require("lodash/fp")
const env = require("../environment")
/**
* The automation orchestrator is a class responsible for executing automations.
@ -74,34 +78,156 @@ class Orchestrator {
this.executionOutput.steps.push(stepObj)
}
updateContextAndOutput(loopStepNumber, step, output, result) {
this.executionOutput.steps.splice(loopStepNumber, 0, {
id: step.id,
stepId: step.stepId,
outputs: {
...output,
success: result.success,
status: result.status,
},
inputs: step.inputs,
})
this._context.steps.splice(loopStepNumber, 0, {
...output,
success: result.success,
status: result.status,
})
}
async execute() {
let automation = this._automation
const app = await this.getApp()
let stopped = false
let loopStep
let stepCount = 0
let loopStepNumber
let loopSteps = []
for (let step of automation.definition.steps) {
stepCount++
let input
if (step.stepId === LOOP_STEP_ID) {
loopStep = step
loopStepNumber = stepCount
continue
}
if (loopStep) {
input = await processObject(loopStep.inputs, this._context)
}
let iterations = loopStep ? input.binding.length : 1
let iterationCount = 0
for (let index = 0; index < iterations; index++) {
let originalStepInput = cloneDeep(step.inputs)
// Handle if the user has set a max iteration count or if it reaches the max limit set by us
if (loopStep) {
// lets first of all handle the input
// if the input is array then use it, if it is a string then split it on every new line
let newInput = await processObject(
loopStep.inputs,
cloneDeep(this._context)
)
newInput = automationUtils.cleanInputValues(
newInput,
loopStep.schema.inputs
)
this._context.steps[loopStepNumber] = {
currentItem: newInput.binding[index],
}
let tempOutput = { items: loopSteps, iterations: iterationCount }
if (
(loopStep.inputs.option === "Array" &&
!Array.isArray(newInput.binding)) ||
(loopStep.inputs.option === "String" &&
typeof newInput.binding !== "string")
) {
this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
status: AutomationErrors.INCORRECT_TYPE,
success: false,
})
loopSteps = null
loopStep = null
break
}
// The "Loop" binding in the front end is "fake", so replace it here so the context can understand it
// Pretty hacky because we need to account for the row object
for (let [key, value] of Object.entries(originalStepInput)) {
if (typeof value === "object") {
for (let [innerKey, innerValue] of Object.entries(
originalStepInput[key]
)) {
if (typeof innerValue === "string") {
originalStepInput[key][innerKey] =
automationUtils.substituteLoopStep(
innerValue,
`steps.${loopStepNumber}`
)
}
}
} else {
if (typeof value === "string") {
originalStepInput[key] = automationUtils.substituteLoopStep(
value,
`steps.${loopStepNumber}`
)
}
}
}
if (
index === parseInt(env.AUTOMATION_MAX_ITERATIONS) ||
index === loopStep.inputs.iterations
) {
this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
status: AutomationErrors.MAX_ITERATIONS,
success: true,
})
loopSteps = null
loopStep = null
break
}
if (
this._context.steps[loopStepNumber]?.currentItem ===
loopStep.inputs.failure
) {
this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
status: AutomationErrors.FAILURE_CONDITION,
success: false,
})
loopSteps = null
loopStep = null
break
}
}
// execution stopped, record state for that
if (stopped) {
this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
continue
}
// If it's a loop step, we need to manually add the bindings to the context
let stepFn = await this.getStepFunctionality(step.stepId)
step.inputs = await processObject(step.inputs, this._context)
step.inputs = automationUtils.cleanInputValues(
step.inputs,
step.schema.inputs
)
// appId is always passed
let inputs = await processObject(originalStepInput, this._context)
inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs)
try {
// appId is always passed
let tenantId = app.tenantId || DEFAULT_TENANT_ID
const outputs = await doInTenant(tenantId, () => {
return stepFn({
inputs: step.inputs,
inputs: inputs,
appId: this._appId,
emitter: this._emitter,
context: this._context,
})
})
this._context.steps.push(outputs)
this._context.steps[stepCount] = outputs
// if filter causes us to stop execution don't break the loop, set a var
// so that we can finish iterating through the steps and record that it stopped
if (step.stepId === FILTER_STEP_ID && !outputs.success) {
@ -112,12 +238,48 @@ class Orchestrator {
})
continue
}
this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs)
if (loopStep && loopSteps) {
loopSteps.push(outputs)
} else {
this.updateExecutionOutput(
step.id,
step.stepId,
step.inputs,
outputs
)
}
} catch (err) {
console.error(`Automation error - ${step.stepId} - ${err}`)
return err
}
if (loopStep) {
iterationCount++
if (index === iterations - 1) {
loopStep = null
this._context.steps.splice(loopStepNumber, 1)
break
}
}
}
if (loopSteps && loopSteps.length) {
let tempOutput = {
success: true,
items: loopSteps,
iterations: iterationCount,
}
this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
id: step.id,
stepId: step.stepId,
outputs: tempOutput,
inputs: step.inputs,
})
this._context.steps.splice(loopStepNumber, 0, tempOutput)
loopSteps = null
}
}
return this.executionOutput
}
}