Merge pull request #630 from Budibase/async-workflow-blocks

Async loading external automation steps

packages/builder/src/components/automation/AutomationBuilder/AutomationBuilder.svelte

@@ -61,7 +61,7 @@
 
   footer {
     position: absolute;
-    bottom: 20px;
+    bottom: var(--spacing-xl);
     right: 30px;
     display: flex;
     align-items: flex-end;
@@ -80,7 +80,7 @@
     justify-content: center;
   }
   footer > button:first-child {
-    margin-right: 20px;
+    margin-right: var(--spacing-m);
   }
 
   .play-button.highlighted {

packages/builder/src/components/automation/AutomationBuilder/flowchart/Arrow.svelte

@@ -10,6 +10,6 @@
 
 <style>
   svg {
-    margin: 8px 0;
+    margin: var(--spacing-m) 0;
   }
 </style>

packages/builder/src/components/automation/AutomationBuilder/flowchart/AutomationBlockTagline.svelte

@@ -29,10 +29,11 @@
       .replace(/}}/, "}}</b>")
 
     // Extract schema paths for any input bindings
-    const inputPaths = formattedTagline
+    let inputPaths = formattedTagline.match(/{{\s*\S+\s*}}/g) || []
-      .match(/{{\s*\S+\s*}}/g)
+    inputPaths = inputPaths.map(path => path.replace(/[{}]/g, "").trim())
-      .map(x => x.replace(/[{}]/g, "").trim())
+    const schemaPaths = inputPaths.map(path =>
-    const schemaPaths = inputPaths.map(x => x.replace(/\./g, ".properties."))
+      path.replace(/\./g, ".properties.")
+    )
 
     // Replace any enum bindings with their pretty equivalents
     schemaPaths.forEach((path, idx) => {

packages/builder/src/components/automation/AutomationBuilder/flowchart/FlowChart.svelte

@@ -37,7 +37,7 @@
 <style>
   section {
     position: absolute;
-    padding: 20px 40px;
+    padding: 40px;
     display: flex;
     align-items: center;
     flex-direction: column;

packages/builder/src/components/automation/AutomationBuilder/flowchart/FlowItem.svelte

@@ -7,7 +7,8 @@
   let selected
 
   $: selected = $automationStore.selectedBlock?.id === block.id
-  $: steps = $automationStore.selectedAutomation?.automation?.definition?.steps ?? []
+  $: steps =
+    $automationStore.selectedAutomation?.automation?.definition?.steps ?? []
   $: blockIdx = steps.findIndex(step => step.id === block.id)
 </script>
 

packages/builder/src/components/automation/AutomationPanel/AutomationList/AutomationList.svelte

@@ -46,22 +46,18 @@
     flex-direction: column;
   }
 
-  i {
-    color: #adaec4;
-  }
-
-  i:hover {
-    cursor: pointer;
-  }
-
   ul {
     list-style-type: none;
     padding: 0;
+    margin: var(--spacing-xl) 0 0 0;
     flex: 1;
   }
 
-  .live {
+  i {
-    color: var(--primary);
+    color: var(--grey-6);
+  }
+  i.live {
+    color: var(--purple);
   }
 
   li {
@@ -70,51 +66,23 @@
 
   .automation-item {
     display: flex;
-    border-radius: 5px;
+    flex-direction: row;
-    padding-left: 12px;
+    justify-content: flex-start;
     align-items: center;
-    height: 36px;
+    border-radius: var(--border-radius-m);
-    margin-bottom: 4px;
+    padding: var(--spacing-s) var(--spacing-m);
+    margin-bottom: var(--spacing-xs);
     color: var(--ink);
   }
-
   .automation-item i {
     font-size: 24px;
-    margin-right: 10px;
+    margin-right: var(--spacing-m);
   }
-
   .automation-item:hover {
     cursor: pointer;
     background: var(--grey-1);
   }
-
   .automation-item.selected {
     background: var(--grey-2);
   }
-
-  .new-automation-button {
-    cursor: pointer;
-    border: 1px solid var(--grey-4);
-    border-radius: 3px;
-    width: 100%;
-    padding: 8px 16px;
-    display: flex;
-    justify-content: center;
-    align-items: center;
-    background: white;
-    color: var(--ink);
-    font-size: 14px;
-    font-weight: 500;
-    transition: all 2ms;
-  }
-
-  .new-automation-button:hover {
-    background: var(--grey-1);
-  }
-
-  .icon {
-    color: var(--ink);
-    font-size: 16px;
-    margin-right: 4px;
-  }
 </style>

packages/builder/src/components/automation/AutomationPanel/AutomationList/CreateAutomationModal.svelte

@@ -22,67 +22,74 @@
   }
 </script>
 
-<header>
+<div class="container">
-  <i class="ri-stackshare-line" />
+  <header>
-  Create Automation
+    <i class="ri-stackshare-line" />
-</header>
+    Create Automation
-<div>
+  </header>
-  <Input bind:value={name} label="Name" />
+  <div class="content">
+    <Input bind:value={name} label="Name" />
+  </div>
+  <footer>
+    <a href="https://docs.budibase.com">
+      <i class="ri-information-line" />
+      <span>Learn about automations</span>
+    </a>
+    <ActionButton secondary on:click={onClosed}>Cancel</ActionButton>
+    <ActionButton disabled={!valid} on:click={createAutomation}>
+      Save
+    </ActionButton>
+  </footer>
 </div>
-<footer>
-  <a href="https://docs.budibase.com">
-    <i class="ri-information-line" />
-    Learn about automations
-  </a>
-  <ActionButton secondary on:click={onClosed}>Cancel</ActionButton>
-  <ActionButton disabled={!valid} on:click={createAutomation}>Save</ActionButton>
-</footer>
 
 <style>
+  .container {
+    padding: var(--spacing-xl);
+  }
+
   header {
-    font-size: 24px;
+    font-size: var(--font-size-xl);
     color: var(--ink);
     font-weight: bold;
-    padding: 30px;
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
   }
-
   header i {
-    margin-right: 10px;
+    margin-right: var(--spacing-m);
     font-size: 20px;
-    background: var(--blue-light);
+    background: var(--purple);
-    color: var(--grey-4);
+    color: var(--white);
-    padding: 8px;
+    padding: var(--spacing-s);
+    border-radius: var(--border-radius-m);
+    display: inline-block;
   }
 
-  div {
+  .content {
-    padding: 0 30px 30px 30px;
+    padding: var(--spacing-xl) 0;
-  }
-
-  label {
-    font-size: 18px;
-    font-weight: 500;
   }
 
   footer {
     display: grid;
     grid-auto-flow: column;
-    grid-gap: 5px;
+    grid-gap: var(--spacing-m);
     grid-auto-columns: 3fr 1fr 1fr;
-    padding: 20px;
-    background: var(--grey-1);
-    border-radius: 0.5rem;
   }
-
   footer a {
-    color: var(--primary);
+    color: var(--ink);
     font-size: 14px;
     vertical-align: middle;
     display: flex;
     align-items: center;
+    text-decoration: none;
+  }
+  footer a span {
+    text-decoration: underline;
   }
-
   footer i {
     font-size: 20px;
-    margin-right: 10px;
+    margin-right: var(--spacing-m);
+    text-decoration: none;
   }
 </style>

packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte

@@ -20,7 +20,7 @@
       class="hoverable"
       class:selected={selectedTab === 'ADD'}
       on:click={() => (selectedTab = 'ADD')}>
-      Add step
+      Steps
     </span>
   {/if}
 </header>
@@ -37,11 +37,11 @@
     background: none;
     display: flex;
     align-items: center;
-    margin-bottom: 20px;
+    margin-bottom: var(--spacing-xl);
   }
 
   .automation-header {
-    margin-right: 20px;
+    margin-right: var(--spacing-xl);
   }
 
   span:not(.selected) {

packages/builder/src/components/automation/AutomationPanel/BlockList/AutomationBlock.svelte

@@ -8,7 +8,7 @@
   function addBlockToAutomation() {
     automationStore.actions.addBlockToAutomation({
       ...blockDefinition,
-      args: blockDefinition.args || {},
+      inputs: blockDefinition.inputs || {},
       stepId,
       type: blockType,
     })
@@ -33,26 +33,15 @@
     display: grid;
     grid-template-columns: 20px auto;
     align-items: center;
-    margin-top: 16px;
+    margin-top: var(--spacing-s);
-    padding: 12px;
+    padding: var(--spacing-m);
     border-radius: var(--border-radius-m);
   }
-
   .automation-block:hover {
     background-color: var(--grey-1);
   }
-
+  .automation-block:first-child {
-  .automation-text {
+    margin-top: 0;
-    margin-left: 16px;
-  }
-
-  .icon {
-    height: 40px;
-    width: 40px;
-    background: var(--blue-light);
-    display: flex;
-    align-items: center;
-    justify-content: center;
   }
 
   i {
@@ -60,14 +49,16 @@
     font-size: 20px;
   }
 
-  h4 {
+  .automation-text {
+    margin-left: 16px;
+  }
+  .automation-text h4 {
     font-size: 14px;
     font-weight: 500;
     margin-bottom: 5px;
     margin-top: 0;
   }
-
+  .automation-text p {
-  p {
     font-size: 12px;
     color: var(--grey-7);
     margin: 0;

packages/builder/src/components/automation/AutomationPanel/BlockList/BlockList.svelte

@@ -1,11 +1,14 @@
 <script>
+  import { sortBy } from "lodash/fp"
   import { automationStore } from "builderStore"
   import AutomationBlock from "./AutomationBlock.svelte"
   import FlatButtonGroup from "components/userInterface/FlatButtonGroup.svelte"
 
   let selectedTab = "TRIGGER"
   let buttonProps = []
-  $: blocks = Object.entries($automationStore.blockDefinitions[selectedTab])
+  $: blocks = sortBy(entry => entry[1].name)(
+    Object.entries($automationStore.blockDefinitions[selectedTab])
+  )
 
   $: {
     if ($automationStore.selectedAutomation.hasTrigger()) {
@@ -37,3 +40,9 @@
     {/each}
   </div>
 </section>
+
+<style>
+  #blocklist {
+    margin-top: var(--spacing-xl);
+  }
+</style>

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -66,7 +66,7 @@
         <RecordSelector bind:value={block.inputs[key]} {bindings} />
       {:else if value.type === 'string' || value.type === 'number'}
         <BindableInput
-          type={value.type}
+          type="string"
           thin
           bind:value={block.inputs[key]}
           {bindings} />

packages/builder/src/components/automation/SetupPanel/DeleteAutomationModal.svelte

@@ -26,7 +26,8 @@
 </header>
 <div>
   <p>
-    Are you sure you want to delete this automation? This action can't be undone.
+    Are you sure you want to delete this automation? This action can't be
+    undone.
   </p>
 </div>
 <footer>

packages/builder/src/components/automation/SetupPanel/ParamInputs/RecordSelector.svelte

@@ -44,7 +44,7 @@
             thin
             bind:value={value[field]}
             label={field}
-            type={schema.type}
+            type="string"
             {bindings} />
         {/if}
       </div>

packages/builder/src/components/automation/SetupPanel/SetupPanel.svelte

@@ -24,7 +24,9 @@
   }
 
   function deleteAutomationBlock() {
-    automationStore.actions.deleteAutomationBlock($automationStore.selectedBlock)
+    automationStore.actions.deleteAutomationBlock(
+      $automationStore.selectedBlock
+    )
   }
 
   async function testAutomation() {
@@ -56,10 +58,24 @@
       Setup
     </span>
   </header>
-  {#if $automationStore.selectedBlock}
+  <div class="content">
-    <AutomationBlockSetup bind:block={$automationStore.selectedBlock} />
+    {#if $automationStore.selectedBlock}
-    <div class="buttons">
+      <AutomationBlockSetup bind:block={$automationStore.selectedBlock} />
-      <Button green wide data-cy="save-automation-setup" on:click={saveAutomation}>
+    {:else if $automationStore.selectedAutomation}
+      <div class="block-label">
+        Automation
+        <b>{automation.name}</b>
+      </div>
+      <Button secondary wide on:click={testAutomation}>Test Automation</Button>
+    {/if}
+  </div>
+  <div class="buttons">
+    {#if $automationStore.selectedBlock}
+      <Button
+        green
+        wide
+        data-cy="save-automation-setup"
+        on:click={saveAutomation}>
         Save Automation
       </Button>
       <Button
@@ -67,30 +83,20 @@
         red
         wide
         on:click={deleteAutomationBlock}>
-        Delete Block
+        Delete Step
       </Button>
-    </div>
+    {:else if $automationStore.selectedAutomation}
-  {:else if $automationStore.selectedAutomation}
+      <Button
-    <div class="panel">
+        green
-      <div class="panel-body">
+        wide
-        <div class="block-label">
+        data-cy="save-automation-setup"
-          Automation
+        on:click={saveAutomation}>
-          <b>{automation.name}</b>
+        Save Automation
-        </div>
+      </Button>
-      </div>
+      <Button red wide on:click={deleteAutomation}>Delete Automation</Button>
-      <Button secondary wide on:click={testAutomation}>Test Automation</Button>
+    {/if}
-      <div class="buttons">
+  </div>
-        <Button
+
-          green
-          wide
-          data-cy="save-automation-setup"
-          on:click={saveAutomation}>
-          Save Automation
-        </Button>
-        <Button red wide on:click={deleteAutomation}>Delete Automation</Button>
-      </div>
-    </div>
-  {/if}
 </section>
 
 <style>
@@ -98,29 +104,24 @@
     display: flex;
     flex-direction: column;
     height: 100%;
-    justify-content: space-between;
+    justify-content: flex-start;
-  }
+    align-items: stretch;
-
-  .panel-body {
-    flex: 1;
-  }
-
-  .panel {
-    display: flex;
-    flex-direction: column;
-    justify-content: space-between;
   }
 
   header {
     font-size: 18px;
     font-weight: 600;
-    font-family: inter;
+    font-family: inter, sans-serif;
     display: flex;
     align-items: center;
-    margin-bottom: 20px;
+    margin-bottom: var(--spacing-xl);
     color: var(--ink);
   }
-
+  header > span {
+    color: var(--grey-5);
+    margin-right: var(--spacing-xl);
+    cursor: pointer;
+  }
   .selected {
     color: var(--ink);
   }
@@ -129,31 +130,15 @@
     font-weight: 500;
     font-size: 14px;
     color: var(--grey-7);
-    margin-bottom: 20px;
+    margin-bottom: var(--spacing-xl);
   }
 
-  header > span {
+  .content {
-    color: var(--grey-5);
+    flex: 1 0 auto;
-    margin-right: 20px;
-    cursor: pointer;
-  }
-
-  label {
-    font-weight: 500;
-    font-size: 14px;
-    color: var(--ink);
   }
 
   .buttons {
-    position: absolute;
-    bottom: 20px;
     display: grid;
-    width: 260px;
+    gap: var(--spacing-m);
-    gap: 12px;
-  }
-
-  .access-level label {
-    font-weight: normal;
-    color: var(--ink);
   }
 </style>

packages/builder/src/pages/[application]/automate/_layout.svelte

@@ -1,23 +1,19 @@
-<!-- routify:options index=3 -->
 <script>
   import { automationStore } from "builderStore"
   import { AutomationPanel, SetupPanel } from "components/automation"
 </script>
 
+<!-- routify:options index=3 -->
 <div class="root">
   <div class="nav">
-    <div class="inner">
+    <AutomationPanel />
-      <AutomationPanel />
-    </div>
   </div>
   <div class="content">
     <slot />
   </div>
   {#if $automationStore.selectedAutomation}
     <div class="nav">
-      <div class="inner">
+      <SetupPanel />
-        <SetupPanel />
-      </div>
     </div>
   {/if}
 </div>
@@ -25,28 +21,19 @@
 <style>
   .content {
     position: relative;
-    background: var(--grey-1);
   }
 
   .root {
-    height: 100%;
+    height: calc(100% - 60px);
     display: grid;
-    grid-template-columns: 300px minmax(0, 1fr) 300px;
+    grid-template-columns: 300px minmax(510px, 1fr) 300px;
     background: var(--grey-1);
     line-height: 1;
   }
 
-  .content {
-    flex: 1 1 auto;
-  }
-
   .nav {
-    overflow: auto;
+    overflow-y: auto;
-    width: 300px;
     background: var(--white);
-  }
+    padding: var(--spacing-xl);
-
-  .inner {
-    padding: 20px;
   }
 </style>

packages/builder/src/pages/[application]/backend/_layout.svelte

@@ -1,4 +1,3 @@
-<!-- routify:options index=1 -->
 <script>
   import { getContext } from "svelte"
   import { store, backendUiStore } from "builderStore"
@@ -6,6 +5,7 @@
   import ModelNavigator from "components/nav/ModelNavigator/ModelNavigator.svelte"
 </script>
 
+<!-- routify:options index=1 -->
 <div class="root">
   <div class="nav">
     <ModelNavigator />

packages/builder/src/pages/[application]/frontend/_layout.svelte

@@ -1,4 +1,3 @@
-<!-- routify:options index=1 -->
 <script>
   import { store, backendUiStore } from "builderStore"
   import { goto } from "@sveltech/routify"
@@ -38,6 +37,7 @@
   const lastPartOfName = c => (c ? last(c.split("/")) : "")
 </script>
 
+<!-- routify:options index=1 -->
 <div class="root">
 
   <div class="ui-nav">

packages/server/package.json

@@ -50,6 +50,7 @@
     "bcryptjs": "^2.4.3",
     "chmodr": "^1.2.0",
     "dotenv": "^8.2.0",
+    "download": "^8.0.0",
     "electron-is-dev": "^1.2.0",
     "electron-unhandled": "^3.0.2",
     "electron-updater": "^4.3.1",

packages/server/src/api/controllers/automation.js

@@ -90,7 +90,7 @@ exports.destroy = async function(ctx) {
 }
 
 exports.getActionList = async function(ctx) {
-  ctx.body = actions.BUILTIN_DEFINITIONS
+  ctx.body = actions.DEFINITIONS
 }
 
 exports.getTriggerList = async function(ctx) {
@@ -105,7 +105,7 @@ module.exports.getDefinitionList = async function(ctx) {
   ctx.body = {
     logic: logic.BUILTIN_DEFINITIONS,
     trigger: triggers.BUILTIN_DEFINITIONS,
-    action: actions.BUILTIN_DEFINITIONS,
+    action: actions.DEFINITIONS,
   }
 }
 

packages/server/src/api/controllers/record.js

@@ -3,18 +3,18 @@ const validateJs = require("validate.js")
 const newid = require("../../db/newid")
 
 function emitEvent(eventType, ctx, record) {
+  let event = {
+    record,
+    instanceId: ctx.user.instanceId,
+  }
   // add syntactic sugar for mustache later
   if (record._id) {
-    record.id = record._id
+    event.id = record._id
   }
   if (record._rev) {
-    record.revision = record._rev
+    event.revision = record._rev
   }
-  ctx.eventEmitter &&
+  ctx.eventEmitter && ctx.eventEmitter.emit(eventType, event)
-    ctx.eventEmitter.emit(eventType, {
-      record,
-      instanceId: ctx.user.instanceId,
-    })
 }
 
 validateJs.extend(validateJs.validators.datetime, {

packages/server/src/api/routes/tests/automation.spec.js

@@ -66,6 +66,15 @@ describe("/automations", () => {
     automation = { ...automation, ...TEST_AUTOMATION }
   }
 
+  const triggerWorkflow = async (automationId) => {
+    return await request
+      .post(`/api/automations/${automationId}/trigger`)
+      .send({ name: "Test", description: "TEST" })
+      .set(defaultHeaders(app._id, instance._id))
+      .expect('Content-Type', /json/)
+      .expect(200)
+  }
+
   describe("get definitions", () => {
     it("returns a list of definitions for actions", async () => {
       const res = await request
@@ -160,16 +169,15 @@ describe("/automations", () => {
       TEST_AUTOMATION.definition.trigger.inputs.modelId = model._id
       TEST_AUTOMATION.definition.steps[0].inputs.record.modelId = model._id
       await createAutomation()
-      const res = await request
+      // this looks a bit mad but we don't actually have a way to wait for a response from the automation to
-        .post(`/api/automations/${automation._id}/trigger`)
+      // know that it has finished all of its actions - this is currently the best way
-        .send({ name: "Test" })
+      // also when this runs in CI it is very temper-mental so for now trying to make run stable by repeating until it works
-        .set(defaultHeaders(app._id, instance._id))
+      // TODO: update when workflow logs are a thing
-        .expect('Content-Type', /json/)
-        .expect(200)
-      expect(res.body.message).toEqual(`Automation ${automation._id} has been triggered.`)
-      expect(res.body.automation.name).toEqual(TEST_AUTOMATION.name)
-      // wait for automation to complete in background
       for (let tries = 0; tries < MAX_RETRIES; tries++) {
+        const res = await triggerWorkflow(automation._id)
+        expect(res.body.message).toEqual(`Automation ${automation._id} has been triggered.`)
+        expect(res.body.automation.name).toEqual(TEST_AUTOMATION.name)
+        await delay(500)
         let elements = await getAllFromModel(request, app._id, instance._id, model._id)
         // don't test it unless there are values to test
         if (elements.length === 1) {
@@ -178,7 +186,6 @@ describe("/automations", () => {
           expect(elements[0].description).toEqual("TEST")
           return
         }
-        await delay(500)
       }
       throw "Failed to find the records"
     })

packages/server/src/automations/actions.js

@@ -1,27 +1,91 @@
 const sendEmail = require("./steps/sendEmail")
 const saveRecord = require("./steps/saveRecord")
+const updateRecord = require("./steps/updateRecord")
 const deleteRecord = require("./steps/deleteRecord")
 const createUser = require("./steps/createUser")
+const environment = require("../environment")
+const download = require("download")
+const fetch = require("node-fetch")
+const path = require("path")
+const os = require("os")
+const fs = require("fs")
+const Sentry = require("@sentry/node")
 
+const DEFAULT_BUCKET =
+  "https://prod-budi-automations.s3-eu-west-1.amazonaws.com"
+const DEFAULT_DIRECTORY = ".budibase-automations"
+const AUTOMATION_MANIFEST = "manifest.json"
 const BUILTIN_ACTIONS = {
   SEND_EMAIL: sendEmail.run,
   SAVE_RECORD: saveRecord.run,
+  UPDATE_RECORD: updateRecord.run,
   DELETE_RECORD: deleteRecord.run,
   CREATE_USER: createUser.run,
 }
-
 const BUILTIN_DEFINITIONS = {
   SEND_EMAIL: sendEmail.definition,
   SAVE_RECORD: saveRecord.definition,
+  UPDATE_RECORD: updateRecord.definition,
   DELETE_RECORD: deleteRecord.definition,
   CREATE_USER: createUser.definition,
 }
 
+let AUTOMATION_BUCKET = environment.AUTOMATION_BUCKET
+let AUTOMATION_DIRECTORY = environment.AUTOMATION_DIRECTORY
+let MANIFEST = null
+
+function buildBundleName(pkgName, version) {
+  return `${pkgName}@${version}.min.js`
+}
+
+async function downloadPackage(name, version, bundleName) {
+  await download(
+    `${AUTOMATION_BUCKET}/${name}/${version}/${bundleName}`,
+    AUTOMATION_DIRECTORY
+  )
+  return require(path.join(AUTOMATION_DIRECTORY, bundleName))
+}
+
 module.exports.getAction = async function(actionName) {
   if (BUILTIN_ACTIONS[actionName] != null) {
     return BUILTIN_ACTIONS[actionName]
   }
-  // TODO: load async actions here
+  // env setup to get async packages
+  if (!MANIFEST || !MANIFEST.packages || !MANIFEST.packages[actionName]) {
+    return null
+  }
+  const pkg = MANIFEST.packages[actionName]
+  const bundleName = buildBundleName(pkg.stepId, pkg.version)
+  try {
+    return require(path.join(AUTOMATION_DIRECTORY, bundleName))
+  } catch (err) {
+    return downloadPackage(pkg.stepId, pkg.version, bundleName)
+  }
 }
 
+module.exports.init = async function() {
+  // set defaults
+  if (!AUTOMATION_DIRECTORY) {
+    AUTOMATION_DIRECTORY = path.join(os.homedir(), DEFAULT_DIRECTORY)
+  }
+  if (!AUTOMATION_BUCKET) {
+    AUTOMATION_BUCKET = DEFAULT_BUCKET
+  }
+  if (!fs.existsSync(AUTOMATION_DIRECTORY)) {
+    fs.mkdirSync(AUTOMATION_DIRECTORY, { recursive: true })
+  }
+  // env setup to get async packages
+  try {
+    let response = await fetch(`${AUTOMATION_BUCKET}/${AUTOMATION_MANIFEST}`)
+    MANIFEST = await response.json()
+    module.exports.DEFINITIONS =
+      MANIFEST && MANIFEST.packages
+        ? Object.assign(MANIFEST.packages, BUILTIN_DEFINITIONS)
+        : BUILTIN_DEFINITIONS
+  } catch (err) {
+    Sentry.captureException(err)
+  }
+}
+
+module.exports.DEFINITIONS = BUILTIN_DEFINITIONS
 module.exports.BUILTIN_DEFINITIONS = BUILTIN_DEFINITIONS

packages/server/src/automations/automationUtils.js

@@ -0,0 +1,116 @@
+const CouchDB = require("../db")
+
+/**
+ * When running mustache statements to execute on the context of the automation it possible user's may input mustache
+ * in a few different forms, some of which are invalid but are logically valid. An example of this would be the mustache
+ * statement "{{steps[0].revision}}" here it is obvious the user is attempting to access an array or object using array
+ * like operators. These are not supported by Mustache and therefore the statement will fail. This function will clean up
+ * the mustache statement so it instead reads as "{{steps.0.revision}}" which is valid and will work. It may also be expanded
+ * to include any other mustache statement cleanup that has been deemed necessary for the system.
+ *
+ * @param {string} string The string which *may* contain mustache statements, it is OK if it does not contain any.
+ * @returns {string} The string that was input with cleaned up mustache statements as required.
+ */
+module.exports.cleanMustache = string => {
+  let charToReplace = {
+    "[": ".",
+    "]": "",
+  }
+  let regex = new RegExp(/{{[^}}]*}}/g)
+  let matches = string.match(regex)
+  if (matches == null) {
+    return string
+  }
+  for (let match of matches) {
+    let baseIdx = string.indexOf(match)
+    for (let key of Object.keys(charToReplace)) {
+      let idxChar = match.indexOf(key)
+      if (idxChar !== -1) {
+        string =
+          string.slice(baseIdx, baseIdx + idxChar) +
+          charToReplace[key] +
+          string.slice(baseIdx + idxChar + 1)
+      }
+    }
+  }
+  return string
+}
+
+/**
+ * When values are input to the system generally they will be of type string as this is required for mustache. This can
+ * generate some odd scenarios as the Schema of the automation requires a number but the builder might supply a string
+ * with mustache syntax to get the number from the rest of the context. To support this the server has to make sure that
+ * the post mustache statement can be cast into the correct type, this function does this for numbers and booleans.
+ *
+ * @param {object} inputs An object of inputs, please note this will not recurse down into any objects within, it simply
+ * cleanses the top level inputs, however it can be used by recursively calling it deeper into the object structures if
+ * the schema is known.
+ * @param {object} schema The defined schema of the inputs, in the form of JSON schema. The schema definition of an
+ * automation is the likely use case of this, however validate.js syntax can be converted closely enough to use this by
+ * wrapping the schema properties in a top level "properties" object.
+ * @returns {object} The inputs object which has had all the various types supported by this function converted to their
+ * primitive types.
+ */
+module.exports.cleanInputValues = (inputs, schema) => {
+  if (schema == null) {
+    return inputs
+  }
+  for (let inputKey of Object.keys(inputs)) {
+    let input = inputs[inputKey]
+    if (typeof input !== "string") {
+      continue
+    }
+    let propSchema = schema.properties[inputKey]
+    if (!propSchema) {
+      continue
+    }
+    if (propSchema.type === "boolean") {
+      let lcInput = input.toLowerCase()
+      if (lcInput === "true") {
+        inputs[inputKey] = true
+      }
+      if (lcInput === "false") {
+        inputs[inputKey] = false
+      }
+    }
+    if (propSchema.type === "number") {
+      let floatInput = parseFloat(input)
+      if (!isNaN(floatInput)) {
+        inputs[inputKey] = floatInput
+      }
+    }
+  }
+  return inputs
+}
+
+/**
+ * Given a record input like a save or update record we need to clean the inputs against a schema that is not part of
+ * the automation but is instead part of the Table/Model. This function will get the model schema and use it to instead
+ * perform the cleanInputValues function on the input record.
+ *
+ * @param {string} instanceId The instance which the Table/Model is contained under.
+ * @param {string} modelId The ID of the Table/Model which the schema is to be retrieved for.
+ * @param {object} record The input record structure which requires clean-up after having been through mustache statements.
+ * @returns {Promise<Object>} The cleaned up records object, will should now have all the required primitive types.
+ */
+module.exports.cleanUpRecord = async (instanceId, modelId, record) => {
+  const db = new CouchDB(instanceId)
+  const model = await db.get(modelId)
+
+  return module.exports.cleanInputValues(record, { properties: model.schema })
+}
+
+/**
+ * A utility function for the cleanUpRecord, which can be used if only the record ID is known (not the model ID) to clean
+ * up a record after mustache statements have been replaced. This is specifically useful for the update record action.
+ *
+ * @param {string} instanceId The instance which the Table/Model is contained under.
+ * @param {string} recordId The ID of the record from which the modelId will be extracted, to get the Table/Model schema.
+ * @param {object} record The input record structure which requires clean-up after having been through mustache statements.
+ * @returns {Promise<Object>} The cleaned up records object, which will now have all the required primitive types.
+ */
+module.exports.cleanUpRecordById = async (instanceId, recordId, record) => {
+  const db = new CouchDB(instanceId)
+  const foundRecord = await db.get(recordId)
+  return module.exports.cleanUpRecord(instanceId, foundRecord.modelId, record)
+}

packages/server/src/automations/index.js

@@ -1,4 +1,5 @@
 const triggers = require("./triggers")
+const actions = require("./actions")
 const environment = require("../environment")
 const workerFarm = require("worker-farm")
 const singleThread = require("./thread")
@@ -21,11 +22,13 @@ function runWorker(job) {
  * This module is built purely to kick off the worker farm and manage the inputs/outputs
  */
 module.exports.init = function() {
-  triggers.automationQueue.process(async job => {
+  actions.init().then(() => {
-    if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
+    triggers.automationQueue.process(async job => {
-      await runWorker(job)
+      if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
-    } else {
+        await runWorker(job)
-      await singleThread(job)
+      } else {
-    }
+        await singleThread(job)
+      }
+    })
   })
 }

packages/server/src/automations/steps/saveRecord.js

@@ -1,4 +1,5 @@
 const recordController = require("../../api/controllers/record")
+const automationUtils = require("../automationUtils")
 
 module.exports.definition = {
   name: "Save Record",
@@ -60,6 +61,11 @@ module.exports.run = async function({ inputs, instanceId }) {
   if (inputs.record == null || inputs.record.modelId == null) {
     return
   }
+  inputs.record = await automationUtils.cleanUpRecord(
+    instanceId,
+    inputs.record.modelId,
+    inputs.record
+  )
   // have to clean up the record, remove the model from it
   const ctx = {
     params: {

packages/server/src/automations/steps/updateRecord.js

@@ -0,0 +1,100 @@
+const recordController = require("../../api/controllers/record")
+const automationUtils = require("../automationUtils")
+
+module.exports.definition = {
+  name: "Update Record",
+  tagline: "Update a {{inputs.enriched.model.name}} record",
+  icon: "ri-refresh-fill",
+  description: "Update a record to your database",
+  type: "ACTION",
+  stepId: "UPDATE_RECORD",
+  inputs: {},
+  schema: {
+    inputs: {
+      properties: {
+        record: {
+          type: "object",
+          customType: "record",
+          title: "Record",
+        },
+        recordId: {
+          type: "string",
+          title: "Record ID",
+        },
+      },
+      required: ["record", "recordId"],
+    },
+    outputs: {
+      properties: {
+        record: {
+          type: "object",
+          customType: "record",
+          description: "The updated record",
+        },
+        response: {
+          type: "object",
+          description: "The response from the table",
+        },
+        success: {
+          type: "boolean",
+          description: "Whether the action was successful",
+        },
+        id: {
+          type: "string",
+          description: "The identifier of the updated record",
+        },
+        revision: {
+          type: "string",
+          description: "The revision of the updated record",
+        },
+      },
+      required: ["success", "id", "revision"],
+    },
+  },
+}
+
+module.exports.run = async function({ inputs, instanceId }) {
+  if (inputs.recordId == null || inputs.record == null) {
+    return
+  }
+
+  inputs.record = await automationUtils.cleanUpRecordById(
+    instanceId,
+    inputs.recordId,
+    inputs.record
+  )
+  // clear any falsy properties so that they aren't updated
+  for (let propKey of Object.keys(inputs.record)) {
+    if (!inputs.record[propKey] || inputs.record[propKey] === "") {
+      delete inputs.record[propKey]
+    }
+  }
+
+  // have to clean up the record, remove the model from it
+  const ctx = {
+    params: {
+      id: inputs.recordId,
+    },
+    request: {
+      body: inputs.record,
+    },
+    user: { instanceId },
+  }
+
+  try {
+    await recordController.patch(ctx)
+    return {
+      record: ctx.body,
+      response: ctx.message,
+      id: ctx.body._id,
+      revision: ctx.body._rev,
+      success: ctx.status === 200,
+    }
+  } catch (err) {
+    console.error(err)
+    return {
+      success: false,
+      response: err,
+    }
+  }
+}

packages/server/src/automations/thread.js

@@ -1,39 +1,15 @@
 const mustache = require("mustache")
 const actions = require("./actions")
 const logic = require("./logic")
+const automationUtils = require("./automationUtils")
 
 const FILTER_STEP_ID = logic.BUILTIN_DEFINITIONS.FILTER.stepId
 
-function cleanMustache(string) {
-  let charToReplace = {
-    "[": ".",
-    "]": "",
-  }
-  let regex = new RegExp(/{{[^}}]*}}/g)
-  let matches = string.match(regex)
-  if (matches == null) {
-    return string
-  }
-  for (let match of matches) {
-    let baseIdx = string.indexOf(match)
-    for (let key of Object.keys(charToReplace)) {
-      let idxChar = match.indexOf(key)
-      if (idxChar !== -1) {
-        string =
-          string.slice(baseIdx, baseIdx + idxChar) +
-          charToReplace[key] +
-          string.slice(baseIdx + idxChar + 1)
-      }
-    }
-  }
-  return string
-}
-
 function recurseMustache(inputs, context) {
   for (let key of Object.keys(inputs)) {
     let val = inputs[key]
     if (typeof val === "string") {
-      val = cleanMustache(inputs[key])
+      val = automationUtils.cleanMustache(inputs[key])
       inputs[key] = mustache.render(val, context)
     }
     // this covers objects and arrays
@@ -77,15 +53,23 @@ class Orchestrator {
     for (let step of automation.definition.steps) {
       let stepFn = await this.getStepFunctionality(step.type, step.stepId)
       step.inputs = recurseMustache(step.inputs, this._context)
+      step.inputs = automationUtils.cleanInputValues(
+        step.inputs,
+        step.schema.inputs
+      )
       // instanceId is always passed
-      const outputs = await stepFn({
+      try {
-        inputs: step.inputs,
+        const outputs = await stepFn({
-        instanceId: this._instanceId,
+          inputs: step.inputs,
-      })
+          instanceId: this._instanceId,
-      if (step.stepId === FILTER_STEP_ID && !outputs.success) {
+        })
-        break
+        if (step.stepId === FILTER_STEP_ID && !outputs.success) {
+          break
+        }
+        this._context.steps.push(outputs)
+      } catch (err) {
+        console.error(`Automation error - ${step.stepId} - ${err}`)
       }
-      this._context.steps.push(outputs)
     }
   }
 }

packages/server/src/automations/triggers.js

@@ -36,8 +36,16 @@ const BUILTIN_DEFINITIONS = {
             customType: "record",
             description: "The new record that was saved",
           },
+          id: {
+            type: "string",
+            description: "Record ID - can be used for updating",
+          },
+          revision: {
+            type: "string",
+            description: "Revision of record",
+          },
         },
-        required: ["record"],
+        required: ["record", "id"],
       },
     },
     type: "TRIGGER",
@@ -69,7 +77,7 @@ const BUILTIN_DEFINITIONS = {
             description: "The record that was deleted",
           },
         },
-        required: ["record"],
+        required: ["record", "id"],
       },
     },
     type: "TRIGGER",
@@ -124,6 +132,7 @@ async function fillRecordOutput(automation, params) {
   const db = new CouchDB(params.instanceId)
   try {
     let model = await db.get(modelId)
+    let record = {}
     for (let schemaKey of Object.keys(model.schema)) {
       if (params[schemaKey] != null) {
         continue
@@ -131,19 +140,20 @@ async function fillRecordOutput(automation, params) {
       let propSchema = model.schema[schemaKey]
       switch (propSchema.constraints.type) {
         case "string":
-          params[schemaKey] = FAKE_STRING
+          record[schemaKey] = FAKE_STRING
           break
         case "boolean":
-          params[schemaKey] = FAKE_BOOL
+          record[schemaKey] = FAKE_BOOL
           break
         case "number":
-          params[schemaKey] = FAKE_NUMBER
+          record[schemaKey] = FAKE_NUMBER
           break
         case "datetime":
-          params[schemaKey] = FAKE_DATETIME
+          record[schemaKey] = FAKE_DATETIME
           break
       }
     }
+    params.record = record
   } catch (err) {
     throw "Failed to find model for trigger"
   }

packages/server/src/environment.js

@@ -7,6 +7,8 @@ module.exports = {
   COUCH_DB_URL: process.env.COUCH_DB_URL,
   SALT_ROUNDS: process.env.SALT_ROUNDS,
   LOGGER: process.env.LOGGER,
+  AUTOMATION_DIRECTORY: process.env.AUTOMATION_DIRECTORY,
+  AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET,
   BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
   SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
 }