Merge branch 'master' into fix/budi-8010

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.19.3",
+  "version": "2.19.4",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

package.json

@@ -58,7 +58,7 @@
     "lint": "yarn run lint:eslint && yarn run lint:prettier",
     "lint:fix:eslint": "eslint --fix --max-warnings=0 packages qa-core",
     "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"",
-    "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
+    "lint:fix": "yarn run lint:fix:eslint && yarn run lint:fix:prettier",
     "build:specs": "lerna run --stream specs",
     "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild",

packages/account-portal

@@ -1 +1 @@
-Subproject commit 1ba8414bed14439512153cf851086146a80560f5
+Subproject commit 8c446c4ba385592127fa31755d3b64467b291882

packages/builder/src/builderStore/store/frontend.js

@@ -39,6 +39,7 @@ import { makePropSafe as safe } from "@budibase/string-templates"
 import { getComponentFieldOptions } from "helpers/formFields"
 import { createBuilderWebsocket } from "builderStore/websocket"
 import { BuilderSocketEvent } from "@budibase/shared-core"
+import componentTreeNodesStore from "stores/portal/componentTreeNodesStore"
 
 const INITIAL_FRONTEND_STATE = {
   initialised: false,
@@ -1053,6 +1054,7 @@ export const getFrontendStore = () => {
         const screen = get(selectedScreen)
         const parent = findComponentParent(screen.props, componentId)
         const index = parent?._children.findIndex(x => x._id === componentId)
+        const componentTreeNodes = get(componentTreeNodesStore)
 
         // Check for screen and navigation component edge cases
         const screenComponentId = `${screen._id}-screen`
@@ -1071,9 +1073,15 @@ export const getFrontendStore = () => {
         if (index > 0) {
           // If sibling before us accepts children, select a descendant
           const previousSibling = parent._children[index - 1]
-          if (previousSibling._children?.length) {
+          if (
+            previousSibling._children?.length &&
+            componentTreeNodes[`nodeOpen-${previousSibling._id}`]
+          ) {
             let target = previousSibling
-            while (target._children?.length) {
+            while (
+              target._children?.length &&
+              componentTreeNodes[`nodeOpen-${target._id}`]
+            ) {
               target = target._children[target._children.length - 1]
             }
             return target._id
@@ -1093,6 +1101,7 @@ export const getFrontendStore = () => {
         const screen = get(selectedScreen)
         const parent = findComponentParent(screen.props, componentId)
         const index = parent?._children.findIndex(x => x._id === componentId)
+        const componentTreeNodes = get(componentTreeNodesStore)
 
         // Check for screen and navigation component edge cases
         const screenComponentId = `${screen._id}-screen`
@@ -1102,7 +1111,11 @@ export const getFrontendStore = () => {
         }
 
         // If we have children, select first child
-        if (component._children?.length) {
+        if (
+          component._children?.length &&
+          (state.selectedComponentId === navComponentId ||
+            componentTreeNodes[`nodeOpen-${component._id}`])
+        ) {
           return component._children[0]._id
         } else if (!parent) {
           return null

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte

@@ -128,10 +128,10 @@
         >
           <div class="item-body">
             <img
-              width="20"
+              width={20}
-              height="20"
+              height={20}
               src={externalActions[action.stepId].icon}
-              alt="zapier"
+              alt={externalActions[action.stepId].name}
             />
             <span class="icon-spacing">
               <Body size="XS">

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ExternalActions.js

@@ -1,5 +1,6 @@
 import DiscordLogo from "assets/discord.svg"
 import ZapierLogo from "assets/zapier.png"
+import n8nLogo from "assets/n8n_square.png"
 import MakeLogo from "assets/make.svg"
 import SlackLogo from "assets/slack.svg"
 
@@ -8,4 +9,5 @@ export const externalActions = {
   discord: { name: "discord", icon: DiscordLogo },
   slack: { name: "slack", icon: SlackLogo },
   integromat: { name: "integromat", icon: MakeLogo },
+  n8n: { name: "n8n", icon: n8nLogo },
 }

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -79,6 +79,7 @@
     disableWrapping: true,
   })
   $: editingJs = codeMode === EditorModes.JS
+  $: requiredProperties = block.schema.inputs.required || []
 
   $: stepCompletions =
     codeMode === EditorModes.Handlebars
@@ -359,6 +360,11 @@
     )
   }
 
+  function getFieldLabel(key, value) {
+    const requiredSuffix = requiredProperties.includes(key) ? "*" : ""
+    return `${value.title || (key === "row" ? "Table" : key)} ${requiredSuffix}`
+  }
+
   onMount(async () => {
     try {
       await environment.loadVariables()
@@ -376,7 +382,7 @@
           <Label
             tooltip={value.title === "Binding / Value"
               ? "If using the String input type, please use a comma or newline separated string"
-              : null}>{value.title || (key === "row" ? "Table" : key)}</Label
+              : null}>{getFieldLabel(key, value)}</Label
           >
         {/if}
         <div class:field-width={shouldRenderField(value)}>

packages/builder/src/constants/backend/automations.js

@@ -27,6 +27,7 @@ export const ActionStepID = {
   slack: "slack",
   zapier: "zapier",
   integromat: "integromat",
+  n8n: "n8n",
 }
 
 export const Features = {

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentDropdownMenu.svelte

@@ -3,6 +3,7 @@
   import { ActionMenu, MenuItem, Icon } from "@budibase/bbui"
 
   export let component
+  export let opened
 
   $: definition = componentStore.getDefinition(component?._component)
   $: noPaste = !$componentStore.componentToPaste
@@ -85,6 +86,39 @@
   >
     Paste
   </MenuItem>
+
+  {#if component?._children?.length}
+    <MenuItem
+      icon="TreeExpand"
+      keyBind="!ArrowRight"
+      on:click={() => keyboardEvent("ArrowRight", false)}
+      disabled={opened}
+    >
+      Expand
+    </MenuItem>
+    <MenuItem
+      icon="TreeCollapse"
+      keyBind="!ArrowLeft"
+      on:click={() => keyboardEvent("ArrowLeft", false)}
+      disabled={!opened}
+    >
+      Collapse
+    </MenuItem>
+    <MenuItem
+      icon="TreeExpandAll"
+      keyBind="Ctrl+!ArrowRight"
+      on:click={() => keyboardEvent("ArrowRight", true)}
+    >
+      Expand All
+    </MenuItem>
+    <MenuItem
+      icon="TreeCollapseAll"
+      keyBind="Ctrl+!ArrowLeft"
+      on:click={() => keyboardEvent("ArrowLeft", true)}
+    >
+      Collapse All
+    </MenuItem>
+  {/if}
 </ActionMenu>
 
 <style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentKeyHandler.svelte

@@ -9,6 +9,7 @@
   import { goto, isActive } from "@roxi/routify"
   import { notifications } from "@budibase/bbui"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+  import componentTreeNodesStore from "stores/portal/componentTreeNodesStore"
 
   let confirmDeleteDialog
   let confirmEjectDialog
@@ -61,6 +62,40 @@
     ["ArrowDown"]: () => {
       componentStore.selectNext()
     },
+    ["ArrowRight"]: component => {
+      componentTreeNodesStore.expandNode(component._id)
+    },
+    ["ArrowLeft"]: component => {
+      componentTreeNodesStore.collapseNode(component._id)
+    },
+    ["Ctrl+ArrowRight"]: component => {
+      componentTreeNodesStore.expandNode(component._id)
+
+      const expandChildren = component => {
+        const children = component._children ?? []
+
+        children.forEach(child => {
+          componentTreeNodesStore.expandNode(child._id)
+          expandChildren(child)
+        })
+      }
+
+      expandChildren(component)
+    },
+    ["Ctrl+ArrowLeft"]: component => {
+      componentTreeNodesStore.collapseNode(component._id)
+
+      const collapseChildren = component => {
+        const children = component._children ?? []
+
+        children.forEach(child => {
+          componentTreeNodesStore.collapseNode(child._id)
+          collapseChildren(child)
+        })
+      }
+
+      collapseChildren(component)
+    },
     ["Escape"]: () => {
       if ($isActive(`./:componentId/new`)) {
         $goto(`./${$componentStore.selectedComponentId}`)

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentTree.svelte

@@ -17,11 +17,12 @@
   } from "helpers/components"
   import { get } from "svelte/store"
   import { dndStore } from "./dndStore"
+  import componentTreeNodesStore from "stores/portal/componentTreeNodesStore"
 
   export let components = []
   export let level = 0
 
-  let closedNodes = {}
+  $: openNodes = $componentTreeNodesStore
 
   $: filteredComponents = components?.filter(component => {
     return (
@@ -54,15 +55,6 @@
     return componentSupportsChildren(component) && component._children?.length
   }
 
-  function toggleNodeOpen(componentId) {
-    if (closedNodes[componentId]) {
-      delete closedNodes[componentId]
-    } else {
-      closedNodes[componentId] = true
-    }
-    closedNodes = closedNodes
-  }
-
   const onDrop = async e => {
     e.stopPropagation()
     try {
@@ -72,14 +64,14 @@
     }
   }
 
-  const isOpen = (component, selectedComponentPath, closedNodes) => {
+  const isOpen = (component, selectedComponentPath, openNodes) => {
     if (!component?._children?.length) {
       return false
     }
-    if (selectedComponentPath.includes(component._id)) {
+    if (selectedComponentPath.slice(0, -1).includes(component._id)) {
       return true
     }
-    return !closedNodes[component._id]
+    return openNodes[`nodeOpen-${component._id}`]
   }
 
   const isChildOfSelectedComponent = component => {
@@ -96,7 +88,7 @@
 
 <ul>
   {#each filteredComponents || [] as component, index (component._id)}
-    {@const opened = isOpen(component, $selectedComponentPath, closedNodes)}
+    {@const opened = isOpen(component, $selectedComponentPath, openNodes)}
     <li
       on:click|stopPropagation={() => {
         componentStore.select(component._id)
@@ -110,7 +102,7 @@
         on:dragend={dndStore.actions.reset}
         on:dragstart={() => dndStore.actions.dragstart(component)}
         on:dragover={dragover(component, index)}
-        on:iconClick={() => toggleNodeOpen(component._id)}
+        on:iconClick={() => componentTreeNodesStore.toggleNode(component._id)}
         on:drop={onDrop}
         hovering={$hoverStore.componentId === component._id}
         on:mouseenter={() => hover(component._id)}
@@ -125,8 +117,9 @@
         highlighted={isChildOfSelectedComponent(component)}
         selectedBy={$userSelectedResourceMap[component._id]}
       >
-        <ComponentDropdownMenu {component} />
+        <ComponentDropdownMenu {opened} {component} />
       </NavItem>
+
       {#if opened}
         <svelte:self
           components={component._children}
@@ -144,13 +137,6 @@
     padding-left: 0;
     margin: 0;
   }
-  ul :global(.icon.arrow) {
-    transition: opacity 130ms ease-out;
-    opacity: 0;
-  }
-  ul:hover :global(.icon.arrow) {
-    opacity: 1;
-  }
   ul,
   li {
     min-width: max-content;

packages/builder/src/stores/builder/components.js

@@ -29,6 +29,7 @@ import {
 } from "constants/backend"
 import BudiStore from "./BudiStore"
 import { Utils } from "@budibase/frontend-core"
+import componentTreeNodesStore from "stores/portal/componentTreeNodesStore"
 
 export const INITIAL_COMPONENTS_STATE = {
   components: {},
@@ -662,6 +663,7 @@ export class ComponentStore extends BudiStore {
     const screen = get(selectedScreen)
     const parent = findComponentParent(screen.props, componentId)
     const index = parent?._children.findIndex(x => x._id === componentId)
+    const componentTreeNodes = get(componentTreeNodesStore)
 
     // Check for screen and navigation component edge cases
     const screenComponentId = `${screen._id}-screen`
@@ -680,9 +682,15 @@ export class ComponentStore extends BudiStore {
     if (index > 0) {
       // If sibling before us accepts children, select a descendant
       const previousSibling = parent._children[index - 1]
-      if (previousSibling._children?.length) {
+      if (
+        previousSibling._children?.length &&
+        componentTreeNodes[`nodeOpen-${previousSibling._id}`]
+      ) {
         let target = previousSibling
-        while (target._children?.length) {
+        while (
+          target._children?.length &&
+          componentTreeNodes[`nodeOpen-${target._id}`]
+        ) {
           target = target._children[target._children.length - 1]
         }
         return target._id
@@ -703,6 +711,7 @@ export class ComponentStore extends BudiStore {
     const screen = get(selectedScreen)
     const parent = findComponentParent(screen.props, componentId)
     const index = parent?._children.findIndex(x => x._id === componentId)
+    const componentTreeNodes = get(componentTreeNodesStore)
 
     // Check for screen and navigation component edge cases
     const screenComponentId = `${screen._id}-screen`
@@ -712,7 +721,11 @@ export class ComponentStore extends BudiStore {
     }
 
     // If we have children, select first child
-    if (component._children?.length) {
+    if (
+      component._children?.length &&
+      (state.selectedComponentId === navComponentId ||
+        componentTreeNodes[`nodeOpen-${component._id}`])
+    ) {
       return component._children[0]._id
     } else if (!parent) {
       return null

packages/builder/src/stores/portal/componentTreeNodesStore.js

@@ -0,0 +1,36 @@
+import { createSessionStorageStore } from "@budibase/frontend-core"
+
+const baseStore = createSessionStorageStore("openNodes", {})
+
+const toggleNode = componentId => {
+  baseStore.update(openNodes => {
+    openNodes[`nodeOpen-${componentId}`] = !openNodes[`nodeOpen-${componentId}`]
+
+    return openNodes
+  })
+}
+
+const expandNode = componentId => {
+  baseStore.update(openNodes => {
+    openNodes[`nodeOpen-${componentId}`] = true
+
+    return openNodes
+  })
+}
+
+const collapseNode = componentId => {
+  baseStore.update(openNodes => {
+    openNodes[`nodeOpen-${componentId}`] = false
+
+    return openNodes
+  })
+}
+
+const store = {
+  subscribe: baseStore.subscribe,
+  toggleNode,
+  expandNode,
+  collapseNode,
+}
+
+export default store

packages/client/manifest.json

@@ -270,6 +270,7 @@
           {
             "type": "buttonConfiguration",
             "key": "buttons",
+            "nested": true,
             "defaultValue": [
               {
                 "type": "cta",

packages/frontend-core/src/stores/index.js

@@ -1 +1,2 @@
 export { createLocalStorageStore } from "./localStorage"
+export { createSessionStorageStore } from "./sessionStorage"

packages/frontend-core/src/stores/sessionStorage.js

@@ -0,0 +1,46 @@
+import { get, writable } from "svelte/store"
+
+export const createSessionStorageStore = (sessionStorageKey, initialValue) => {
+  const store = writable(initialValue, () => {
+    // Hydrate from session storage when we get a new subscriber
+    hydrate()
+
+    // Listen for session storage changes and keep store in sync
+    const storageListener = ({ key }) => {
+      return key === sessionStorageKey && hydrate()
+    }
+
+    window.addEventListener("storage", storageListener)
+    return () => window.removeEventListener("storage", storageListener)
+  })
+
+  // New store setter which updates the store and sessionstorage
+  const set = value => {
+    store.set(value)
+    sessionStorage.setItem(sessionStorageKey, JSON.stringify(value))
+  }
+
+  // New store updater which updates the store and sessionstorage
+  const update = updaterFn => set(updaterFn(get(store)))
+
+  // Hydrates the store from sessionstorage
+  const hydrate = () => {
+    const sessionValue = sessionStorage.getItem(sessionStorageKey)
+    if (sessionValue == null) {
+      set(initialValue)
+    } else {
+      try {
+        store.set(JSON.parse(sessionValue))
+      } catch {
+        set(initialValue)
+      }
+    }
+  }
+
+  // Patch the default svelte store functions with our overrides
+  return {
+    ...store,
+    set,
+    update,
+  }
+}

packages/server/src/api/routes/tests/queries/mysql.spec.ts

@@ -0,0 +1,239 @@
+import { Datasource, Query } from "@budibase/types"
+import * as setup from "../utilities"
+import { databaseTestProviders } from "../../../../integrations/tests/utils"
+import mysql from "mysql2/promise"
+
+jest.unmock("mysql2")
+jest.unmock("mysql2/promise")
+
+const createTableSQL = `
+CREATE TABLE test_table (
+    id INT AUTO_INCREMENT PRIMARY KEY,
+    name VARCHAR(50) NOT NULL
+)
+`
+
+const insertSQL = `
+INSERT INTO test_table (name) VALUES ('one'), ('two'), ('three'), ('four'), ('five')
+`
+
+const dropTableSQL = `
+DROP TABLE test_table
+`
+
+describe("/queries", () => {
+  let config = setup.getConfig()
+  let datasource: Datasource
+
+  async function createQuery(query: Partial<Query>): Promise<Query> {
+    const defaultQuery: Query = {
+      datasourceId: datasource._id!,
+      name: "New Query",
+      parameters: [],
+      fields: {},
+      schema: {},
+      queryVerb: "read",
+      transformer: "return data",
+      readable: true,
+    }
+    return await config.api.query.create({ ...defaultQuery, ...query })
+  }
+
+  async function withConnection(
+    callback: (client: mysql.Connection) => Promise<void>
+  ): Promise<void> {
+    const ds = await databaseTestProviders.mysql.datasource()
+    const con = await mysql.createConnection(ds.config!)
+    try {
+      await callback(con)
+    } finally {
+      con.end()
+    }
+  }
+
+  afterAll(async () => {
+    await databaseTestProviders.mysql.stop()
+    setup.afterAll()
+  })
+
+  beforeAll(async () => {
+    await config.init()
+    datasource = await config.api.datasource.create(
+      await databaseTestProviders.mysql.datasource()
+    )
+  })
+
+  beforeEach(async () => {
+    await withConnection(async connection => {
+      const resp = await connection.query(createTableSQL)
+      await connection.query(insertSQL)
+    })
+  })
+
+  afterEach(async () => {
+    await withConnection(async connection => {
+      await connection.query(dropTableSQL)
+    })
+  })
+
+  it("should execute a query", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "SELECT * FROM test_table ORDER BY id",
+      },
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      {
+        id: 1,
+        name: "one",
+      },
+      {
+        id: 2,
+        name: "two",
+      },
+      {
+        id: 3,
+        name: "three",
+      },
+      {
+        id: 4,
+        name: "four",
+      },
+      {
+        id: 5,
+        name: "five",
+      },
+    ])
+  })
+
+  it("should be able to transform a query", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "SELECT * FROM test_table WHERE id = 1",
+      },
+      transformer: `
+        data[0].id = data[0].id + 1;
+        return data;
+      `,
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      {
+        id: 2,
+        name: "one",
+      },
+    ])
+  })
+
+  it("should be able to insert with bindings", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "INSERT INTO test_table (name) VALUES ({{ foo }})",
+      },
+      parameters: [
+        {
+          name: "foo",
+          default: "bar",
+        },
+      ],
+      queryVerb: "create",
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: {
+        foo: "baz",
+      },
+    })
+
+    expect(result.data).toEqual([
+      {
+        created: true,
+      },
+    ])
+
+    await withConnection(async connection => {
+      const [rows] = await connection.query(
+        "SELECT * FROM test_table WHERE name = 'baz'"
+      )
+      expect(rows).toHaveLength(1)
+    })
+  })
+
+  it("should be able to update rows", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "UPDATE test_table SET name = {{ name }} WHERE id = {{ id }}",
+      },
+      parameters: [
+        {
+          name: "id",
+          default: "",
+        },
+        {
+          name: "name",
+          default: "updated",
+        },
+      ],
+      queryVerb: "update",
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: {
+        id: "1",
+        name: "foo",
+      },
+    })
+
+    expect(result.data).toEqual([
+      {
+        updated: true,
+      },
+    ])
+
+    await withConnection(async connection => {
+      const [rows] = await connection.query(
+        "SELECT * FROM test_table WHERE id = 1"
+      )
+      expect(rows).toEqual([{ id: 1, name: "foo" }])
+    })
+  })
+
+  it("should be able to delete rows", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "DELETE FROM test_table WHERE id = {{ id }}",
+      },
+      parameters: [
+        {
+          name: "id",
+          default: "",
+        },
+      ],
+      queryVerb: "delete",
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: {
+        id: "1",
+      },
+    })
+
+    expect(result.data).toEqual([
+      {
+        deleted: true,
+      },
+    ])
+
+    await withConnection(async connection => {
+      const [rows] = await connection.query(
+        "SELECT * FROM test_table WHERE id = 1"
+      )
+      expect(rows).toHaveLength(0)
+    })
+  })
+})

packages/server/src/api/routes/tests/queries/postgres.spec.ts

@@ -167,4 +167,77 @@ describe("/queries", () => {
       expect(rows).toHaveLength(1)
     })
   })
+
+  it("should be able to update rows", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "UPDATE test_table SET name = {{ name }} WHERE id = {{ id }}",
+      },
+      parameters: [
+        {
+          name: "id",
+          default: "",
+        },
+        {
+          name: "name",
+          default: "updated",
+        },
+      ],
+      queryVerb: "update",
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: {
+        id: "1",
+        name: "foo",
+      },
+    })
+
+    expect(result.data).toEqual([
+      {
+        updated: true,
+      },
+    ])
+
+    await withClient(async client => {
+      const { rows } = await client.query(
+        "SELECT * FROM test_table WHERE id = 1"
+      )
+      expect(rows).toEqual([{ id: 1, name: "foo" }])
+    })
+  })
+
+  it("should be able to delete rows", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "DELETE FROM test_table WHERE id = {{ id }}",
+      },
+      parameters: [
+        {
+          name: "id",
+          default: "",
+        },
+      ],
+      queryVerb: "delete",
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: {
+        id: "1",
+      },
+    })
+
+    expect(result.data).toEqual([
+      {
+        deleted: true,
+      },
+    ])
+
+    await withClient(async client => {
+      const { rows } = await client.query(
+        "SELECT * FROM test_table WHERE id = 1"
+      )
+      expect(rows).toHaveLength(0)
+    })
+  })
 })

packages/server/src/automations/actions.ts

@@ -9,6 +9,7 @@ import * as serverLog from "./steps/serverLog"
 import * as discord from "./steps/discord"
 import * as slack from "./steps/slack"
 import * as zapier from "./steps/zapier"
+import * as n8n from "./steps/n8n"
 import * as make from "./steps/make"
 import * as filter from "./steps/filter"
 import * as delay from "./steps/delay"
@@ -48,6 +49,7 @@ const ACTION_IMPLS: Record<
   slack: slack.run,
   zapier: zapier.run,
   integromat: make.run,
+  n8n: n8n.run,
 }
 export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
   {
@@ -70,6 +72,7 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
     slack: slack.definition,
     zapier: zapier.definition,
     integromat: make.definition,
+    n8n: n8n.definition,
   }
 
 // don't add the bash script/definitions unless in self host

packages/server/src/automations/steps/make.ts

@@ -34,28 +34,8 @@ export const definition: AutomationStepSchema = {
           type: AutomationIOType.JSON,
           title: "Payload",
         },
-        value1: {
-          type: AutomationIOType.STRING,
-          title: "Input Value 1",
-        },
-        value2: {
-          type: AutomationIOType.STRING,
-          title: "Input Value 2",
-        },
-        value3: {
-          type: AutomationIOType.STRING,
-          title: "Input Value 3",
-        },
-        value4: {
-          type: AutomationIOType.STRING,
-          title: "Input Value 4",
-        },
-        value5: {
-          type: AutomationIOType.STRING,
-          title: "Input Value 5",
-        },
       },
-      required: ["url", "value1", "value2", "value3", "value4", "value5"],
+      required: ["url", "body"],
     },
     outputs: {
       properties: {

packages/server/src/automations/steps/n8n.ts

@@ -0,0 +1,125 @@
+import fetch, { HeadersInit } from "node-fetch"
+import { getFetchResponse } from "./utils"
+import {
+  AutomationActionStepId,
+  AutomationStepSchema,
+  AutomationStepInput,
+  AutomationStepType,
+  AutomationIOType,
+  AutomationFeature,
+  HttpMethod,
+} from "@budibase/types"
+
+export const definition: AutomationStepSchema = {
+  name: "n8n Integration",
+  stepTitle: "n8n",
+  tagline: "Trigger an n8n workflow",
+  description:
+    "Performs a webhook call to n8n and gets the response (if configured)",
+  icon: "ri-shut-down-line",
+  stepId: AutomationActionStepId.n8n,
+  type: AutomationStepType.ACTION,
+  internal: false,
+  features: {
+    [AutomationFeature.LOOPING]: true,
+  },
+  inputs: {},
+  schema: {
+    inputs: {
+      properties: {
+        url: {
+          type: AutomationIOType.STRING,
+          title: "Webhook URL",
+        },
+        method: {
+          type: AutomationIOType.STRING,
+          title: "Method",
+          enum: Object.values(HttpMethod),
+        },
+        authorization: {
+          type: AutomationIOType.STRING,
+          title: "Authorization",
+        },
+        body: {
+          type: AutomationIOType.JSON,
+          title: "Payload",
+        },
+      },
+      required: ["url", "method"],
+    },
+    outputs: {
+      properties: {
+        success: {
+          type: AutomationIOType.BOOLEAN,
+          description: "Whether call was successful",
+        },
+        httpStatus: {
+          type: AutomationIOType.NUMBER,
+          description: "The HTTP status code returned",
+        },
+        response: {
+          type: AutomationIOType.OBJECT,
+          description: "The webhook response - this can have properties",
+        },
+      },
+      required: ["success", "response"],
+    },
+  },
+}
+
+export async function run({ inputs }: AutomationStepInput) {
+  const { url, body, method, authorization } = inputs
+
+  let payload = {}
+  try {
+    payload = body?.value ? JSON.parse(body?.value) : {}
+  } catch (err) {
+    return {
+      httpStatus: 400,
+      response: "Invalid payload JSON",
+      success: false,
+    }
+  }
+
+  if (!url?.trim()?.length) {
+    return {
+      httpStatus: 400,
+      response: "Missing Webhook URL",
+      success: false,
+    }
+  }
+  let response
+  let request: {
+    method: string
+    headers: HeadersInit
+    body?: string
+  } = {
+    method: method || HttpMethod.GET,
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: authorization,
+    },
+  }
+  if (!["GET", "HEAD"].includes(request.method)) {
+    request.body = JSON.stringify({
+      ...payload,
+    })
+  }
+
+  try {
+    response = await fetch(url, request)
+  } catch (err: any) {
+    return {
+      httpStatus: 400,
+      response: err.message,
+      success: false,
+    }
+  }
+
+  const { status, message } = await getFetchResponse(response)
+  return {
+    httpStatus: status,
+    success: status === 200,
+    response: message,
+  }
+}

packages/server/src/automations/steps/zapier.ts

@@ -32,26 +32,6 @@ export const definition: AutomationStepSchema = {
           type: AutomationIOType.JSON,
           title: "Payload",
         },
-        value1: {
-          type: AutomationIOType.STRING,
-          title: "Payload Value 1",
-        },
-        value2: {
-          type: AutomationIOType.STRING,
-          title: "Payload Value 2",
-        },
-        value3: {
-          type: AutomationIOType.STRING,
-          title: "Payload Value 3",
-        },
-        value4: {
-          type: AutomationIOType.STRING,
-          title: "Payload Value 4",
-        },
-        value5: {
-          type: AutomationIOType.STRING,
-          title: "Payload Value 5",
-        },
       },
       required: ["url"],
     },

packages/server/src/automations/tests/n8n.spec.ts

@@ -0,0 +1,68 @@
+import { getConfig, afterAll, runStep, actions } from "./utilities"
+
+describe("test the outgoing webhook action", () => {
+  let config = getConfig()
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  afterAll()
+
+  it("should be able to run the action and default to 'get'", async () => {
+    const res = await runStep(actions.n8n.stepId, {
+      url: "http://www.example.com",
+      body: {
+        test: "IGNORE_ME",
+      },
+    })
+    expect(res.response.url).toEqual("http://www.example.com")
+    expect(res.response.method).toEqual("GET")
+    expect(res.response.body).toBeUndefined()
+    expect(res.success).toEqual(true)
+  })
+
+  it("should add the payload props when a JSON string is provided", async () => {
+    const payload = `{ "name": "Adam", "age": 9 }`
+    const res = await runStep(actions.n8n.stepId, {
+      body: {
+        value: payload,
+      },
+      method: "POST",
+      url: "http://www.example.com",
+    })
+    expect(res.response.url).toEqual("http://www.example.com")
+    expect(res.response.method).toEqual("POST")
+    expect(res.response.body).toEqual(`{"name":"Adam","age":9}`)
+    expect(res.success).toEqual(true)
+  })
+
+  it("should return a 400 if the JSON payload string is malformed", async () => {
+    const payload = `{ value1 1 }`
+    const res = await runStep(actions.n8n.stepId, {
+      value1: "ONE",
+      body: {
+        value: payload,
+      },
+      method: "POST",
+      url: "http://www.example.com",
+    })
+    expect(res.httpStatus).toEqual(400)
+    expect(res.response).toEqual("Invalid payload JSON")
+    expect(res.success).toEqual(false)
+  })
+
+  it("should not append the body if the method is HEAD", async () => {
+    const res = await runStep(actions.n8n.stepId, {
+      url: "http://www.example.com",
+      method: "HEAD",
+      body: {
+        test: "IGNORE_ME",
+      },
+    })
+    expect(res.response.url).toEqual("http://www.example.com")
+    expect(res.response.method).toEqual("HEAD")
+    expect(res.response.body).toBeUndefined()
+    expect(res.success).toEqual(true)
+  })
+})

packages/server/src/integrations/rest.ts

@@ -10,6 +10,7 @@ import {
   RestAuthType,
   RestBasicAuthConfig,
   RestBearerAuthConfig,
+  HttpMethod,
 } from "@budibase/types"
 import get from "lodash/get"
 import * as https from "https"
@@ -86,30 +87,30 @@ const SCHEMA: Integration = {
   query: {
     create: {
       readable: true,
-      displayName: "POST",
+      displayName: HttpMethod.POST,
       type: QueryType.FIELDS,
       fields: coreFields,
     },
     read: {
-      displayName: "GET",
+      displayName: HttpMethod.GET,
       readable: true,
       type: QueryType.FIELDS,
       fields: coreFields,
     },
     update: {
-      displayName: "PUT",
+      displayName: HttpMethod.PUT,
       readable: true,
       type: QueryType.FIELDS,
       fields: coreFields,
     },
     patch: {
-      displayName: "PATCH",
+      displayName: HttpMethod.PATCH,
       readable: true,
       type: QueryType.FIELDS,
       fields: coreFields,
     },
     delete: {
-      displayName: "DELETE",
+      displayName: HttpMethod.DELETE,
       type: QueryType.FIELDS,
       fields: coreFields,
     },
@@ -358,7 +359,7 @@ class RestIntegration implements IntegrationBase {
       path = "",
       queryString = "",
       headers = {},
-      method = "GET",
+      method = HttpMethod.GET,
       disabledHeaders,
       bodyType,
       requestBody,
@@ -413,23 +414,23 @@ class RestIntegration implements IntegrationBase {
   }
 
   async create(opts: RestQuery) {
-    return this._req({ ...opts, method: "POST" })
+    return this._req({ ...opts, method: HttpMethod.POST })
   }
 
   async read(opts: RestQuery) {
-    return this._req({ ...opts, method: "GET" })
+    return this._req({ ...opts, method: HttpMethod.GET })
   }
 
   async update(opts: RestQuery) {
-    return this._req({ ...opts, method: "PUT" })
+    return this._req({ ...opts, method: HttpMethod.PUT })
   }
 
   async patch(opts: RestQuery) {
-    return this._req({ ...opts, method: "PATCH" })
+    return this._req({ ...opts, method: HttpMethod.PATCH })
   }
 
   async delete(opts: RestQuery) {
-    return this._req({ ...opts, method: "DELETE" })
+    return this._req({ ...opts, method: HttpMethod.DELETE })
   }
 }
 

packages/server/src/integrations/tests/utils/index.ts

@@ -3,6 +3,7 @@ jest.unmock("pg")
 import { Datasource } from "@budibase/types"
 import * as postgres from "./postgres"
 import * as mongodb from "./mongodb"
+import * as mysql from "./mysql"
 import { StartedTestContainer } from "testcontainers"
 
 jest.setTimeout(30000)
@@ -13,4 +14,4 @@ export interface DatabaseProvider {
   datasource(): Promise<Datasource>
 }
 
-export const databaseTestProviders = { postgres, mongodb }
+export const databaseTestProviders = { postgres, mongodb, mysql }

packages/server/src/integrations/tests/utils/mysql.ts

@@ -0,0 +1,53 @@
+import { Datasource, SourceName } from "@budibase/types"
+import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+
+let container: StartedTestContainer | undefined
+
+export async function start(): Promise<StartedTestContainer> {
+  return await new GenericContainer("mysql:8.3")
+    .withExposedPorts(3306)
+    .withEnvironment({ MYSQL_ROOT_PASSWORD: "password" })
+    .withWaitStrategy(
+      Wait.forSuccessfulCommand(
+        // Because MySQL first starts itself up, runs an init script, then restarts,
+        // it's possible for the mysqladmin ping to succeed early and then tests to
+        // run against a MySQL that's mid-restart and fail. To avoid this, we run
+        // the ping command three times with a small delay between each.
+        `
+          mysqladmin ping -h localhost -P 3306 -u root -ppassword && sleep 1 &&
+          mysqladmin ping -h localhost -P 3306 -u root -ppassword && sleep 1 &&
+          mysqladmin ping -h localhost -P 3306 -u root -ppassword && sleep 1 &&
+          mysqladmin ping -h localhost -P 3306 -u root -ppassword
+        `
+      )
+    )
+    .start()
+}
+
+export async function datasource(): Promise<Datasource> {
+  if (!container) {
+    container = await start()
+  }
+  const host = container.getHost()
+  const port = container.getMappedPort(3306)
+
+  return {
+    type: "datasource_plus",
+    source: SourceName.MYSQL,
+    plus: true,
+    config: {
+      host,
+      port,
+      user: "root",
+      password: "password",
+      database: "mysql",
+    },
+  }
+}
+
+export async function stop() {
+  if (container) {
+    await container.stop()
+    container = undefined
+  }
+}

packages/server/src/integrations/tests/utils/postgres.ts

@@ -8,9 +8,7 @@ export async function start(): Promise<StartedTestContainer> {
     .withExposedPorts(5432)
     .withEnvironment({ POSTGRES_PASSWORD: "password" })
     .withWaitStrategy(
-      Wait.forSuccessfulCommand(
+      Wait.forSuccessfulCommand("pg_isready -h localhost -p 5432")
-        "pg_isready -h localhost -p 5432"
-      ).withStartupTimeout(10000)
     )
     .start()
 }

packages/server/src/utilities/workerRequests.ts

@@ -14,12 +14,23 @@ export function request(ctx?: Ctx, request?: any) {
   if (!request.headers) {
     request.headers = {}
   }
+
   if (!ctx) {
     request.headers[constants.Header.API_KEY] = coreEnv.INTERNAL_API_KEY
-    if (tenancy.isTenantIdSet()) {
+  } else if (ctx.headers) {
-      request.headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
+    // copy all Budibase utilised headers over - copying everything can have
+    // side effects like requests being rejected due to odd content types etc
+    for (let header of Object.values(constants.Header)) {
+      if (ctx.headers[header]) {
+        request.headers[header] = ctx.headers[header]
+      }
     }
   }
+
+  // apply tenancy if its available
+  if (tenancy.isTenantIdSet()) {
+    request.headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
+  }
   if (request.body && Object.keys(request.body).length > 0) {
     request.headers["Content-Type"] = "application/json"
     request.body =
@@ -29,9 +40,6 @@ export function request(ctx?: Ctx, request?: any) {
   } else {
     delete request.body
   }
-  if (ctx && ctx.headers) {
-    request.headers = ctx.headers
-  }
 
   // add x-budibase-correlation-id header
   logging.correlation.setHeader(request.headers)
@@ -54,7 +62,7 @@ async function checkResponse(
     }
     const msg = `Unable to ${errorMsg} - ${responseErrorMessage}`
     if (ctx) {
-      ctx.throw(msg, response.status)
+      ctx.throw(response.status || 500, msg)
     } else {
       throw msg
     }

packages/types/src/documents/app/automation.ts

@@ -69,6 +69,7 @@ export enum AutomationActionStepId {
   slack = "slack",
   zapier = "zapier",
   integromat = "integromat",
+  n8n = "n8n",
 }
 
 export interface EmailInvite {

packages/types/src/documents/app/query.ts

@@ -64,3 +64,12 @@ export interface ExecuteQueryRequest {
 export interface ExecuteQueryResponse {
   data: Row[]
 }
+
+export enum HttpMethod {
+  GET = "GET",
+  POST = "POST",
+  PATCH = "PATCH",
+  PUT = "PUT",
+  HEAD = "HEAD",
+  DELETE = "DELETE",
+}