Merge branch 'master' into fix-formula-crash

hosting/docker-compose.yaml

@@ -98,7 +98,6 @@ services:
   couchdb-service:
     restart: unless-stopped
     image: budibase/couchdb
-    pull_policy: always
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.19.3",
+  "version": "2.19.6",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

package.json

@@ -58,7 +58,7 @@
     "lint": "yarn run lint:eslint && yarn run lint:prettier",
     "lint:fix:eslint": "eslint --fix --max-warnings=0 packages qa-core",
     "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"",
-    "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
+    "lint:fix": "yarn run lint:fix:eslint && yarn run lint:fix:prettier",
     "build:specs": "lerna run --stream specs",
     "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild",

packages/backend-core/src/events/publishers/query.ts

@@ -3,6 +3,7 @@ import {
   Event,
   Datasource,
   Query,
+  QueryPreview,
   QueryCreatedEvent,
   QueryUpdatedEvent,
   QueryDeletedEvent,
@@ -68,9 +69,9 @@ const run = async (count: number, timestamp?: string | number) => {
   await publishEvent(Event.QUERIES_RUN, properties, timestamp)
 }
 
-const previewed = async (datasource: Datasource, query: Query) => {
+const previewed = async (datasource: Datasource, query: QueryPreview) => {
   const properties: QueryPreviewedEvent = {
-    queryId: query._id,
+    queryId: query.queryId,
     datasourceId: datasource._id as string,
     source: datasource.source,
     queryVerb: query.queryVerb,

packages/backend-core/src/installation.ts

@@ -6,6 +6,7 @@ import * as context from "./context"
 import semver from "semver"
 import { bustCache, withCache, TTL, CacheKey } from "./cache/generic"
 import environment from "./environment"
+import { logAlert } from "./logging"
 
 export const getInstall = async (): Promise<Installation> => {
   return withCache(CacheKey.INSTALLATION, TTL.ONE_DAY, getInstallFromDB, {
@@ -80,27 +81,35 @@ export const checkInstallVersion = async (): Promise<void> => {
   const currentVersion = install.version
   const newVersion = environment.VERSION
 
-  if (currentVersion !== newVersion) {
+  try {
-    const isUpgrade = semver.gt(newVersion, currentVersion)
+    if (currentVersion !== newVersion) {
-    const isDowngrade = semver.lt(newVersion, currentVersion)
+      const isUpgrade = semver.gt(newVersion, currentVersion)
+      const isDowngrade = semver.lt(newVersion, currentVersion)
 
-    const success = await updateVersion(newVersion)
+      const success = await updateVersion(newVersion)
 
-    if (success) {
+      if (success) {
-      await context.doInIdentityContext(
+        await context.doInIdentityContext(
-        {
+          {
-          _id: install.installId,
+            _id: install.installId,
-          type: IdentityType.INSTALLATION,
+            type: IdentityType.INSTALLATION,
-        },
+          },
-        async () => {
+          async () => {
-          if (isUpgrade) {
+            if (isUpgrade) {
-            await events.installation.upgraded(currentVersion, newVersion)
+              await events.installation.upgraded(currentVersion, newVersion)
-          } else if (isDowngrade) {
+            } else if (isDowngrade) {
-            await events.installation.downgraded(currentVersion, newVersion)
+              await events.installation.downgraded(currentVersion, newVersion)
+            }
           }
-        }
+        )
-      )
+        await events.identification.identifyInstallationGroup(install.installId)
-      await events.identification.identifyInstallationGroup(install.installId)
+      }
+    }
+  } catch (err: any) {
+    if (err?.message?.includes("Invalid Version")) {
+      logAlert(`Invalid version "${newVersion}" - is it semver?`)
+    } else {
+      logAlert("Failed to retrieve version", err)
     }
   }
 }

packages/backend-core/src/logging/correlation/correlation.ts

@@ -2,11 +2,12 @@ import { Header } from "../../constants"
 
 const correlator = require("correlation-id")
 
-export const setHeader = (headers: any) => {
+export const setHeader = (headers: Record<string, string>) => {
   const correlationId = correlator.getId()
-  if (correlationId) {
+  if (!correlationId) {
-    headers[Header.CORRELATION_ID] = correlationId
+    return
   }
+  headers[Header.CORRELATION_ID] = correlationId
 }
 
 export function getId() {

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte

@@ -128,10 +128,10 @@
         >
           <div class="item-body">
             <img
-              width="20"
+              width={20}
-              height="20"
+              height={20}
               src={externalActions[action.stepId].icon}
-              alt="zapier"
+              alt={externalActions[action.stepId].name}
             />
             <span class="icon-spacing">
               <Body size="XS">

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ExternalActions.js

@@ -1,5 +1,6 @@
 import DiscordLogo from "assets/discord.svg"
 import ZapierLogo from "assets/zapier.png"
+import n8nLogo from "assets/n8n_square.png"
 import MakeLogo from "assets/make.svg"
 import SlackLogo from "assets/slack.svg"
 
@@ -8,4 +9,5 @@ export const externalActions = {
   discord: { name: "discord", icon: DiscordLogo },
   slack: { name: "slack", icon: SlackLogo },
   integromat: { name: "integromat", icon: MakeLogo },
+  n8n: { name: "n8n", icon: n8nLogo },
 }

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -79,6 +79,7 @@
     disableWrapping: true,
   })
   $: editingJs = codeMode === EditorModes.JS
+  $: requiredProperties = block.schema.inputs.required || []
 
   $: stepCompletions =
     codeMode === EditorModes.Handlebars
@@ -359,6 +360,11 @@
     )
   }
 
+  function getFieldLabel(key, value) {
+    const requiredSuffix = requiredProperties.includes(key) ? "*" : ""
+    return `${value.title || (key === "row" ? "Table" : key)} ${requiredSuffix}`
+  }
+
   onMount(async () => {
     try {
       await environment.loadVariables()
@@ -376,7 +382,7 @@
           <Label
             tooltip={value.title === "Binding / Value"
               ? "If using the String input type, please use a comma or newline separated string"
-              : null}>{value.title || (key === "row" ? "Table" : key)}</Label
+              : null}>{getFieldLabel(key, value)}</Label
           >
         {/if}
         <div class:field-width={shouldRenderField(value)}>

packages/builder/src/components/design/settings/controls/DataSourceSelect/DataSourceSelect.svelte

@@ -127,10 +127,14 @@
       }
     })
   $: jsonArrays = bindings
-    .filter(x => x.fieldSchema?.type === "jsonarray")
+    .filter(
+      x =>
+        x.fieldSchema?.type === "jsonarray" ||
+        (x.fieldSchema?.type === "json" && x.fieldSchema?.subtype === "array")
+    )
     .map(binding => {
       const { providerId, readableBinding, runtimeBinding, tableId } = binding
-      const { name, type, prefixKeys } = binding.fieldSchema
+      const { name, type, prefixKeys, subtype } = binding.fieldSchema
       return {
         providerId,
         label: readableBinding,
@@ -138,7 +142,8 @@
         fieldType: type,
         tableId,
         prefixKeys,
-        type: "jsonarray",
+        type: type === "jsonarray" ? "jsonarray" : "queryarray",
+        subtype,
         value: `{{ literal ${runtimeBinding} }}`,
       }
     })

packages/builder/src/components/integration/KeyValueBuilder.svelte

@@ -85,6 +85,16 @@
     activity = newActivity
     dispatch("change", fields)
   }
+
+  function isJsonArray(value) {
+    if (!value || typeof value === "string") {
+      return false
+    }
+    if (value.type === "array") {
+      return true
+    }
+    return value.type === "json" && value.subtype === "array"
+  }
 </script>
 
 <!-- Builds Objects with Key Value Pairs. Useful for building things like Request Headers. -->
@@ -112,7 +122,9 @@
         bind:value={field.name}
         on:blur={changed}
       />
-      {#if options}
+      {#if isJsonArray(field.value)}
+        <Select readonly={true} value="Array" options={["Array"]} />
+      {:else if options}
         <Select
           bind:value={field.value}
           {compare}

packages/builder/src/components/integration/QueryViewer.svelte

@@ -40,6 +40,7 @@
   let schemaType
 
   let autoSchema = {}
+  let nestedSchemaFields = {}
   let rows = []
   let keys = {}
 
@@ -83,13 +84,14 @@
         return
       }
 
+      nestedSchemaFields = response.nestedSchemaFields
+
       if (Object.keys(newQuery.schema).length === 0) {
         // Assign this to a variable instead of directly to the newQuery.schema so that a user
         // can change the table they're querying and have the schema update until they first
         // edit it
         autoSchema = response.schema
       }
-
       rows = response.rows
 
       notifications.success("Query executed successfully")
@@ -120,6 +122,7 @@
           Object.keys(newQuery.schema).length === 0
             ? autoSchema
             : newQuery.schema,
+        nestedSchemaFields,
       })
 
       notifications.success("Query saved successfully")

packages/builder/src/components/integration/RestQueryViewer.svelte

@@ -159,7 +159,7 @@
     newQuery.fields.queryString = queryString
     newQuery.fields.authConfigId = authConfigId
     newQuery.fields.disabledHeaders = restUtils.flipHeaderState(enabledHeaders)
-    newQuery.schema = schema
+    newQuery.schema = schema || {}
 
     return newQuery
   }

packages/builder/src/components/integration/index.svelte

@@ -5,7 +5,6 @@
     Label,
     Input,
     Select,
-    Divider,
     Layout,
     Icon,
     Button,
@@ -124,7 +123,6 @@
         {#each query.fields.steps ?? [] as step, index}
           <div class="block">
             <div class="subblock">
-              <Divider noMargin />
               <div class="blockSection">
                 <div class="block-options">
                   Stage {index + 1}

packages/builder/src/constants/backend/automations.js

@@ -27,6 +27,7 @@ export const ActionStepID = {
   slack: "slack",
   zapier: "zapier",
   integromat: "integromat",
+  n8n: "n8n",
 }
 
 export const Features = {

packages/builder/src/constants/backend/index.js

@@ -310,6 +310,7 @@ export const BannedSearchTypes = [
   "formula",
   "json",
   "jsonarray",
+  "queryarray",
 ]
 
 export const DatasourceTypes = {

packages/builder/src/dataBinding.js

@@ -425,7 +425,7 @@ const generateComponentContextBindings = (asset, componentContext) => {
       table = info.table
 
       // Determine what to prefix bindings with
-      if (datasource.type === "jsonarray") {
+      if (datasource.type === "jsonarray" || datasource.type === "queryarray") {
         // For JSON arrays, use the array name as the readable prefix
         const split = datasource.label.split(".")
         readablePrefix = split[split.length - 1]
@@ -904,6 +904,19 @@ export const getSchemaForDatasource = (asset, datasource, options) => {
       schema = JSONUtils.getJSONArrayDatasourceSchema(tableSchema, datasource)
     }
 
+    // "queryarray" datasources are arrays inside JSON responses
+    else if (type === "queryarray") {
+      const queries = get(queriesStores).list
+      table = queries.find(query => query._id === datasource.tableId)
+      let tableSchema = table?.schema
+      let nestedSchemaFields = table?.nestedSchemaFields
+      schema = JSONUtils.generateQueryArraySchemas(
+        tableSchema,
+        nestedSchemaFields
+      )
+      schema = JSONUtils.getJSONArrayDatasourceSchema(schema, datasource)
+    }
+
     // Otherwise we assume we're targeting an internal table or a plus
     // datasource, and we can treat it as a table with a schema
     else {

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentDropdownMenu.svelte

@@ -3,6 +3,7 @@
   import { ActionMenu, MenuItem, Icon } from "@budibase/bbui"
 
   export let component
+  export let opened
 
   $: definition = componentStore.getDefinition(component?._component)
   $: noPaste = !$componentStore.componentToPaste
@@ -85,6 +86,39 @@
   >
     Paste
   </MenuItem>
+
+  {#if component?._children?.length}
+    <MenuItem
+      icon="TreeExpand"
+      keyBind="!ArrowRight"
+      on:click={() => keyboardEvent("ArrowRight", false)}
+      disabled={opened}
+    >
+      Expand
+    </MenuItem>
+    <MenuItem
+      icon="TreeCollapse"
+      keyBind="!ArrowLeft"
+      on:click={() => keyboardEvent("ArrowLeft", false)}
+      disabled={!opened}
+    >
+      Collapse
+    </MenuItem>
+    <MenuItem
+      icon="TreeExpandAll"
+      keyBind="Ctrl+!ArrowRight"
+      on:click={() => keyboardEvent("ArrowRight", true)}
+    >
+      Expand All
+    </MenuItem>
+    <MenuItem
+      icon="TreeCollapseAll"
+      keyBind="Ctrl+!ArrowLeft"
+      on:click={() => keyboardEvent("ArrowLeft", true)}
+    >
+      Collapse All
+    </MenuItem>
+  {/if}
 </ActionMenu>
 
 <style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentKeyHandler.svelte

@@ -9,6 +9,7 @@
   import { goto, isActive } from "@roxi/routify"
   import { notifications } from "@budibase/bbui"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+  import componentTreeNodesStore from "stores/portal/componentTreeNodesStore"
 
   let confirmDeleteDialog
   let confirmEjectDialog
@@ -61,6 +62,40 @@
     ["ArrowDown"]: () => {
       componentStore.selectNext()
     },
+    ["ArrowRight"]: component => {
+      componentTreeNodesStore.expandNode(component._id)
+    },
+    ["ArrowLeft"]: component => {
+      componentTreeNodesStore.collapseNode(component._id)
+    },
+    ["Ctrl+ArrowRight"]: component => {
+      componentTreeNodesStore.expandNode(component._id)
+
+      const expandChildren = component => {
+        const children = component._children ?? []
+
+        children.forEach(child => {
+          componentTreeNodesStore.expandNode(child._id)
+          expandChildren(child)
+        })
+      }
+
+      expandChildren(component)
+    },
+    ["Ctrl+ArrowLeft"]: component => {
+      componentTreeNodesStore.collapseNode(component._id)
+
+      const collapseChildren = component => {
+        const children = component._children ?? []
+
+        children.forEach(child => {
+          componentTreeNodesStore.collapseNode(child._id)
+          collapseChildren(child)
+        })
+      }
+
+      collapseChildren(component)
+    },
     ["Escape"]: () => {
       if ($isActive(`./:componentId/new`)) {
         $goto(`./${$componentStore.selectedComponentId}`)

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentTree.svelte

@@ -17,11 +17,12 @@
   } from "helpers/components"
   import { get } from "svelte/store"
   import { dndStore } from "./dndStore"
+  import componentTreeNodesStore from "stores/portal/componentTreeNodesStore"
 
   export let components = []
   export let level = 0
 
-  let closedNodes = {}
+  $: openNodes = $componentTreeNodesStore
 
   $: filteredComponents = components?.filter(component => {
     return (
@@ -54,15 +55,6 @@
     return componentSupportsChildren(component) && component._children?.length
   }
 
-  function toggleNodeOpen(componentId) {
-    if (closedNodes[componentId]) {
-      delete closedNodes[componentId]
-    } else {
-      closedNodes[componentId] = true
-    }
-    closedNodes = closedNodes
-  }
-
   const onDrop = async e => {
     e.stopPropagation()
     try {
@@ -72,14 +64,14 @@
     }
   }
 
-  const isOpen = (component, selectedComponentPath, closedNodes) => {
+  const isOpen = (component, selectedComponentPath, openNodes) => {
     if (!component?._children?.length) {
       return false
     }
-    if (selectedComponentPath.includes(component._id)) {
+    if (selectedComponentPath.slice(0, -1).includes(component._id)) {
       return true
     }
-    return !closedNodes[component._id]
+    return openNodes[`nodeOpen-${component._id}`]
   }
 
   const isChildOfSelectedComponent = component => {
@@ -96,7 +88,7 @@
 
 <ul>
   {#each filteredComponents || [] as component, index (component._id)}
-    {@const opened = isOpen(component, $selectedComponentPath, closedNodes)}
+    {@const opened = isOpen(component, $selectedComponentPath, openNodes)}
     <li
       on:click|stopPropagation={() => {
         componentStore.select(component._id)
@@ -110,7 +102,7 @@
         on:dragend={dndStore.actions.reset}
         on:dragstart={() => dndStore.actions.dragstart(component)}
         on:dragover={dragover(component, index)}
-        on:iconClick={() => toggleNodeOpen(component._id)}
+        on:iconClick={() => componentTreeNodesStore.toggleNode(component._id)}
         on:drop={onDrop}
         hovering={$hoverStore.componentId === component._id}
         on:mouseenter={() => hover(component._id)}
@@ -125,8 +117,9 @@
         highlighted={isChildOfSelectedComponent(component)}
         selectedBy={$userSelectedResourceMap[component._id]}
       >
-        <ComponentDropdownMenu {component} />
+        <ComponentDropdownMenu {opened} {component} />
       </NavItem>
+
       {#if opened}
         <svelte:self
           components={component._children}
@@ -144,13 +137,6 @@
     padding-left: 0;
     margin: 0;
   }
-  ul :global(.icon.arrow) {
-    transition: opacity 130ms ease-out;
-    opacity: 0;
-  }
-  ul:hover :global(.icon.arrow) {
-    opacity: 1;
-  }
   ul,
   li {
     min-width: max-content;

packages/builder/src/stores/builder/components.js

@@ -29,6 +29,7 @@ import {
 } from "constants/backend"
 import BudiStore from "./BudiStore"
 import { Utils } from "@budibase/frontend-core"
+import componentTreeNodesStore from "stores/portal/componentTreeNodesStore"
 
 export const INITIAL_COMPONENTS_STATE = {
   components: {},
@@ -662,6 +663,7 @@ export class ComponentStore extends BudiStore {
     const screen = get(selectedScreen)
     const parent = findComponentParent(screen.props, componentId)
     const index = parent?._children.findIndex(x => x._id === componentId)
+    const componentTreeNodes = get(componentTreeNodesStore)
 
     // Check for screen and navigation component edge cases
     const screenComponentId = `${screen._id}-screen`
@@ -680,9 +682,15 @@ export class ComponentStore extends BudiStore {
     if (index > 0) {
       // If sibling before us accepts children, select a descendant
       const previousSibling = parent._children[index - 1]
-      if (previousSibling._children?.length) {
+      if (
+        previousSibling._children?.length &&
+        componentTreeNodes[`nodeOpen-${previousSibling._id}`]
+      ) {
         let target = previousSibling
-        while (target._children?.length) {
+        while (
+          target._children?.length &&
+          componentTreeNodes[`nodeOpen-${target._id}`]
+        ) {
           target = target._children[target._children.length - 1]
         }
         return target._id
@@ -703,6 +711,7 @@ export class ComponentStore extends BudiStore {
     const screen = get(selectedScreen)
     const parent = findComponentParent(screen.props, componentId)
     const index = parent?._children.findIndex(x => x._id === componentId)
+    const componentTreeNodes = get(componentTreeNodesStore)
 
     // Check for screen and navigation component edge cases
     const screenComponentId = `${screen._id}-screen`
@@ -712,7 +721,11 @@ export class ComponentStore extends BudiStore {
     }
 
     // If we have children, select first child
-    if (component._children?.length) {
+    if (
+      component._children?.length &&
+      (state.selectedComponentId === navComponentId ||
+        componentTreeNodes[`nodeOpen-${component._id}`])
+    ) {
       return component._children[0]._id
     } else if (!parent) {
       return null

packages/builder/src/stores/builder/index.js

@@ -92,12 +92,13 @@ const resetBuilderHistory = () => {
 
 export const initialise = async pkg => {
   const { application } = pkg
+  // must be first operation to make sure subsequent requests have correct app ID
+  appStore.syncAppPackage(pkg)
   await Promise.all([
     appStore.syncAppRoutes(),
     componentStore.refreshDefinitions(application?.appId),
   ])
   builderStore.init(application)
-  appStore.syncAppPackage(pkg)
   navigationStore.syncAppNavigation(application?.navigation)
   themeStore.syncAppTheme(application)
   screenStore.syncAppScreens(pkg)

packages/builder/src/stores/portal/componentTreeNodesStore.js

@@ -0,0 +1,36 @@
+import { createSessionStorageStore } from "@budibase/frontend-core"
+
+const baseStore = createSessionStorageStore("openNodes", {})
+
+const toggleNode = componentId => {
+  baseStore.update(openNodes => {
+    openNodes[`nodeOpen-${componentId}`] = !openNodes[`nodeOpen-${componentId}`]
+
+    return openNodes
+  })
+}
+
+const expandNode = componentId => {
+  baseStore.update(openNodes => {
+    openNodes[`nodeOpen-${componentId}`] = true
+
+    return openNodes
+  })
+}
+
+const collapseNode = componentId => {
+  baseStore.update(openNodes => {
+    openNodes[`nodeOpen-${componentId}`] = false
+
+    return openNodes
+  })
+}
+
+const store = {
+  subscribe: baseStore.subscribe,
+  toggleNode,
+  expandNode,
+  collapseNode,
+}
+
+export default store

packages/cli/src/hosting/update.ts

@@ -13,7 +13,12 @@ import { COMPOSE_PATH } from "./makeFiles"
 import { info, success } from "../utils"
 import { start } from "./start"
 
-const BB_COMPOSE_SERVICES = ["app-service", "worker-service", "proxy-service"]
+const BB_COMPOSE_SERVICES = [
+  "app-service",
+  "worker-service",
+  "proxy-service",
+  "couchdb-service",
+]
 const BB_SINGLE_SERVICE = ["budibase"]
 
 export async function update() {

packages/client/manifest.json

@@ -270,6 +270,7 @@
           {
             "type": "buttonConfiguration",
             "key": "buttons",
+            "nested": true,
             "defaultValue": [
               {
                 "type": "cta",

packages/client/src/components/app/forms/Form.svelte

@@ -84,7 +84,7 @@
 
   // Fetches the form schema from this form's dataSource
   const fetchSchema = async dataSource => {
-    if (dataSource?.tableId && dataSource?.type !== "query") {
+    if (dataSource?.tableId && !dataSource?.type?.startsWith("query")) {
       try {
         table = await API.fetchTableDefinition(dataSource.tableId)
       } catch (error) {

packages/client/src/utils/schema.js

@@ -7,6 +7,7 @@ import NestedProviderFetch from "@budibase/frontend-core/src/fetch/NestedProvide
 import FieldFetch from "@budibase/frontend-core/src/fetch/FieldFetch.js"
 import JSONArrayFetch from "@budibase/frontend-core/src/fetch/JSONArrayFetch.js"
 import ViewV2Fetch from "@budibase/frontend-core/src/fetch/ViewV2Fetch.js"
+import QueryArrayFetch from "@budibase/frontend-core/src/fetch/QueryArrayFetch"
 
 /**
  * Fetches the schema of any kind of datasource.
@@ -28,6 +29,7 @@ export const fetchDatasourceSchema = async (
     provider: NestedProviderFetch,
     field: FieldFetch,
     jsonarray: JSONArrayFetch,
+    queryarray: QueryArrayFetch,
   }[datasource?.type]
   if (!handler) {
     return null

packages/frontend-core/src/fetch/QueryArrayFetch.js

@@ -0,0 +1,25 @@
+import FieldFetch from "./FieldFetch.js"
+import {
+  getJSONArrayDatasourceSchema,
+  generateQueryArraySchemas,
+} from "../utils/json"
+
+export default class QueryArrayFetch extends FieldFetch {
+  async getDefinition(datasource) {
+    if (!datasource?.tableId) {
+      return null
+    }
+    // JSON arrays need their table definitions fetched.
+    // We can then extract their schema as a subset of the table schema.
+    try {
+      const table = await this.API.fetchQueryDefinition(datasource.tableId)
+      const schema = generateQueryArraySchemas(
+        table?.schema,
+        table?.nestedSchemaFields
+      )
+      return { schema: getJSONArrayDatasourceSchema(schema, datasource) }
+    } catch (error) {
+      return null
+    }
+  }
+}

packages/frontend-core/src/fetch/index.js

@@ -9,6 +9,7 @@ import JSONArrayFetch from "./JSONArrayFetch.js"
 import UserFetch from "./UserFetch.js"
 import GroupUserFetch from "./GroupUserFetch.js"
 import CustomFetch from "./CustomFetch.js"
+import QueryArrayFetch from "./QueryArrayFetch.js"
 
 const DataFetchMap = {
   table: TableFetch,
@@ -24,6 +25,7 @@ const DataFetchMap = {
   provider: NestedProviderFetch,
   field: FieldFetch,
   jsonarray: JSONArrayFetch,
+  queryarray: QueryArrayFetch,
 }
 
 // Constructs a new fetch model for a certain datasource

packages/frontend-core/src/stores/index.js

@@ -1 +1,2 @@
 export { createLocalStorageStore } from "./localStorage"
+export { createSessionStorageStore } from "./sessionStorage"

packages/frontend-core/src/stores/sessionStorage.js

@@ -0,0 +1,46 @@
+import { get, writable } from "svelte/store"
+
+export const createSessionStorageStore = (sessionStorageKey, initialValue) => {
+  const store = writable(initialValue, () => {
+    // Hydrate from session storage when we get a new subscriber
+    hydrate()
+
+    // Listen for session storage changes and keep store in sync
+    const storageListener = ({ key }) => {
+      return key === sessionStorageKey && hydrate()
+    }
+
+    window.addEventListener("storage", storageListener)
+    return () => window.removeEventListener("storage", storageListener)
+  })
+
+  // New store setter which updates the store and sessionstorage
+  const set = value => {
+    store.set(value)
+    sessionStorage.setItem(sessionStorageKey, JSON.stringify(value))
+  }
+
+  // New store updater which updates the store and sessionstorage
+  const update = updaterFn => set(updaterFn(get(store)))
+
+  // Hydrates the store from sessionstorage
+  const hydrate = () => {
+    const sessionValue = sessionStorage.getItem(sessionStorageKey)
+    if (sessionValue == null) {
+      set(initialValue)
+    } else {
+      try {
+        store.set(JSON.parse(sessionValue))
+      } catch {
+        set(initialValue)
+      }
+    }
+  }
+
+  // Patch the default svelte store functions with our overrides
+  return {
+    ...store,
+    set,
+    update,
+  }
+}

packages/frontend-core/src/utils/json.js

@@ -1,3 +1,5 @@
+import { utils } from "@budibase/shared-core"
+
 /**
  * Gets the schema for a datasource which is targeting a JSON array, including
  * nested JSON arrays. The returned schema is a squashed, table-like schema
@@ -119,3 +121,33 @@ const extractJSONSchemaKeys = (jsonSchema, squashObjects = false) => {
   })
   return keys
 }
+
+export const generateQueryArraySchemas = (schema, nestedSchemaFields) => {
+  for (let key in schema) {
+    if (
+      schema[key]?.type === "json" &&
+      schema[key]?.subtype === "array" &&
+      utils.hasSchema(nestedSchemaFields[key])
+    ) {
+      schema[key] = {
+        schema: {
+          schema: Object.entries(nestedSchemaFields[key] || {}).reduce(
+            (acc, [nestedKey, fieldSchema]) => {
+              acc[nestedKey] = {
+                name: nestedKey,
+                type: fieldSchema.type,
+                subtype: fieldSchema.subtype,
+              }
+              return acc
+            },
+            {}
+          ),
+          type: "json",
+        },
+        type: "json",
+        subtype: "array",
+      }
+    }
+  }
+  return schema
+}

packages/server/Dockerfile

@@ -66,6 +66,7 @@ COPY packages/server/dist/ dist/
 COPY packages/server/docker_run.sh .
 COPY packages/server/builder/ builder/
 COPY packages/server/client/ client/
+COPY packages/server/pm2.config.js .
 
 ARG BUDIBASE_VERSION
 ARG GIT_COMMIT_SHA

packages/server/src/api/controllers/automation.ts

@@ -1,4 +1,3 @@
-import * as actions from "../../automations/actions"
 import * as triggers from "../../automations/triggers"
 import {
   getAutomationParams,
@@ -20,11 +19,12 @@ import {
   Automation,
   AutomationActionStepId,
   AutomationResults,
-  BBContext,
+  UserCtx,
 } from "@budibase/types"
 import { getActionDefinitions as actionDefs } from "../../automations/actions"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
+import env from "../../environment"
 
 async function getActionDefinitions() {
   return removeDeprecated(await actionDefs())
@@ -72,7 +72,7 @@ function cleanAutomationInputs(automation: Automation) {
   return automation
 }
 
-export async function create(ctx: BBContext) {
+export async function create(ctx: UserCtx) {
   const db = context.getAppDB()
   let automation = ctx.request.body
   automation.appId = ctx.appId
@@ -141,7 +141,7 @@ export async function handleStepEvents(
   }
 }
 
-export async function update(ctx: BBContext) {
+export async function update(ctx: UserCtx) {
   const db = context.getAppDB()
   let automation = ctx.request.body
   automation.appId = ctx.appId
@@ -192,7 +192,7 @@ export async function update(ctx: BBContext) {
   builderSocket?.emitAutomationUpdate(ctx, automation)
 }
 
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: UserCtx) {
   const db = context.getAppDB()
   const response = await db.allDocs(
     getAutomationParams(null, {
@@ -202,12 +202,12 @@ export async function fetch(ctx: BBContext) {
   ctx.body = response.rows.map(row => row.doc)
 }
 
-export async function find(ctx: BBContext) {
+export async function find(ctx: UserCtx) {
   const db = context.getAppDB()
   ctx.body = await db.get(ctx.params.id)
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
   const automationId = ctx.params.id
   const oldAutomation = await db.get<Automation>(automationId)
@@ -221,11 +221,11 @@ export async function destroy(ctx: BBContext) {
   builderSocket?.emitAutomationDeletion(ctx, automationId)
 }
 
-export async function logSearch(ctx: BBContext) {
+export async function logSearch(ctx: UserCtx) {
   ctx.body = await automations.logs.logSearch(ctx.request.body)
 }
 
-export async function clearLogError(ctx: BBContext) {
+export async function clearLogError(ctx: UserCtx) {
   const { automationId, appId } = ctx.request.body
   await context.doInAppContext(appId, async () => {
     const db = context.getProdAppDB()
@@ -244,15 +244,15 @@ export async function clearLogError(ctx: BBContext) {
   })
 }
 
-export async function getActionList(ctx: BBContext) {
+export async function getActionList(ctx: UserCtx) {
   ctx.body = await getActionDefinitions()
 }
 
-export async function getTriggerList(ctx: BBContext) {
+export async function getTriggerList(ctx: UserCtx) {
   ctx.body = getTriggerDefinitions()
 }
 
-export async function getDefinitionList(ctx: BBContext) {
+export async function getDefinitionList(ctx: UserCtx) {
   ctx.body = {
     trigger: getTriggerDefinitions(),
     action: await getActionDefinitions(),
@@ -265,7 +265,7 @@ export async function getDefinitionList(ctx: BBContext) {
  *                   *
  *********************/
 
-export async function trigger(ctx: BBContext) {
+export async function trigger(ctx: UserCtx) {
   const db = context.getAppDB()
   let automation = await db.get<Automation>(ctx.params.id)
 
@@ -275,7 +275,9 @@ export async function trigger(ctx: BBContext) {
       automation,
       {
         fields: ctx.request.body.fields,
-        timeout: ctx.request.body.timeout * 1000 || 120000,
+        timeout:
+          ctx.request.body.timeout * 1000 ||
+          env.getDefaults().AUTOMATION_SYNC_TIMEOUT,
       },
       { getResponses: true }
     )
@@ -310,7 +312,7 @@ function prepareTestInput(input: any) {
   return input
 }
 
-export async function test(ctx: BBContext) {
+export async function test(ctx: UserCtx) {
   const db = context.getAppDB()
   let automation = await db.get<Automation>(ctx.params.id)
   await setTestFlag(automation._id!)

packages/server/src/api/controllers/dev.ts

@@ -1,7 +1,7 @@
 import fetch from "node-fetch"
 import env from "../../environment"
 import { checkSlashesInUrl } from "../../utilities"
-import { request } from "../../utilities/workerRequests"
+import { createRequest } from "../../utilities/workerRequests"
 import { clearLock as redisClearLock } from "../../utilities/redis"
 import { DocumentType } from "../../db/utils"
 import {
@@ -13,14 +13,19 @@ import {
 } from "@budibase/backend-core"
 import { App } from "@budibase/types"
 
-async function redirect(ctx: any, method: string, path: string = "global") {
+async function redirect(
+  ctx: any,
+  method: "GET" | "POST" | "DELETE",
+  path: string = "global"
+) {
   const { devPath } = ctx.params
   const queryString = ctx.originalUrl.split("?")[1] || ""
   const response = await fetch(
     checkSlashesInUrl(
       `${env.WORKER_URL}/api/${path}/${devPath}?${queryString}`
     ),
-    request(ctx, {
+    createRequest({
+      ctx,
       method,
       body: ctx.request.body,
     })

packages/server/src/api/controllers/query/index.ts

@@ -1,5 +1,4 @@
 import { generateQueryID } from "../../../db/utils"
-import { BaseQueryVerbs } from "../../../constants"
 import { Thread, ThreadType } from "../../../threads"
 import { save as saveDatasource } from "../datasource"
 import { RestImporter } from "./import"
@@ -7,36 +6,27 @@ import { invalidateDynamicVariables } from "../../../threads/utils"
 import env from "../../../environment"
 import { events, context, utils, constants } from "@budibase/backend-core"
 import sdk from "../../../sdk"
-import { QueryEvent, QueryResponse } from "../../../threads/definitions"
+import { QueryEvent } from "../../../threads/definitions"
 import {
   ConfigType,
   Query,
   UserCtx,
   SessionCookie,
+  JsonFieldSubType,
+  QueryResponse,
+  QueryPreview,
   QuerySchema,
   FieldType,
   type ExecuteQueryRequest,
   type ExecuteQueryResponse,
   type Row,
 } from "@budibase/types"
-import { ValidQueryNameRegex } from "@budibase/shared-core"
+import { ValidQueryNameRegex, utils as JsonUtils } from "@budibase/shared-core"
 
 const Runner = new Thread(ThreadType.QUERY, {
-  timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000,
+  timeoutMs: env.QUERY_THREAD_TIMEOUT,
 })
 
-// simple function to append "readable" to all read queries
-function enrichQueries(input: any) {
-  const wasArray = Array.isArray(input)
-  const queries = wasArray ? input : [input]
-  for (let query of queries) {
-    if (query.queryVerb === BaseQueryVerbs.READ) {
-      query.readable = true
-    }
-  }
-  return wasArray ? queries : queries[0]
-}
-
 export async function fetch(ctx: UserCtx) {
   ctx.body = await sdk.queries.fetch()
 }
@@ -84,7 +74,7 @@ export { _import as import }
 
 export async function save(ctx: UserCtx) {
   const db = context.getAppDB()
-  const query = ctx.request.body
+  const query: Query = ctx.request.body
 
   // Validate query name
   if (!query?.name.match(ValidQueryNameRegex)) {
@@ -100,7 +90,6 @@ export async function save(ctx: UserCtx) {
   } else {
     eventFn = () => events.query.updated(datasource, query)
   }
-
   const response = await db.put(query)
   await eventFn()
   query._rev = response.rev
@@ -133,7 +122,7 @@ export async function preview(ctx: UserCtx) {
   const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
     ctx.request.body.datasourceId
   )
-  const query = ctx.request.body
+  const query: QueryPreview = ctx.request.body
   // preview may not have a queryId as it hasn't been saved, but if it does
   // this stops dynamic variables from calling the same query
   const { fields, parameters, queryVerb, transformer, queryId, schema } = query
@@ -153,6 +142,69 @@ export async function preview(ctx: UserCtx) {
 
   const authConfigCtx: any = getAuthConfig(ctx)
 
+  function getSchemaFields(
+    rows: any[],
+    keys: string[]
+  ): {
+    previewSchema: Record<string, string | QuerySchema>
+    nestedSchemaFields: {
+      [key: string]: Record<string, string | QuerySchema>
+    }
+  } {
+    const previewSchema: Record<string, string | QuerySchema> = {}
+    const nestedSchemaFields: {
+      [key: string]: Record<string, string | QuerySchema>
+    } = {}
+    const makeQuerySchema = (
+      type: FieldType,
+      name: string,
+      subtype?: string
+    ): QuerySchema => ({
+      type,
+      name,
+      subtype,
+    })
+    if (rows?.length > 0) {
+      for (let key of [...new Set(keys)] as string[]) {
+        const field = rows[0][key]
+        let type = typeof field,
+          fieldMetadata = makeQuerySchema(FieldType.STRING, key)
+        if (field)
+          switch (type) {
+            case "boolean":
+              fieldMetadata = makeQuerySchema(FieldType.BOOLEAN, key)
+              break
+            case "object":
+              if (field instanceof Date) {
+                fieldMetadata = makeQuerySchema(FieldType.DATETIME, key)
+              } else if (Array.isArray(field)) {
+                if (JsonUtils.hasSchema(field[0])) {
+                  fieldMetadata = makeQuerySchema(
+                    FieldType.JSON,
+                    key,
+                    JsonFieldSubType.ARRAY
+                  )
+                } else {
+                  fieldMetadata = makeQuerySchema(FieldType.ARRAY, key)
+                }
+                nestedSchemaFields[key] = getSchemaFields(
+                  field,
+                  Object.keys(field[0])
+                ).previewSchema
+              } else {
+                fieldMetadata = makeQuerySchema(FieldType.JSON, key)
+              }
+              break
+            case "number":
+              fieldMetadata = makeQuerySchema(FieldType.NUMBER, key)
+              break
+          }
+        previewSchema[key] = fieldMetadata
+      }
+    }
+    return { previewSchema, nestedSchemaFields }
+  }
+
   try {
     const inputs: QueryEvent = {
       appId: ctx.appId,
@@ -171,38 +223,11 @@ export async function preview(ctx: UserCtx) {
       },
     }
 
-    const { rows, keys, info, extra } = await Runner.run<QueryResponse>(inputs)
+    const { rows, keys, info, extra } = (await Runner.run(
-    const previewSchema: Record<string, QuerySchema> = {}
+      inputs
-    const makeQuerySchema = (type: FieldType, name: string): QuerySchema => ({
+    )) as QueryResponse
-      type,
+    const { previewSchema, nestedSchemaFields } = getSchemaFields(rows, keys)
-      name,
+
-    })
-    if (rows?.length > 0) {
-      for (let key of [...new Set(keys)] as string[]) {
-        const field = rows[0][key]
-        let type = typeof field,
-          fieldMetadata = makeQuerySchema(FieldType.STRING, key)
-        if (field)
-          switch (type) {
-            case "boolean":
-              fieldMetadata = makeQuerySchema(FieldType.BOOLEAN, key)
-              break
-            case "object":
-              if (field instanceof Date) {
-                fieldMetadata = makeQuerySchema(FieldType.DATETIME, key)
-              } else if (Array.isArray(field)) {
-                fieldMetadata = makeQuerySchema(FieldType.ARRAY, key)
-              } else {
-                fieldMetadata = makeQuerySchema(FieldType.JSON, key)
-              }
-              break
-            case "number":
-              fieldMetadata = makeQuerySchema(FieldType.NUMBER, key)
-              break
-          }
-        previewSchema[key] = fieldMetadata
-      }
-    }
     // if existing schema, update to include any previous schema keys
     if (existingSchema) {
       for (let key of Object.keys(previewSchema)) {
@@ -216,6 +241,7 @@ export async function preview(ctx: UserCtx) {
     await events.query.previewed(datasource, query)
     ctx.body = {
       rows,
+      nestedSchemaFields,
       schema: previewSchema,
       info,
       extra,

packages/server/src/api/routes/tests/queries/mysql.spec.ts

@@ -0,0 +1,239 @@
+import { Datasource, Query } from "@budibase/types"
+import * as setup from "../utilities"
+import { databaseTestProviders } from "../../../../integrations/tests/utils"
+import mysql from "mysql2/promise"
+
+jest.unmock("mysql2")
+jest.unmock("mysql2/promise")
+
+const createTableSQL = `
+CREATE TABLE test_table (
+    id INT AUTO_INCREMENT PRIMARY KEY,
+    name VARCHAR(50) NOT NULL
+)
+`
+
+const insertSQL = `
+INSERT INTO test_table (name) VALUES ('one'), ('two'), ('three'), ('four'), ('five')
+`
+
+const dropTableSQL = `
+DROP TABLE test_table
+`
+
+describe("/queries", () => {
+  let config = setup.getConfig()
+  let datasource: Datasource
+
+  async function createQuery(query: Partial<Query>): Promise<Query> {
+    const defaultQuery: Query = {
+      datasourceId: datasource._id!,
+      name: "New Query",
+      parameters: [],
+      fields: {},
+      schema: {},
+      queryVerb: "read",
+      transformer: "return data",
+      readable: true,
+    }
+    return await config.api.query.create({ ...defaultQuery, ...query })
+  }
+
+  async function withConnection(
+    callback: (client: mysql.Connection) => Promise<void>
+  ): Promise<void> {
+    const ds = await databaseTestProviders.mysql.datasource()
+    const con = await mysql.createConnection(ds.config!)
+    try {
+      await callback(con)
+    } finally {
+      con.end()
+    }
+  }
+
+  afterAll(async () => {
+    await databaseTestProviders.mysql.stop()
+    setup.afterAll()
+  })
+
+  beforeAll(async () => {
+    await config.init()
+    datasource = await config.api.datasource.create(
+      await databaseTestProviders.mysql.datasource()
+    )
+  })
+
+  beforeEach(async () => {
+    await withConnection(async connection => {
+      const resp = await connection.query(createTableSQL)
+      await connection.query(insertSQL)
+    })
+  })
+
+  afterEach(async () => {
+    await withConnection(async connection => {
+      await connection.query(dropTableSQL)
+    })
+  })
+
+  it("should execute a query", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "SELECT * FROM test_table ORDER BY id",
+      },
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      {
+        id: 1,
+        name: "one",
+      },
+      {
+        id: 2,
+        name: "two",
+      },
+      {
+        id: 3,
+        name: "three",
+      },
+      {
+        id: 4,
+        name: "four",
+      },
+      {
+        id: 5,
+        name: "five",
+      },
+    ])
+  })
+
+  it("should be able to transform a query", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "SELECT * FROM test_table WHERE id = 1",
+      },
+      transformer: `
+        data[0].id = data[0].id + 1;
+        return data;
+      `,
+    })
+
+    const result = await config.api.query.execute(query._id!)
+
+    expect(result.data).toEqual([
+      {
+        id: 2,
+        name: "one",
+      },
+    ])
+  })
+
+  it("should be able to insert with bindings", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "INSERT INTO test_table (name) VALUES ({{ foo }})",
+      },
+      parameters: [
+        {
+          name: "foo",
+          default: "bar",
+        },
+      ],
+      queryVerb: "create",
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: {
+        foo: "baz",
+      },
+    })
+
+    expect(result.data).toEqual([
+      {
+        created: true,
+      },
+    ])
+
+    await withConnection(async connection => {
+      const [rows] = await connection.query(
+        "SELECT * FROM test_table WHERE name = 'baz'"
+      )
+      expect(rows).toHaveLength(1)
+    })
+  })
+
+  it("should be able to update rows", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "UPDATE test_table SET name = {{ name }} WHERE id = {{ id }}",
+      },
+      parameters: [
+        {
+          name: "id",
+          default: "",
+        },
+        {
+          name: "name",
+          default: "updated",
+        },
+      ],
+      queryVerb: "update",
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: {
+        id: "1",
+        name: "foo",
+      },
+    })
+
+    expect(result.data).toEqual([
+      {
+        updated: true,
+      },
+    ])
+
+    await withConnection(async connection => {
+      const [rows] = await connection.query(
+        "SELECT * FROM test_table WHERE id = 1"
+      )
+      expect(rows).toEqual([{ id: 1, name: "foo" }])
+    })
+  })
+
+  it("should be able to delete rows", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "DELETE FROM test_table WHERE id = {{ id }}",
+      },
+      parameters: [
+        {
+          name: "id",
+          default: "",
+        },
+      ],
+      queryVerb: "delete",
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: {
+        id: "1",
+      },
+    })
+
+    expect(result.data).toEqual([
+      {
+        deleted: true,
+      },
+    ])
+
+    await withConnection(async connection => {
+      const [rows] = await connection.query(
+        "SELECT * FROM test_table WHERE id = 1"
+      )
+      expect(rows).toHaveLength(0)
+    })
+  })
+})

packages/server/src/api/routes/tests/queries/postgres.spec.ts

@@ -167,4 +167,77 @@ describe("/queries", () => {
       expect(rows).toHaveLength(1)
     })
   })
+
+  it("should be able to update rows", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "UPDATE test_table SET name = {{ name }} WHERE id = {{ id }}",
+      },
+      parameters: [
+        {
+          name: "id",
+          default: "",
+        },
+        {
+          name: "name",
+          default: "updated",
+        },
+      ],
+      queryVerb: "update",
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: {
+        id: "1",
+        name: "foo",
+      },
+    })
+
+    expect(result.data).toEqual([
+      {
+        updated: true,
+      },
+    ])
+
+    await withClient(async client => {
+      const { rows } = await client.query(
+        "SELECT * FROM test_table WHERE id = 1"
+      )
+      expect(rows).toEqual([{ id: 1, name: "foo" }])
+    })
+  })
+
+  it("should be able to delete rows", async () => {
+    const query = await createQuery({
+      fields: {
+        sql: "DELETE FROM test_table WHERE id = {{ id }}",
+      },
+      parameters: [
+        {
+          name: "id",
+          default: "",
+        },
+      ],
+      queryVerb: "delete",
+    })
+
+    const result = await config.api.query.execute(query._id!, {
+      parameters: {
+        id: "1",
+      },
+    })
+
+    expect(result.data).toEqual([
+      {
+        deleted: true,
+      },
+    ])
+
+    await withClient(async client => {
+      const { rows } = await client.query(
+        "SELECT * FROM test_table WHERE id = 1"
+      )
+      expect(rows).toHaveLength(0)
+    })
+  })
 })

packages/server/src/automations/actions.ts

@@ -9,6 +9,7 @@ import * as serverLog from "./steps/serverLog"
 import * as discord from "./steps/discord"
 import * as slack from "./steps/slack"
 import * as zapier from "./steps/zapier"
+import * as n8n from "./steps/n8n"
 import * as make from "./steps/make"
 import * as filter from "./steps/filter"
 import * as delay from "./steps/delay"
@@ -48,6 +49,7 @@ const ACTION_IMPLS: Record<
   slack: slack.run,
   zapier: zapier.run,
   integromat: make.run,
+  n8n: n8n.run,
 }
 export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
   {
@@ -70,6 +72,7 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<string, AutomationStepSchema> =
     slack: slack.definition,
     zapier: zapier.definition,
     integromat: make.definition,
+    n8n: n8n.definition,
   }
 
 // don't add the bash script/definitions unless in self host

packages/server/src/automations/bullboard.ts

@@ -15,7 +15,7 @@ const PATH_PREFIX = "/bulladmin"
 
 export async function init() {
   // Set up queues for bull board admin
-  const backupQueue = await backups.getBackupQueue()
+  const backupQueue = backups.getBackupQueue()
   const queues = [automationQueue]
   if (backupQueue) {
     queues.push(backupQueue)

packages/server/src/automations/steps/bash.ts

@@ -65,7 +65,7 @@ export async function run({ inputs, context }: AutomationStepInput) {
       success = true
     try {
       stdout = execSync(command, {
-        timeout: environment.QUERY_THREAD_TIMEOUT || 500,
+        timeout: environment.QUERY_THREAD_TIMEOUT,
       }).toString()
     } catch (err: any) {
       stdout = err.message

packages/server/src/automations/steps/executeQuery.ts

@@ -10,6 +10,8 @@ import {
   AutomationStepSchema,
   AutomationStepType,
 } from "@budibase/types"
+import { utils } from "@budibase/backend-core"
+import env from "../../environment"
 
 export const definition: AutomationStepSchema = {
   name: "External Data Connector",

packages/server/src/automations/steps/make.ts

@@ -34,28 +34,8 @@ export const definition: AutomationStepSchema = {
           type: AutomationIOType.JSON,
           title: "Payload",
         },
-        value1: {
-          type: AutomationIOType.STRING,
-          title: "Input Value 1",
-        },
-        value2: {
-          type: AutomationIOType.STRING,
-          title: "Input Value 2",
-        },
-        value3: {
-          type: AutomationIOType.STRING,
-          title: "Input Value 3",
-        },
-        value4: {
-          type: AutomationIOType.STRING,
-          title: "Input Value 4",
-        },
-        value5: {
-          type: AutomationIOType.STRING,
-          title: "Input Value 5",
-        },
       },
-      required: ["url", "value1", "value2", "value3", "value4", "value5"],
+      required: ["url", "body"],
     },
     outputs: {
       properties: {

packages/server/src/automations/steps/n8n.ts

@@ -0,0 +1,125 @@
+import fetch, { HeadersInit } from "node-fetch"
+import { getFetchResponse } from "./utils"
+import {
+  AutomationActionStepId,
+  AutomationStepSchema,
+  AutomationStepInput,
+  AutomationStepType,
+  AutomationIOType,
+  AutomationFeature,
+  HttpMethod,
+} from "@budibase/types"
+
+export const definition: AutomationStepSchema = {
+  name: "n8n Integration",
+  stepTitle: "n8n",
+  tagline: "Trigger an n8n workflow",
+  description:
+    "Performs a webhook call to n8n and gets the response (if configured)",
+  icon: "ri-shut-down-line",
+  stepId: AutomationActionStepId.n8n,
+  type: AutomationStepType.ACTION,
+  internal: false,
+  features: {
+    [AutomationFeature.LOOPING]: true,
+  },
+  inputs: {},
+  schema: {
+    inputs: {
+      properties: {
+        url: {
+          type: AutomationIOType.STRING,
+          title: "Webhook URL",
+        },
+        method: {
+          type: AutomationIOType.STRING,
+          title: "Method",
+          enum: Object.values(HttpMethod),
+        },
+        authorization: {
+          type: AutomationIOType.STRING,
+          title: "Authorization",
+        },
+        body: {
+          type: AutomationIOType.JSON,
+          title: "Payload",
+        },
+      },
+      required: ["url", "method"],
+    },
+    outputs: {
+      properties: {
+        success: {
+          type: AutomationIOType.BOOLEAN,
+          description: "Whether call was successful",
+        },
+        httpStatus: {
+          type: AutomationIOType.NUMBER,
+          description: "The HTTP status code returned",
+        },
+        response: {
+          type: AutomationIOType.OBJECT,
+          description: "The webhook response - this can have properties",
+        },
+      },
+      required: ["success", "response"],
+    },
+  },
+}
+
+export async function run({ inputs }: AutomationStepInput) {
+  const { url, body, method, authorization } = inputs
+
+  let payload = {}
+  try {
+    payload = body?.value ? JSON.parse(body?.value) : {}
+  } catch (err) {
+    return {
+      httpStatus: 400,
+      response: "Invalid payload JSON",
+      success: false,
+    }
+  }
+
+  if (!url?.trim()?.length) {
+    return {
+      httpStatus: 400,
+      response: "Missing Webhook URL",
+      success: false,
+    }
+  }
+  let response
+  let request: {
+    method: string
+    headers: HeadersInit
+    body?: string
+  } = {
+    method: method || HttpMethod.GET,
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: authorization,
+    },
+  }
+  if (!["GET", "HEAD"].includes(request.method)) {
+    request.body = JSON.stringify({
+      ...payload,
+    })
+  }
+
+  try {
+    response = await fetch(url, request)
+  } catch (err: any) {
+    return {
+      httpStatus: 400,
+      response: err.message,
+      success: false,
+    }
+  }
+
+  const { status, message } = await getFetchResponse(response)
+  return {
+    httpStatus: status,
+    success: status === 200,
+    response: message,
+  }
+}

packages/server/src/automations/steps/triggerAutomationRun.ts

@@ -9,8 +9,9 @@ import {
   AutomationCustomIOType,
 } from "@budibase/types"
 import * as triggers from "../triggers"
-import { db as dbCore, context } from "@budibase/backend-core"
+import { context } from "@budibase/backend-core"
 import { features } from "@budibase/pro"
+import env from "../../environment"
 
 export const definition: AutomationStepSchema = {
   name: "Trigger an automation",
@@ -76,7 +77,8 @@ export async function run({ inputs }: AutomationStepInput) {
         automation,
         {
           fields: { ...fieldParams },
-          timeout: inputs.timeout * 1000 || 120000,
+          timeout:
+            inputs.timeout * 1000 || env.getDefaults().AUTOMATION_SYNC_TIMEOUT,
         },
         { getResponses: true }
       )

packages/server/src/automations/steps/zapier.ts

@@ -32,26 +32,6 @@ export const definition: AutomationStepSchema = {
           type: AutomationIOType.JSON,
           title: "Payload",
         },
-        value1: {
-          type: AutomationIOType.STRING,
-          title: "Payload Value 1",
-        },
-        value2: {
-          type: AutomationIOType.STRING,
-          title: "Payload Value 2",
-        },
-        value3: {
-          type: AutomationIOType.STRING,
-          title: "Payload Value 3",
-        },
-        value4: {
-          type: AutomationIOType.STRING,
-          title: "Payload Value 4",
-        },
-        value5: {
-          type: AutomationIOType.STRING,
-          title: "Payload Value 5",
-        },
       },
       required: ["url"],
     },

packages/server/src/automations/tests/n8n.spec.ts

@@ -0,0 +1,68 @@
+import { getConfig, afterAll, runStep, actions } from "./utilities"
+
+describe("test the outgoing webhook action", () => {
+  let config = getConfig()
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  afterAll()
+
+  it("should be able to run the action and default to 'get'", async () => {
+    const res = await runStep(actions.n8n.stepId, {
+      url: "http://www.example.com",
+      body: {
+        test: "IGNORE_ME",
+      },
+    })
+    expect(res.response.url).toEqual("http://www.example.com")
+    expect(res.response.method).toEqual("GET")
+    expect(res.response.body).toBeUndefined()
+    expect(res.success).toEqual(true)
+  })
+
+  it("should add the payload props when a JSON string is provided", async () => {
+    const payload = `{ "name": "Adam", "age": 9 }`
+    const res = await runStep(actions.n8n.stepId, {
+      body: {
+        value: payload,
+      },
+      method: "POST",
+      url: "http://www.example.com",
+    })
+    expect(res.response.url).toEqual("http://www.example.com")
+    expect(res.response.method).toEqual("POST")
+    expect(res.response.body).toEqual(`{"name":"Adam","age":9}`)
+    expect(res.success).toEqual(true)
+  })
+
+  it("should return a 400 if the JSON payload string is malformed", async () => {
+    const payload = `{ value1 1 }`
+    const res = await runStep(actions.n8n.stepId, {
+      value1: "ONE",
+      body: {
+        value: payload,
+      },
+      method: "POST",
+      url: "http://www.example.com",
+    })
+    expect(res.httpStatus).toEqual(400)
+    expect(res.response).toEqual("Invalid payload JSON")
+    expect(res.success).toEqual(false)
+  })
+
+  it("should not append the body if the method is HEAD", async () => {
+    const res = await runStep(actions.n8n.stepId, {
+      url: "http://www.example.com",
+      method: "HEAD",
+      body: {
+        test: "IGNORE_ME",
+      },
+    })
+    expect(res.response.url).toEqual("http://www.example.com")
+    expect(res.response.method).toEqual("HEAD")
+    expect(res.response.body).toBeUndefined()
+    expect(res.success).toEqual(true)
+  })
+})

packages/server/src/automations/tests/triggerAutomationRun.spec.ts

@@ -3,6 +3,7 @@ jest.spyOn(global.console, "error")
 import * as setup from "./utilities"
 import * as automation from "../index"
 import { serverLogAutomation } from "../../tests/utilities/structures"
+import env from "../../environment"
 
 describe("Test triggering an automation from another automation", () => {
   let config = setup.getConfig()
@@ -22,7 +23,10 @@ describe("Test triggering an automation from another automation", () => {
     let newAutomation = await config.createAutomation(automation)
 
     const inputs: any = {
-      automation: { automationId: newAutomation._id, timeout: 12000 },
+      automation: {
+        automationId: newAutomation._id,
+        timeout: env.getDefaults().AUTOMATION_THREAD_TIMEOUT,
+      },
     }
     const res = await setup.runStep(
       setup.actions.TRIGGER_AUTOMATION_RUN.stepId,
@@ -33,7 +37,12 @@ describe("Test triggering an automation from another automation", () => {
   })
 
   it("should fail gracefully if the automation id is incorrect", async () => {
-    const inputs: any = { automation: { automationId: null, timeout: 12000 } }
+    const inputs: any = {
+      automation: {
+        automationId: null,
+        timeout: env.getDefaults().AUTOMATION_THREAD_TIMEOUT,
+      },
+    }
     const res = await setup.runStep(
       setup.actions.TRIGGER_AUTOMATION_RUN.stepId,
       inputs

packages/server/src/environment.ts

@@ -18,6 +18,21 @@ function parseIntSafe(number?: string) {
   }
 }
 
+const DEFAULTS = {
+  QUERY_THREAD_TIMEOUT: 10000,
+  AUTOMATION_THREAD_TIMEOUT: 12000,
+  AUTOMATION_SYNC_TIMEOUT: 120000,
+  AUTOMATION_MAX_ITERATIONS: 200,
+  JS_PER_EXECUTION_TIME_LIMIT_MS: 1000,
+  TEMPLATE_REPOSITORY: "app",
+  PLUGINS_DIR: "/plugins",
+  FORKED_PROCESS_NAME: "main",
+  JS_RUNNER_MEMORY_LIMIT: 64,
+}
+
+const QUERY_THREAD_TIMEOUT =
+  parseIntSafe(process.env.QUERY_THREAD_TIMEOUT) ||
+  DEFAULTS.QUERY_THREAD_TIMEOUT
 const environment = {
   // features
   APP_FEATURES: process.env.APP_FEATURES,
@@ -42,7 +57,8 @@ const environment = {
   JEST_WORKER_ID: process.env.JEST_WORKER_ID,
   BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
   DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
-  TEMPLATE_REPOSITORY: process.env.TEMPLATE_REPOSITORY || "app",
+  TEMPLATE_REPOSITORY:
+    process.env.TEMPLATE_REPOSITORY || DEFAULTS.TEMPLATE_REPOSITORY,
   DISABLE_AUTO_PROD_APP_SYNC: process.env.DISABLE_AUTO_PROD_APP_SYNC,
   SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD,
   // minor
@@ -50,14 +66,20 @@ const environment = {
   LOGGER: process.env.LOGGER,
   ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
   AUTOMATION_MAX_ITERATIONS:
-    parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) || 200,
+    parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) ||
+    DEFAULTS.AUTOMATION_MAX_ITERATIONS,
   SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
   DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
-  QUERY_THREAD_TIMEOUT: parseIntSafe(process.env.QUERY_THREAD_TIMEOUT),
+  QUERY_THREAD_TIMEOUT: QUERY_THREAD_TIMEOUT,
+  AUTOMATION_THREAD_TIMEOUT:
+    parseIntSafe(process.env.AUTOMATION_THREAD_TIMEOUT) ||
+    DEFAULTS.AUTOMATION_THREAD_TIMEOUT > QUERY_THREAD_TIMEOUT
+      ? DEFAULTS.AUTOMATION_THREAD_TIMEOUT
+      : QUERY_THREAD_TIMEOUT,
   SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
   BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
   BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
-  PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins",
+  PLUGINS_DIR: process.env.PLUGINS_DIR || DEFAULTS.PLUGINS_DIR,
   OPENAI_API_KEY: process.env.OPENAI_API_KEY,
   MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB,
   SESSION_EXPIRY_SECONDS: process.env.SESSION_EXPIRY_SECONDS,
@@ -70,12 +92,21 @@ const environment = {
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   SELF_HOSTED: process.env.SELF_HOSTED,
   HTTP_MB_LIMIT: process.env.HTTP_MB_LIMIT,
-  FORKED_PROCESS_NAME: process.env.FORKED_PROCESS_NAME || "main",
+  FORKED_PROCESS_NAME:
+    process.env.FORKED_PROCESS_NAME || DEFAULTS.FORKED_PROCESS_NAME,
   JS_PER_INVOCATION_TIMEOUT_MS:
-    parseIntSafe(process.env.JS_PER_EXECUTION_TIME_LIMIT_MS) || 1000,
+    parseIntSafe(process.env.JS_PER_EXECUTION_TIME_LIMIT_MS) ||
+    DEFAULTS.JS_PER_EXECUTION_TIME_LIMIT_MS,
   JS_PER_REQUEST_TIMEOUT_MS: parseIntSafe(
     process.env.JS_PER_REQUEST_TIME_LIMIT_MS
   ),
+  TOP_LEVEL_PATH:
+    process.env.TOP_LEVEL_PATH || process.env.SERVER_TOP_LEVEL_PATH,
+  APP_MIGRATION_TIMEOUT: parseIntSafe(process.env.APP_MIGRATION_TIMEOUT),
+  JS_RUNNER_MEMORY_LIMIT:
+    parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) ||
+    DEFAULTS.JS_RUNNER_MEMORY_LIMIT,
+  LOG_JS_ERRORS: process.env.LOG_JS_ERRORS,
   // old
   CLIENT_ID: process.env.CLIENT_ID,
   _set(key: string, value: any) {
@@ -92,12 +123,9 @@ const environment = {
   isInThread: () => {
     return process.env.FORKED_PROCESS
   },
-  TOP_LEVEL_PATH:
+  getDefaults: () => {
-    process.env.TOP_LEVEL_PATH || process.env.SERVER_TOP_LEVEL_PATH,
+    return DEFAULTS
-  APP_MIGRATION_TIMEOUT: parseIntSafe(process.env.APP_MIGRATION_TIMEOUT),
+  },
-  JS_RUNNER_MEMORY_LIMIT:
-    parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) || 64,
-  LOG_JS_ERRORS: process.env.LOG_JS_ERRORS,
 }
 
 // clean up any environment variable edge cases

packages/server/src/integrations/rest.ts

@@ -10,6 +10,7 @@ import {
   RestAuthType,
   RestBasicAuthConfig,
   RestBearerAuthConfig,
+  HttpMethod,
 } from "@budibase/types"
 import get from "lodash/get"
 import * as https from "https"
@@ -86,30 +87,30 @@ const SCHEMA: Integration = {
   query: {
     create: {
       readable: true,
-      displayName: "POST",
+      displayName: HttpMethod.POST,
       type: QueryType.FIELDS,
       fields: coreFields,
     },
     read: {
-      displayName: "GET",
+      displayName: HttpMethod.GET,
       readable: true,
       type: QueryType.FIELDS,
       fields: coreFields,
     },
     update: {
-      displayName: "PUT",
+      displayName: HttpMethod.PUT,
       readable: true,
       type: QueryType.FIELDS,
       fields: coreFields,
     },
     patch: {
-      displayName: "PATCH",
+      displayName: HttpMethod.PATCH,
       readable: true,
       type: QueryType.FIELDS,
       fields: coreFields,
     },
     delete: {
-      displayName: "DELETE",
+      displayName: HttpMethod.DELETE,
       type: QueryType.FIELDS,
       fields: coreFields,
     },
@@ -358,7 +359,7 @@ class RestIntegration implements IntegrationBase {
       path = "",
       queryString = "",
       headers = {},
-      method = "GET",
+      method = HttpMethod.GET,
       disabledHeaders,
       bodyType,
       requestBody,
@@ -413,23 +414,23 @@ class RestIntegration implements IntegrationBase {
   }
 
   async create(opts: RestQuery) {
-    return this._req({ ...opts, method: "POST" })
+    return this._req({ ...opts, method: HttpMethod.POST })
   }
 
   async read(opts: RestQuery) {
-    return this._req({ ...opts, method: "GET" })
+    return this._req({ ...opts, method: HttpMethod.GET })
   }
 
   async update(opts: RestQuery) {
-    return this._req({ ...opts, method: "PUT" })
+    return this._req({ ...opts, method: HttpMethod.PUT })
   }
 
   async patch(opts: RestQuery) {
-    return this._req({ ...opts, method: "PATCH" })
+    return this._req({ ...opts, method: HttpMethod.PATCH })
   }
 
   async delete(opts: RestQuery) {
-    return this._req({ ...opts, method: "DELETE" })
+    return this._req({ ...opts, method: HttpMethod.DELETE })
   }
 }
 

packages/server/src/integrations/tests/utils/index.ts

@@ -3,6 +3,7 @@ jest.unmock("pg")
 import { Datasource } from "@budibase/types"
 import * as postgres from "./postgres"
 import * as mongodb from "./mongodb"
+import * as mysql from "./mysql"
 import { StartedTestContainer } from "testcontainers"
 
 jest.setTimeout(30000)
@@ -13,4 +14,4 @@ export interface DatabaseProvider {
   datasource(): Promise<Datasource>
 }
 
-export const databaseTestProviders = { postgres, mongodb }
+export const databaseTestProviders = { postgres, mongodb, mysql }

packages/server/src/integrations/tests/utils/mongodb.ts

@@ -11,7 +11,9 @@ export async function start(): Promise<StartedTestContainer> {
       MONGO_INITDB_ROOT_PASSWORD: "password",
     })
     .withWaitStrategy(
-      Wait.forSuccessfulCommand(`mongosh --eval "db.version()"`)
+      Wait.forSuccessfulCommand(
+        `mongosh --eval "db.version()"`
+      ).withStartupTimeout(10000)
     )
     .start()
 }

packages/server/src/integrations/tests/utils/mysql.ts

@@ -0,0 +1,61 @@
+import { Datasource, SourceName } from "@budibase/types"
+import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
+
+let container: StartedTestContainer | undefined
+
+class MySQLWaitStrategy extends AbstractWaitStrategy {
+  async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
+    // Because MySQL first starts itself up, runs an init script, then restarts,
+    // it's possible for the mysqladmin ping to succeed early and then tests to
+    // run against a MySQL that's mid-restart and fail. To get around this, we
+    // wait for logs and then do a ping check.
+
+    const logs = Wait.forLogMessage(
+      "/usr/sbin/mysqld: ready for connections",
+      2
+    )
+    await logs.waitUntilReady(container, boundPorts, startTime)
+
+    const command = Wait.forSuccessfulCommand(
+      `mysqladmin ping -h localhost -P 3306 -u root -ppassword`
+    )
+    await command.waitUntilReady(container)
+  }
+}
+
+export async function start(): Promise<StartedTestContainer> {
+  return await new GenericContainer("mysql:8.3")
+    .withExposedPorts(3306)
+    .withEnvironment({ MYSQL_ROOT_PASSWORD: "password" })
+    .withWaitStrategy(new MySQLWaitStrategy().withStartupTimeout(10000))
+    .start()
+}
+
+export async function datasource(): Promise<Datasource> {
+  if (!container) {
+    container = await start()
+  }
+  const host = container.getHost()
+  const port = container.getMappedPort(3306)
+
+  return {
+    type: "datasource_plus",
+    source: SourceName.MYSQL,
+    plus: true,
+    config: {
+      host,
+      port,
+      user: "root",
+      password: "password",
+      database: "mysql",
+    },
+  }
+}
+
+export async function stop() {
+  if (container) {
+    await container.stop()
+    container = undefined
+  }
+}

packages/server/src/threads/automation.ts

@@ -303,7 +303,7 @@ class Orchestrator {
             if (timeout) {
               setTimeout(() => {
                 timeoutFlag = true
-              }, timeout || 12000)
+              }, timeout || env.AUTOMATION_THREAD_TIMEOUT)
             }
 
             stepCount++
@@ -621,7 +621,7 @@ export async function executeInThread(job: Job<AutomationData>) {
   const timeoutPromise = new Promise((resolve, reject) => {
     setTimeout(() => {
       reject(new Error("Timeout exceeded"))
-    }, job.data.event.timeout || 12000)
+    }, job.data.event.timeout || env.AUTOMATION_THREAD_TIMEOUT)
   })
 
   return await context.doInAppContext(appId, async () => {

packages/server/src/utilities/workerRequests.ts

@@ -1,4 +1,10 @@
-import { Response, default as fetch } from "node-fetch"
+import {
+  Response,
+  default as fetch,
+  type RequestInit,
+  Headers,
+  HeadersInit,
+} from "node-fetch"
 import env from "../environment"
 import { checkSlashesInUrl } from "./index"
 import {
@@ -7,36 +13,62 @@ import {
   tenancy,
   logging,
   env as coreEnv,
+  utils,
 } from "@budibase/backend-core"
 import { Ctx, User, EmailInvite } from "@budibase/types"
 
-export function request(ctx?: Ctx, request?: any) {
+interface Request {
-  if (!request.headers) {
+  ctx?: Ctx
-    request.headers = {}
+  method: "GET" | "POST" | "PUT" | "DELETE" | "PATCH"
+  headers?: { [key: string]: string }
+  body?: { [key: string]: any }
+}
+
+export function createRequest(request: Request): RequestInit {
+  const headers: Record<string, string> = {}
+  const requestInit: RequestInit = {
+    method: request.method,
   }
-  if (!ctx) {
+
-    request.headers[constants.Header.API_KEY] = coreEnv.INTERNAL_API_KEY
+  const ctx = request.ctx
-    if (tenancy.isTenantIdSet()) {
+
-      request.headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
+  if (!ctx && coreEnv.INTERNAL_API_KEY) {
+    headers[constants.Header.API_KEY] = coreEnv.INTERNAL_API_KEY
+  } else if (ctx && ctx.headers) {
+    // copy all Budibase utilised headers over - copying everything can have
+    // side effects like requests being rejected due to odd content types etc
+    for (let header of Object.values(constants.Header)) {
+      const value = ctx.headers[header]
+      if (value === undefined) {
+        continue
+      }
+      headers[header] = Array.isArray(value) ? value[0] : value
+    }
+    // be specific about auth headers
+    const cookie = ctx.headers[constants.Header.COOKIE],
+      apiKey = ctx.headers[constants.Header.API_KEY]
+    if (cookie) {
+      headers[constants.Header.COOKIE] = cookie
+    } else if (apiKey) {
+      headers[constants.Header.API_KEY] = Array.isArray(apiKey)
+        ? apiKey[0]
+        : apiKey
     }
   }
+
+  // apply tenancy if its available
+  if (tenancy.isTenantIdSet()) {
+    headers[constants.Header.TENANT_ID] = tenancy.getTenantId()
+  }
+
   if (request.body && Object.keys(request.body).length > 0) {
-    request.headers["Content-Type"] = "application/json"
+    headers["Content-Type"] = "application/json"
-    request.body =
+    requestInit.body = JSON.stringify(request.body)
-      typeof request.body === "object"
-        ? JSON.stringify(request.body)
-        : request.body
-  } else {
-    delete request.body
-  }
-  if (ctx && ctx.headers) {
-    request.headers = ctx.headers
   }
 
-  // add x-budibase-correlation-id header
+  logging.correlation.setHeader(headers)
-  logging.correlation.setHeader(request.headers)
+  requestInit.headers = headers
-
+  return requestInit
-  return request
 }
 
 async function checkResponse(
@@ -54,7 +86,7 @@ async function checkResponse(
     }
     const msg = `Unable to ${errorMsg} - ${responseErrorMessage}`
     if (ctx) {
-      ctx.throw(msg, response.status)
+      ctx.throw(response.status || 500, msg)
     } else {
       throw msg
     }
@@ -85,7 +117,7 @@ export async function sendSmtpEmail({
   // tenant ID will be set in header
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + `/api/global/email/send`),
-    request(undefined, {
+    createRequest({
       method: "POST",
       body: {
         email: to,
@@ -107,7 +139,8 @@ export async function removeAppFromUserRoles(ctx: Ctx, appId: string) {
   const prodAppId = dbCore.getProdAppID(appId)
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + `/api/global/roles/${prodAppId}`),
-    request(ctx, {
+    createRequest({
+      ctx,
       method: "DELETE",
     })
   )
@@ -118,7 +151,7 @@ export async function allGlobalUsers(ctx: Ctx) {
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + "/api/global/users"),
     // we don't want to use API key when getting self
-    request(ctx, { method: "GET" })
+    createRequest({ ctx, method: "GET" })
   )
   return checkResponse(response, "get users", { ctx })
 }
@@ -127,7 +160,7 @@ export async function saveGlobalUser(ctx: Ctx) {
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + "/api/global/users"),
     // we don't want to use API key when getting self
-    request(ctx, { method: "POST", body: ctx.request.body })
+    createRequest({ ctx, method: "POST", body: ctx.request.body })
   )
   return checkResponse(response, "save user", { ctx })
 }
@@ -138,7 +171,7 @@ export async function deleteGlobalUser(ctx: Ctx) {
       env.WORKER_URL + `/api/global/users/${ctx.params.userId}`
     ),
     // we don't want to use API key when getting self
-    request(ctx, { method: "DELETE" })
+    createRequest({ ctx, method: "DELETE" })
   )
   return checkResponse(response, "delete user", { ctx })
 }
@@ -149,7 +182,7 @@ export async function readGlobalUser(ctx: Ctx): Promise<User> {
       env.WORKER_URL + `/api/global/users/${ctx.params.userId}`
     ),
     // we don't want to use API key when getting self
-    request(ctx, { method: "GET" })
+    createRequest({ ctx, method: "GET" })
   )
   return checkResponse(response, "get user", { ctx })
 }
@@ -159,7 +192,7 @@ export async function getChecklist(): Promise<{
 }> {
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + "/api/global/configs/checklist"),
-    request(undefined, { method: "GET" })
+    createRequest({ method: "GET" })
   )
   return checkResponse(response, "get checklist")
 }
@@ -167,7 +200,7 @@ export async function getChecklist(): Promise<{
 export async function generateApiKey(userId: string) {
   const response = await fetch(
     checkSlashesInUrl(env.WORKER_URL + "/api/global/self/api_key"),
-    request(undefined, { method: "POST", body: { userId } })
+    createRequest({ method: "POST", body: { userId } })
   )
   return checkResponse(response, "generate API key")
 }

packages/shared-core/src/constants/api.ts

@@ -16,4 +16,5 @@ export enum Header {
   CORRELATION_ID = "x-budibase-correlation-id",
   AUTHORIZATION = "authorization",
   MIGRATING_APP = "x-budibase-migrating-app",
+  COOKIE = "cookie",
 }

packages/shared-core/src/utils.ts

@@ -57,3 +57,13 @@ export function filterValueToLabel() {
     {}
   )
 }
+
+export function hasSchema(test: any) {
+  return (
+    typeof test === "object" &&
+    !Array.isArray(test) &&
+    test !== null &&
+    !(test instanceof Date) &&
+    Object.keys(test).length > 0
+  )
+}

packages/types/src/documents/app/automation.ts

@@ -69,6 +69,7 @@ export enum AutomationActionStepId {
   slack = "slack",
   zapier = "zapier",
   integromat = "integromat",
+  n8n = "n8n",
 }
 
 export interface EmailInvite {

packages/types/src/documents/app/query.ts

@@ -4,6 +4,7 @@ import type { Row } from "./row"
 export interface QuerySchema {
   name?: string
   type: string
+  subtype?: string
 }
 
 export interface Query extends Document {
@@ -17,11 +18,23 @@ export interface Query extends Document {
   queryVerb: string
 }
 
+export interface QueryPreview extends Omit<Query, "_id"> {
+  queryId: string
+}
+
 export interface QueryParameter {
   name: string
   default: string
 }
 
+export interface QueryResponse {
+  rows: any[]
+  keys: string[]
+  info: any
+  extra: any
+  pagination: any
+}
+
 export interface RestQueryFields {
   path: string
   queryString?: string
@@ -64,3 +77,12 @@ export interface ExecuteQueryRequest {
 export interface ExecuteQueryResponse {
   data: Row[]
 }
+
+export enum HttpMethod {
+  GET = "GET",
+  POST = "POST",
+  PATCH = "PATCH",
+  PUT = "PUT",
+  HEAD = "HEAD",
+  DELETE = "DELETE",
+}

packages/types/src/documents/app/table/constants.ts

@@ -16,6 +16,10 @@ export enum AutoFieldSubType {
   AUTO_ID = "autoID",
 }
 
+export enum JsonFieldSubType {
+  ARRAY = "array",
+}
+
 export enum FormulaType {
   STATIC = "static",
   DYNAMIC = "dynamic",

packages/types/src/documents/app/table/schema.ts

@@ -5,6 +5,7 @@ import {
   AutoFieldSubType,
   AutoReason,
   FormulaType,
+  JsonFieldSubType,
   RelationshipType,
 } from "./constants"
 
@@ -81,6 +82,11 @@ export interface NumberFieldMetadata extends Omit<BaseFieldSchema, "subtype"> {
   }
 }
 
+export interface JsonFieldMetadata extends Omit<BaseFieldSchema, "subtype"> {
+  type: FieldType.JSON
+  subtype?: JsonFieldSubType.ARRAY
+}
+
 export interface DateFieldMetadata extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.DATETIME
   ignoreTimezones?: boolean
@@ -162,6 +168,7 @@ export type FieldSchema =
   | NumberFieldMetadata
   | LongFormFieldMetadata
   | BBReferenceFieldMetadata
+  | JsonFieldMetadata
 
 export interface TableSchema {
   [key: string]: FieldSchema

packages/worker/Dockerfile

@@ -37,6 +37,7 @@ RUN apk del .gyp \
 
 COPY packages/worker/dist/ dist/
 COPY packages/worker/docker_run.sh .
+COPY packages/server/pm2.config.js .
 
 EXPOSE 4001
 

yarn.lock

@@ -5572,9 +5572,9 @@
   integrity sha512-7GgtHCs/QZrBrDzgIJnQtuSvhFSwhyYSI2uafSwZoNt1iOGhEN5fwNrQMjtONyHm9+/LoA4453jH0CMYcr06Pg==
 
 "@types/node@^18.11.18":
-  version "18.19.10"
+  version "18.19.13"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.10.tgz#4de314ab66faf6bc8ba691021a091ddcdf13a158"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.13.tgz#c3e989ca967b862a1f6c8c4148fe31865eedaf1a"
-  integrity sha512-IZD8kAM02AW1HRDTPOlz3npFava678pr8Ie9Vp8uRhBROXAv8MXT2pCnGZZAKYdromsNQLHQcfWQ6EOatVLtqA==
+  integrity sha512-kgnbRDj8ioDyGxoiaXsiu1Ybm/K14ajCgMOkwiqpHrnF7d7QiYRoRqHIpglMMs3DwXinlK4qJ8TZGlj4hfleJg==
   dependencies:
     undici-types "~5.26.4"
 
@@ -10763,7 +10763,7 @@ fetch-cookie@0.11.0:
   dependencies:
     tough-cookie "^2.3.3 || ^3.0.1 || ^4.0.0"
 
-fflate@^0.4.1:
+fflate@^0.4.1, fflate@^0.4.8:
   version "0.4.8"
   resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.4.8.tgz#f90b82aefbd8ac174213abb338bd7ef848f0f5ae"
   integrity sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA==