Removing outgoing webhook functionality from automations and adding XML response/sending, as well as form data/encoded data.

packages/builder/src/components/common/CodeMirrorEditor.svelte

@@ -8,6 +8,9 @@
       name: "javascript",
       json: true,
     },
+    XML: {
+      name: "xml",
+    },
     SQL: {
       name: "sql",
     },
@@ -17,7 +20,6 @@
     },
     Text: {
       name: "text/html",
-      json: false,
     },
   }
 </script>

packages/builder/src/components/integration/codemirror.js

@@ -4,6 +4,7 @@ import "codemirror/lib/codemirror.css"
 // Modes
 import "codemirror/mode/javascript/javascript"
 import "codemirror/mode/sql/sql"
+import "codemirror/mode/xml/xml"
 import "codemirror/mode/css/css"
 import "codemirror/mode/handlebars/handlebars"
 

packages/builder/src/constants/backend/index.js

@@ -200,6 +200,7 @@ export const RawRestBodyTypes = {
   ENCODED: "encoded",
   JSON: "json",
   TEXT: "text",
+  XML: "xml",
 }
 
 export const RestBodyTypes = [
@@ -207,5 +208,6 @@ export const RestBodyTypes = [
   { name: "form-data", value: "form" },
   { name: "x-www-form-encoded", value: "encoded" },
   { name: "raw (JSON)", value: "json" },
+  { name: "raw (XML)", value: "xml" },
   { name: "raw (Text)", value: "text" },
 ]

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/RestBodyInput.svelte

@@ -7,7 +7,11 @@
   } from "components/common/CodeMirrorEditor.svelte"
 
   const objectTypes = [RawRestBodyTypes.FORM, RawRestBodyTypes.ENCODED]
-  const textTypes = [RawRestBodyTypes.JSON, RawRestBodyTypes.TEXT]
+  const textTypes = [
+    RawRestBodyTypes.JSON,
+    RawRestBodyTypes.XML,
+    RawRestBodyTypes.TEXT,
+  ]
 
   export let query
   export let bodyType
@@ -25,6 +29,18 @@
       query.fields.requestBody = ""
     }
   }
+
+  function editorMode(type) {
+    switch (type) {
+      case RawRestBodyTypes.JSON:
+        return EditorModes.JSON
+      case RawRestBodyTypes.XML:
+        return EditorModes.XML
+      default:
+      case RawRestBodyTypes.TEXT:
+        return EditorModes.Text
+    }
+  }
 </script>
 
 <div class="margin">
@@ -41,9 +57,7 @@
   {:else if textTypes.includes(bodyType)}
     <CodeMirrorEditor
       height={200}
-      mode={bodyType === RawRestBodyTypes.JSON
+      mode={editorMode(bodyType)}
-        ? EditorModes.JSON
-        : EditorModes.Text}
       value={query.fields.requestBody}
       resize="vertical"
       on:change={e => (query.fields.requestBody = e.detail)}

packages/server/package.json

@@ -90,6 +90,7 @@
     "dotenv": "8.2.0",
     "download": "8.0.0",
     "fix-path": "3.0.0",
+    "form-data": "^4.0.0",
     "fs-extra": "8.1.0",
     "jimp": "0.16.1",
     "joi": "17.2.1",
@@ -126,6 +127,7 @@
     "validate.js": "0.13.1",
     "vm2": "^3.9.3",
     "worker-farm": "^1.7.0",
+    "xml2js": "^0.4.23",
     "yargs": "13.2.4",
     "zlib": "1.0.5"
   },

packages/server/src/api/controllers/automation.js

@@ -5,11 +5,15 @@ const { getAutomationParams, generateAutomationID } = require("../../db/utils")
 const {
   checkForWebhooks,
   updateTestHistory,
+  removeDeprecated,
 } = require("../../automations/utils")
 const { deleteEntityMetadata } = require("../../utilities")
 const { MetadataTypes } = require("../../constants")
 const { setTestFlag, clearTestFlag } = require("../../utilities/redis")
 
+const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS)
+const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
+
 /*************************
  *                       *
  *   BUILDER FUNCTIONS   *
@@ -155,17 +159,17 @@ exports.destroy = async function (ctx) {
 }
 
 exports.getActionList = async function (ctx) {
-  ctx.body = actions.ACTION_DEFINITIONS
+  ctx.body = ACTION_DEFS
 }
 
 exports.getTriggerList = async function (ctx) {
-  ctx.body = triggers.TRIGGER_DEFINITIONS
+  ctx.body = TRIGGER_DEFS
 }
 
 module.exports.getDefinitionList = async function (ctx) {
   ctx.body = {
-    trigger: triggers.TRIGGER_DEFINITIONS,
+    trigger: TRIGGER_DEFS,
-    action: actions.ACTION_DEFINITIONS,
+    action: ACTION_DEFS,
   }
 }
 

packages/server/src/automations/steps/outgoingWebhook.js

@@ -13,12 +13,11 @@ const RequestType = {
 const BODY_REQUESTS = [RequestType.POST, RequestType.PUT, RequestType.PATCH]
 
 /**
- * Note, there is some functionality in this that is not currently exposed as it
+ * NOTE: this functionality is deprecated - it no longer should be used.
- * is complex and maybe better to be opinionated here.
- * GET/DELETE requests cannot handle body elements so they will not be sent if configured.
  */
 
 exports.definition = {
+  deprecated: true,
   name: "Outgoing webhook",
   tagline: "Send a {{inputs.requestMethod}} request",
   icon: "Send",

packages/server/src/automations/utils.js

@@ -7,6 +7,7 @@ const newid = require("../db/newid")
 const { updateEntityMetadata } = require("../utilities")
 const { MetadataTypes } = require("../constants")
 const { getDeployedAppID } = require("@budibase/auth/db")
+const { cloneDeep } = require("lodash/fp")
 
 const WH_STEP_ID = definitions.WEBHOOK.stepId
 const CRON_STEP_ID = definitions.CRON.stepId
@@ -42,6 +43,16 @@ exports.updateTestHistory = async (appId, automation, history) => {
   )
 }
 
+exports.removeDeprecated = definitions => {
+  const base = cloneDeep(definitions)
+  for (let key of Object.keys(base)) {
+    if (base[key].deprecated) {
+      delete base[key]
+    }
+  }
+  return base
+}
+
 // end the repetition and the job itself
 exports.disableAllCrons = async appId => {
   const promises = []

packages/server/src/integrations/rest.ts

@@ -10,6 +10,7 @@ import { IntegrationBase } from "./base/IntegrationBase"
 const BodyTypes = {
   NONE: "none",
   FORM_DATA: "form",
+  XML: "xml",
   ENCODED: "encoded",
   JSON: "json",
   TEXT: "text",
@@ -42,6 +43,9 @@ module RestModule {
   const fetch = require("node-fetch")
   const { formatBytes } = require("../utilities")
   const { performance } = require("perf_hooks")
+  const FormData = require("form-data")
+  const { URLSearchParams } = require("url")
+  const xmlParser = require("xml2js").parseStringPromise
 
   const SCHEMA: Integration = {
     docs: "https://github.com/node-fetch/node-fetch",
@@ -107,14 +111,27 @@ module RestModule {
 
     async parseResponse(response: any) {
       let data, raw, headers
-      const contentType = response.headers.get("content-type")
+      const contentType = response.headers.get("content-type") || ""
-      if (contentType && contentType.indexOf("application/json") !== -1) {
+      try {
+        if (contentType.includes("application/json")) {
           data = await response.json()
           raw = JSON.stringify(data)
+        } else if (contentType.includes("text/xml") || contentType.includes("application/xml")) {
+          const rawXml = await response.text()
+          data = await xmlParser(rawXml, { explicitArray: false, trim: true, explicitRoot: false }) || {}
+          // there is only one structure, its an array, return the array so it appears as rows
+          const keys = Object.keys(data)
+          if (keys.length === 1 && Array.isArray(data[keys[0]])) {
+            data = data[keys[0]]
+          }
+          raw = rawXml
         } else {
           data = await response.text()
           raw = data
         }
+      } catch (err) {
+        throw "Failed to parse response body."
+      }
       const size = formatBytes(
         response.headers.get("content-length") || Buffer.byteLength(raw, "utf8")
       )
@@ -149,6 +166,50 @@ module RestModule {
       return complete
     }
 
+    addBody(bodyType: string, body: string | any, input: any) {
+      let error, object, string
+      try {
+        string = typeof body !== "string" ? JSON.stringify(body) : body
+        object = typeof body === "object" ? body : JSON.parse(body)
+      } catch (err) {
+        error = err
+      }
+      switch (bodyType) {
+        case BodyTypes.TEXT:
+          // content type defaults to plaintext
+          input.body = string
+          break
+        case BodyTypes.ENCODED:
+          const params = new URLSearchParams()
+          for (let [key, value] of Object.entries(object)) {
+            params.append(key, value)
+          }
+          input.body = params
+          break
+        case BodyTypes.FORM_DATA:
+          const form = new FormData()
+          for (let [key, value] of Object.entries(object)) {
+            form.append(key, value)
+          }
+          input.body = form
+          break
+        case BodyTypes.XML:
+          input.body = string
+          input.headers["Content-Type"] = "text/xml"
+          break
+        default:
+        case BodyTypes.JSON:
+          // if JSON error, throw it
+          if (error) {
+            throw "Invalid JSON for request body"
+          }
+          input.body = object
+          input.headers["Content-Type"] = "application/json"
+          break
+      }
+      return input
+    }
+
     async _req(query: RestQuery) {
       const {
         path = "",
@@ -172,35 +233,9 @@ module RestModule {
         }
       }
 
-      const input: any = { method, headers: this.headers }
+      let input: any = { method, headers: this.headers }
       if (requestBody) {
-        switch (bodyType) {
+        input = this.addBody(bodyType, requestBody, input)
-          case BodyTypes.TEXT:
-            const text =
-              typeof requestBody !== "string"
-                ? JSON.stringify(requestBody)
-                : requestBody
-            // content type defaults to plaintext
-            input.body = text
-            break
-          default:
-          case BodyTypes.JSON:
-            try {
-              // confirm its json
-              const json = JSON.parse(requestBody)
-              if (
-                json &&
-                typeof json === "object" &&
-                Object.keys(json).length > 0
-              ) {
-                input.body = requestBody
-                input.headers["Content-Type"] = "application/json"
-              }
-            } catch (err) {
-              throw "Invalid JSON for request body"
-            }
-            break
-        }
       }
 
       this.startTimeMs = performance.now()

packages/server/yarn.lock

@@ -983,10 +983,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/auth@^1.0.18":
+"@budibase/auth@^1.0.19-alpha.1":
-  version "1.0.19"
+  version "1.0.22"
-  resolved "https://registry.yarnpkg.com/@budibase/auth/-/auth-1.0.19.tgz#b5a8ad51170443d2136d244f51cfe7dbcc0db116"
+  resolved "https://registry.yarnpkg.com/@budibase/auth/-/auth-1.0.22.tgz#a93ea2fea46e00138ad3fa129c9ea19b056654e2"
-  integrity sha512-6H1K80KX8RUseLXD307tKRc+b0B7/b2SZmAYYGq5qrUSdUotydZaZ90pt5pXVdE754duxyc8DlrwmRfri5xu+A==
+  integrity sha512-eHCNEzGl6HxYlMpfRTXBokq2ALTK5f+CDSgJmGaL/jfPc2NlzCI5NoigZUkSrdwDiYZnnWLfDDR4dArYyLlFuA==
   dependencies:
     "@techpass/passport-openidconnect" "^0.3.0"
     aws-sdk "^2.901.0"
@@ -1056,10 +1056,10 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/bbui@^1.0.19":
+"@budibase/bbui@^1.0.22":
-  version "1.0.19"
+  version "1.0.22"
-  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.19.tgz#d79c99e8c0adcf24d9b83f00a15eb262ad73a7e2"
+  resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.22.tgz#ac3bd3a8699bd0be84aac3c5dff9d093e5b08462"
-  integrity sha512-GhsyqkDjHMvU1MCr7oXKIZi6NOhmkunJ6eAoob8obCLDm+LXC/1Q8ymSuJicctQpDpraaFS7zqQ6vYY9v7kpiQ==
+  integrity sha512-8/5rXEOwkr0OcQD1fn5GpmI3d5dS1cIJBAODjTVtlZrTdacwlz5W2j3zIh+CBG0X7zhVxEze3zs2b1vDNTvK6A==
   dependencies:
     "@adobe/spectrum-css-workflow-icons" "^1.2.1"
     "@spectrum-css/actionbutton" "^1.0.1"
@@ -1106,14 +1106,14 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/client@^1.0.18":
+"@budibase/client@^1.0.19-alpha.1":
-  version "1.0.19"
+  version "1.0.22"
-  resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.19.tgz#50ba2ad91ac2fd57c51306b80fbab24a26ba1403"
+  resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.22.tgz#80d6c3fb2b57a050199dde4a4b3e82b221601c25"
-  integrity sha512-8vAsD7VkLfq9ZrD+QPXGUcj/2D3vGO++IPr0zIKGNVG5FlOLFceQ9b7itExSFWutyVAjK/e/yq56tugnf0S+Fg==
+  integrity sha512-Cpao7l2lIWyJZJs8+zq1wFnQGaWRTDiRG+HkkjvqQZDkZexlo89zWPkY56NBbMT1qAXd6K3zAdRNNKVCBCtOaA==
   dependencies:
-    "@budibase/bbui" "^1.0.19"
+    "@budibase/bbui" "^1.0.22"
     "@budibase/standard-components" "^0.9.139"
-    "@budibase/string-templates" "^1.0.19"
+    "@budibase/string-templates" "^1.0.22"
     regexparam "^1.3.0"
     shortid "^2.2.15"
     svelte-spa-router "^3.0.5"
@@ -1163,10 +1163,10 @@
     svelte-apexcharts "^1.0.2"
     svelte-flatpickr "^3.1.0"
 
-"@budibase/string-templates@^1.0.18", "@budibase/string-templates@^1.0.19":
+"@budibase/string-templates@^1.0.19-alpha.1", "@budibase/string-templates@^1.0.22":
-  version "1.0.19"
+  version "1.0.22"
-  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.19.tgz#4b476dfc5d317e56d84a24dffd34715cc74c7b37"
+  resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.22.tgz#b795c61e53d541c0aa346a90d04b50dcca6ae117"
-  integrity sha512-MmSHF2HK3JS3goyNr3mUQi3azt5vSWlmSGlYFyw473jplRVYkmI8wXrP8gVy9mNJ4vksn3bkgFPI8Hi9RoNSbA==
+  integrity sha512-1ZhxzL75kVhP44fJlCWwqmGIPjZol1eB/xi3O11xJPYQ7lfzeJcGUpksvlgbLgBlw+MKkgppK7gEoMP247E0Qw==
   dependencies:
     "@budibase/handlebars-helpers" "^0.11.7"
     dayjs "^1.10.4"
@@ -5829,6 +5829,15 @@ form-data@^3.0.0:
     combined-stream "^1.0.8"
     mime-types "^2.1.12"
 
+form-data@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452"
+  integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==
+  dependencies:
+    asynckit "^0.4.0"
+    combined-stream "^1.0.8"
+    mime-types "^2.1.12"
+
 formidable@^1.1.1, formidable@^1.2.0:
   version "1.2.6"
   resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.6.tgz#d2a51d60162bbc9b4a055d8457a7c75315d1a168"
@@ -13081,7 +13090,7 @@ xml2js@0.4.19:
     sax ">=0.6.0"
     xmlbuilder "~9.0.1"
 
-xml2js@^0.4.19, xml2js@^0.4.5:
+xml2js@^0.4.19, xml2js@^0.4.23, xml2js@^0.4.5:
   version "0.4.23"
   resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"
   integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==