From ac49d718a46ca7a73f74dca31efc3316c5a7183d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 13 Dec 2021 18:17:20 +0000 Subject: [PATCH 1/6] Stopping get requests from having bodies (Node fetch doesn't allow this) and allow text body type. --- .../rest/[query]/index.svelte | 5 ++-- packages/server/src/integrations/rest.ts | 29 ++++++++++++------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte index 838646d806..d174770602 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte @@ -48,7 +48,7 @@ let breakQs = {}, bindings = {} let url = "" - let saveId + let saveId, isGet let response, schema, enabledHeaders let datasourceType, integrationInfo, queryConfig, responseSuccess @@ -57,6 +57,7 @@ $: queryConfig = integrationInfo?.query $: url = buildUrl(url, breakQs) $: checkQueryName(url) + $: isGet = query?.queryVerb === "read" $: responseSuccess = response?.info?.code >= 200 && response?.info?.code <= 206 @@ -226,7 +227,7 @@ option.name} getOptionValue={option => option.value} diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index 10c2c1215b..582bbc5ad9 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -162,18 +162,25 @@ module RestModule { } } - let json - if (bodyType === BodyTypes.JSON && requestBody) { - try { - json = JSON.parse(requestBody) - } catch (err) { - throw "Invalid JSON for request body" - } - } - const input: any = { method, headers: this.headers } - if (json && typeof json === "object" && Object.keys(json).length > 0) { - input.body = JSON.stringify(json) + if (requestBody) { + switch (bodyType) { + case BodyTypes.TEXT: + const text = typeof requestBody !== "string" ? JSON.stringify(requestBody) : requestBody + input.body = text + break + default: case BodyTypes.JSON: + try { + // confirm its json + const json = JSON.parse(requestBody) + if (json && typeof json === "object" && Object.keys(json).length > 0) { + input.body = requestBody + } + } catch (err) { + throw "Invalid JSON for request body" + } + break + } } this.startTimeMs = performance.now() From 8d6dc14608c0d0d256c9988b4df8904e495a0fc7 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 13 Dec 2021 18:20:02 +0000 Subject: [PATCH 2/6] Linting and adding JSON header. --- .../src/api/controllers/query/import/index.ts | 2 +- .../controllers/query/import/sources/curl.ts | 2 +- .../src/integrations/base/IntegrationBase.ts | 8 ++--- packages/server/src/integrations/rest.ts | 30 +++++++++++++++---- 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/packages/server/src/api/controllers/query/import/index.ts b/packages/server/src/api/controllers/query/import/index.ts index c99d89c2de..933d6b101c 100644 --- a/packages/server/src/api/controllers/query/import/index.ts +++ b/packages/server/src/api/controllers/query/import/index.ts @@ -3,7 +3,7 @@ import { queryValidation } from "../validation" import { generateQueryID } from "../../../../db/utils" import { ImportInfo, ImportSource } from "./sources/base" import { OpenAPI2 } from "./sources/openapi2" -import { Query } from './../../../../definitions/common'; +import { Query } from "./../../../../definitions/common" import { Curl } from "./sources/curl" interface ImportResult { errorQueries: Query[] diff --git a/packages/server/src/api/controllers/query/import/sources/curl.ts b/packages/server/src/api/controllers/query/import/sources/curl.ts index ae580f65e5..61e1ae0215 100644 --- a/packages/server/src/api/controllers/query/import/sources/curl.ts +++ b/packages/server/src/api/controllers/query/import/sources/curl.ts @@ -29,7 +29,7 @@ const parseBody = (curl: any) => { } const parseCookie = (curl: any) => { - if (curl.cookies){ + if (curl.cookies) { return Object.entries(curl.cookies).reduce((acc, entry) => { const [key, value] = entry return acc + `${key}=${value}; ` diff --git a/packages/server/src/integrations/base/IntegrationBase.ts b/packages/server/src/integrations/base/IntegrationBase.ts index d87a98c73b..bfda4fbd4d 100644 --- a/packages/server/src/integrations/base/IntegrationBase.ts +++ b/packages/server/src/integrations/base/IntegrationBase.ts @@ -1,6 +1,6 @@ export interface IntegrationBase { - create?(query: any): Promise - read?(query: any): Promise - update?(query: any): Promise - delete?(query: any): Promise + create?(query: any): Promise + read?(query: any): Promise + update?(query: any): Promise + delete?(query: any): Promise } diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index 582bbc5ad9..bba3c87d68 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -115,7 +115,9 @@ module RestModule { data = await response.text() raw = data } - const size = formatBytes(response.headers.get("content-length") || Buffer.byteLength(raw, "utf8")) + const size = formatBytes( + response.headers.get("content-length") || Buffer.byteLength(raw, "utf8") + ) const time = `${Math.round(performance.now() - this.startTimeMs)}ms` headers = response.headers.raw() for (let [key, value] of Object.entries(headers)) { @@ -148,7 +150,15 @@ module RestModule { } async _req(query: RestQuery) { - const { path = "", queryString = "", headers = {}, method = "GET", disabledHeaders, bodyType, requestBody } = query + const { + path = "", + queryString = "", + headers = {}, + method = "GET", + disabledHeaders, + bodyType, + requestBody, + } = query this.headers = { ...this.config.defaultHeaders, ...headers, @@ -166,15 +176,25 @@ module RestModule { if (requestBody) { switch (bodyType) { case BodyTypes.TEXT: - const text = typeof requestBody !== "string" ? JSON.stringify(requestBody) : requestBody + const text = + typeof requestBody !== "string" + ? JSON.stringify(requestBody) + : requestBody + // content type defaults to plaintext input.body = text break - default: case BodyTypes.JSON: + default: + case BodyTypes.JSON: try { // confirm its json const json = JSON.parse(requestBody) - if (json && typeof json === "object" && Object.keys(json).length > 0) { + if ( + json && + typeof json === "object" && + Object.keys(json).length > 0 + ) { input.body = requestBody + input.headers["Content-Type"] = "application/json" } } catch (err) { throw "Invalid JSON for request body" From 187ca44884ebe1d6e2d33d7776f588d8491c7d69 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 14 Dec 2021 10:45:38 +0000 Subject: [PATCH 3/6] Fixing code mirror mode reactivity. --- .../builder/src/components/common/CodeMirrorEditor.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/common/CodeMirrorEditor.svelte b/packages/builder/src/components/common/CodeMirrorEditor.svelte index 3eaebde031..65dad9d015 100644 --- a/packages/builder/src/components/common/CodeMirrorEditor.svelte +++ b/packages/builder/src/components/common/CodeMirrorEditor.svelte @@ -17,6 +17,7 @@ }, Text: { name: "text/html", + json: false, }, } @@ -40,11 +41,12 @@ let editor // Keep editor up to date with value + $: editor?.setOption("mode", mode) $: editor?.setValue(value || "") // Creates an instance of a code mirror editor async function createEditor(mode, value) { - if (!CodeMirror || !textarea || editor) { + if (!CodeMirror || !textarea) { return } From 53dba1cccb07a3f9436ceaf741044d6a0db2640a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 14 Dec 2021 17:59:02 +0000 Subject: [PATCH 4/6] Removing outgoing webhook functionality from automations and adding XML response/sending, as well as form data/encoded data. --- .../components/common/CodeMirrorEditor.svelte | 4 +- .../src/components/integration/codemirror.js | 1 + .../builder/src/constants/backend/index.js | 2 + .../_components/RestBodyInput.svelte | 22 +++- packages/server/package.json | 2 + .../server/src/api/controllers/automation.js | 12 +- .../src/automations/steps/outgoingWebhook.js | 5 +- packages/server/src/automations/utils.js | 11 ++ packages/server/src/integrations/rest.ts | 105 ++++++++++++------ packages/server/yarn.lock | 47 ++++---- 10 files changed, 145 insertions(+), 66 deletions(-) diff --git a/packages/builder/src/components/common/CodeMirrorEditor.svelte b/packages/builder/src/components/common/CodeMirrorEditor.svelte index 65dad9d015..9d036e21ce 100644 --- a/packages/builder/src/components/common/CodeMirrorEditor.svelte +++ b/packages/builder/src/components/common/CodeMirrorEditor.svelte @@ -8,6 +8,9 @@ name: "javascript", json: true, }, + XML: { + name: "xml", + }, SQL: { name: "sql", }, @@ -17,7 +20,6 @@ }, Text: { name: "text/html", - json: false, }, } diff --git a/packages/builder/src/components/integration/codemirror.js b/packages/builder/src/components/integration/codemirror.js index c4afd8488c..78ff8e15c0 100644 --- a/packages/builder/src/components/integration/codemirror.js +++ b/packages/builder/src/components/integration/codemirror.js @@ -4,6 +4,7 @@ import "codemirror/lib/codemirror.css" // Modes import "codemirror/mode/javascript/javascript" import "codemirror/mode/sql/sql" +import "codemirror/mode/xml/xml" import "codemirror/mode/css/css" import "codemirror/mode/handlebars/handlebars" diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index d3c8d7d08d..1f49b0b26f 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -200,6 +200,7 @@ export const RawRestBodyTypes = { ENCODED: "encoded", JSON: "json", TEXT: "text", + XML: "xml", } export const RestBodyTypes = [ @@ -207,5 +208,6 @@ export const RestBodyTypes = [ { name: "form-data", value: "form" }, { name: "x-www-form-encoded", value: "encoded" }, { name: "raw (JSON)", value: "json" }, + { name: "raw (XML)", value: "xml" }, { name: "raw (Text)", value: "text" }, ] diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/RestBodyInput.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/RestBodyInput.svelte index a70b0f02f9..c387f0b492 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/RestBodyInput.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/RestBodyInput.svelte @@ -7,7 +7,11 @@ } from "components/common/CodeMirrorEditor.svelte" const objectTypes = [RawRestBodyTypes.FORM, RawRestBodyTypes.ENCODED] - const textTypes = [RawRestBodyTypes.JSON, RawRestBodyTypes.TEXT] + const textTypes = [ + RawRestBodyTypes.JSON, + RawRestBodyTypes.XML, + RawRestBodyTypes.TEXT, + ] export let query export let bodyType @@ -25,6 +29,18 @@ query.fields.requestBody = "" } } + + function editorMode(type) { + switch (type) { + case RawRestBodyTypes.JSON: + return EditorModes.JSON + case RawRestBodyTypes.XML: + return EditorModes.XML + default: + case RawRestBodyTypes.TEXT: + return EditorModes.Text + } + }
@@ -41,9 +57,7 @@ {:else if textTypes.includes(bodyType)} (query.fields.requestBody = e.detail)} diff --git a/packages/server/package.json b/packages/server/package.json index df9d23a6be..1f30a67695 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -90,6 +90,7 @@ "dotenv": "8.2.0", "download": "8.0.0", "fix-path": "3.0.0", + "form-data": "^4.0.0", "fs-extra": "8.1.0", "jimp": "0.16.1", "joi": "17.2.1", @@ -126,6 +127,7 @@ "validate.js": "0.13.1", "vm2": "^3.9.3", "worker-farm": "^1.7.0", + "xml2js": "^0.4.23", "yargs": "13.2.4", "zlib": "1.0.5" }, diff --git a/packages/server/src/api/controllers/automation.js b/packages/server/src/api/controllers/automation.js index 841473a4ff..05337579a0 100644 --- a/packages/server/src/api/controllers/automation.js +++ b/packages/server/src/api/controllers/automation.js @@ -5,11 +5,15 @@ const { getAutomationParams, generateAutomationID } = require("../../db/utils") const { checkForWebhooks, updateTestHistory, + removeDeprecated, } = require("../../automations/utils") const { deleteEntityMetadata } = require("../../utilities") const { MetadataTypes } = require("../../constants") const { setTestFlag, clearTestFlag } = require("../../utilities/redis") +const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS) +const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS) + /************************* * * * BUILDER FUNCTIONS * @@ -155,17 +159,17 @@ exports.destroy = async function (ctx) { } exports.getActionList = async function (ctx) { - ctx.body = actions.ACTION_DEFINITIONS + ctx.body = ACTION_DEFS } exports.getTriggerList = async function (ctx) { - ctx.body = triggers.TRIGGER_DEFINITIONS + ctx.body = TRIGGER_DEFS } module.exports.getDefinitionList = async function (ctx) { ctx.body = { - trigger: triggers.TRIGGER_DEFINITIONS, - action: actions.ACTION_DEFINITIONS, + trigger: TRIGGER_DEFS, + action: ACTION_DEFS, } } diff --git a/packages/server/src/automations/steps/outgoingWebhook.js b/packages/server/src/automations/steps/outgoingWebhook.js index f0637c3351..01d1e8d6be 100644 --- a/packages/server/src/automations/steps/outgoingWebhook.js +++ b/packages/server/src/automations/steps/outgoingWebhook.js @@ -13,12 +13,11 @@ const RequestType = { const BODY_REQUESTS = [RequestType.POST, RequestType.PUT, RequestType.PATCH] /** - * Note, there is some functionality in this that is not currently exposed as it - * is complex and maybe better to be opinionated here. - * GET/DELETE requests cannot handle body elements so they will not be sent if configured. + * NOTE: this functionality is deprecated - it no longer should be used. */ exports.definition = { + deprecated: true, name: "Outgoing webhook", tagline: "Send a {{inputs.requestMethod}} request", icon: "Send", diff --git a/packages/server/src/automations/utils.js b/packages/server/src/automations/utils.js index 6e58e9aae0..d9c96963ab 100644 --- a/packages/server/src/automations/utils.js +++ b/packages/server/src/automations/utils.js @@ -7,6 +7,7 @@ const newid = require("../db/newid") const { updateEntityMetadata } = require("../utilities") const { MetadataTypes } = require("../constants") const { getDeployedAppID } = require("@budibase/auth/db") +const { cloneDeep } = require("lodash/fp") const WH_STEP_ID = definitions.WEBHOOK.stepId const CRON_STEP_ID = definitions.CRON.stepId @@ -42,6 +43,16 @@ exports.updateTestHistory = async (appId, automation, history) => { ) } +exports.removeDeprecated = definitions => { + const base = cloneDeep(definitions) + for (let key of Object.keys(base)) { + if (base[key].deprecated) { + delete base[key] + } + } + return base +} + // end the repetition and the job itself exports.disableAllCrons = async appId => { const promises = [] diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index bba3c87d68..4fa43b9f84 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -10,6 +10,7 @@ import { IntegrationBase } from "./base/IntegrationBase" const BodyTypes = { NONE: "none", FORM_DATA: "form", + XML: "xml", ENCODED: "encoded", JSON: "json", TEXT: "text", @@ -42,6 +43,9 @@ module RestModule { const fetch = require("node-fetch") const { formatBytes } = require("../utilities") const { performance } = require("perf_hooks") + const FormData = require("form-data") + const { URLSearchParams } = require("url") + const xmlParser = require("xml2js").parseStringPromise const SCHEMA: Integration = { docs: "https://github.com/node-fetch/node-fetch", @@ -107,13 +111,26 @@ module RestModule { async parseResponse(response: any) { let data, raw, headers - const contentType = response.headers.get("content-type") - if (contentType && contentType.indexOf("application/json") !== -1) { - data = await response.json() - raw = JSON.stringify(data) - } else { - data = await response.text() - raw = data + const contentType = response.headers.get("content-type") || "" + try { + if (contentType.includes("application/json")) { + data = await response.json() + raw = JSON.stringify(data) + } else if (contentType.includes("text/xml") || contentType.includes("application/xml")) { + const rawXml = await response.text() + data = await xmlParser(rawXml, { explicitArray: false, trim: true, explicitRoot: false }) || {} + // there is only one structure, its an array, return the array so it appears as rows + const keys = Object.keys(data) + if (keys.length === 1 && Array.isArray(data[keys[0]])) { + data = data[keys[0]] + } + raw = rawXml + } else { + data = await response.text() + raw = data + } + } catch (err) { + throw "Failed to parse response body." } const size = formatBytes( response.headers.get("content-length") || Buffer.byteLength(raw, "utf8") @@ -149,6 +166,50 @@ module RestModule { return complete } + addBody(bodyType: string, body: string | any, input: any) { + let error, object, string + try { + string = typeof body !== "string" ? JSON.stringify(body) : body + object = typeof body === "object" ? body : JSON.parse(body) + } catch (err) { + error = err + } + switch (bodyType) { + case BodyTypes.TEXT: + // content type defaults to plaintext + input.body = string + break + case BodyTypes.ENCODED: + const params = new URLSearchParams() + for (let [key, value] of Object.entries(object)) { + params.append(key, value) + } + input.body = params + break + case BodyTypes.FORM_DATA: + const form = new FormData() + for (let [key, value] of Object.entries(object)) { + form.append(key, value) + } + input.body = form + break + case BodyTypes.XML: + input.body = string + input.headers["Content-Type"] = "text/xml" + break + default: + case BodyTypes.JSON: + // if JSON error, throw it + if (error) { + throw "Invalid JSON for request body" + } + input.body = object + input.headers["Content-Type"] = "application/json" + break + } + return input + } + async _req(query: RestQuery) { const { path = "", @@ -172,35 +233,9 @@ module RestModule { } } - const input: any = { method, headers: this.headers } + let input: any = { method, headers: this.headers } if (requestBody) { - switch (bodyType) { - case BodyTypes.TEXT: - const text = - typeof requestBody !== "string" - ? JSON.stringify(requestBody) - : requestBody - // content type defaults to plaintext - input.body = text - break - default: - case BodyTypes.JSON: - try { - // confirm its json - const json = JSON.parse(requestBody) - if ( - json && - typeof json === "object" && - Object.keys(json).length > 0 - ) { - input.body = requestBody - input.headers["Content-Type"] = "application/json" - } - } catch (err) { - throw "Invalid JSON for request body" - } - break - } + input = this.addBody(bodyType, requestBody, input) } this.startTimeMs = performance.now() diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index ff2f63d531..8e2c9db352 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -983,10 +983,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/auth@^1.0.18": - version "1.0.19" - resolved "https://registry.yarnpkg.com/@budibase/auth/-/auth-1.0.19.tgz#b5a8ad51170443d2136d244f51cfe7dbcc0db116" - integrity sha512-6H1K80KX8RUseLXD307tKRc+b0B7/b2SZmAYYGq5qrUSdUotydZaZ90pt5pXVdE754duxyc8DlrwmRfri5xu+A== +"@budibase/auth@^1.0.19-alpha.1": + version "1.0.22" + resolved "https://registry.yarnpkg.com/@budibase/auth/-/auth-1.0.22.tgz#a93ea2fea46e00138ad3fa129c9ea19b056654e2" + integrity sha512-eHCNEzGl6HxYlMpfRTXBokq2ALTK5f+CDSgJmGaL/jfPc2NlzCI5NoigZUkSrdwDiYZnnWLfDDR4dArYyLlFuA== dependencies: "@techpass/passport-openidconnect" "^0.3.0" aws-sdk "^2.901.0" @@ -1056,10 +1056,10 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/bbui@^1.0.19": - version "1.0.19" - resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.19.tgz#d79c99e8c0adcf24d9b83f00a15eb262ad73a7e2" - integrity sha512-GhsyqkDjHMvU1MCr7oXKIZi6NOhmkunJ6eAoob8obCLDm+LXC/1Q8ymSuJicctQpDpraaFS7zqQ6vYY9v7kpiQ== +"@budibase/bbui@^1.0.22": + version "1.0.22" + resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.22.tgz#ac3bd3a8699bd0be84aac3c5dff9d093e5b08462" + integrity sha512-8/5rXEOwkr0OcQD1fn5GpmI3d5dS1cIJBAODjTVtlZrTdacwlz5W2j3zIh+CBG0X7zhVxEze3zs2b1vDNTvK6A== dependencies: "@adobe/spectrum-css-workflow-icons" "^1.2.1" "@spectrum-css/actionbutton" "^1.0.1" @@ -1106,14 +1106,14 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/client@^1.0.18": - version "1.0.19" - resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.19.tgz#50ba2ad91ac2fd57c51306b80fbab24a26ba1403" - integrity sha512-8vAsD7VkLfq9ZrD+QPXGUcj/2D3vGO++IPr0zIKGNVG5FlOLFceQ9b7itExSFWutyVAjK/e/yq56tugnf0S+Fg== +"@budibase/client@^1.0.19-alpha.1": + version "1.0.22" + resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.22.tgz#80d6c3fb2b57a050199dde4a4b3e82b221601c25" + integrity sha512-Cpao7l2lIWyJZJs8+zq1wFnQGaWRTDiRG+HkkjvqQZDkZexlo89zWPkY56NBbMT1qAXd6K3zAdRNNKVCBCtOaA== dependencies: - "@budibase/bbui" "^1.0.19" + "@budibase/bbui" "^1.0.22" "@budibase/standard-components" "^0.9.139" - "@budibase/string-templates" "^1.0.19" + "@budibase/string-templates" "^1.0.22" regexparam "^1.3.0" shortid "^2.2.15" svelte-spa-router "^3.0.5" @@ -1163,10 +1163,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/string-templates@^1.0.18", "@budibase/string-templates@^1.0.19": - version "1.0.19" - resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.19.tgz#4b476dfc5d317e56d84a24dffd34715cc74c7b37" - integrity sha512-MmSHF2HK3JS3goyNr3mUQi3azt5vSWlmSGlYFyw473jplRVYkmI8wXrP8gVy9mNJ4vksn3bkgFPI8Hi9RoNSbA== +"@budibase/string-templates@^1.0.19-alpha.1", "@budibase/string-templates@^1.0.22": + version "1.0.22" + resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.22.tgz#b795c61e53d541c0aa346a90d04b50dcca6ae117" + integrity sha512-1ZhxzL75kVhP44fJlCWwqmGIPjZol1eB/xi3O11xJPYQ7lfzeJcGUpksvlgbLgBlw+MKkgppK7gEoMP247E0Qw== dependencies: "@budibase/handlebars-helpers" "^0.11.7" dayjs "^1.10.4" @@ -5829,6 +5829,15 @@ form-data@^3.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" +form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + formidable@^1.1.1, formidable@^1.2.0: version "1.2.6" resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.6.tgz#d2a51d60162bbc9b4a055d8457a7c75315d1a168" @@ -13081,7 +13090,7 @@ xml2js@0.4.19: sax ">=0.6.0" xmlbuilder "~9.0.1" -xml2js@^0.4.19, xml2js@^0.4.5: +xml2js@^0.4.19, xml2js@^0.4.23, xml2js@^0.4.5: version "0.4.23" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== From 06490c5fa622f69bf957d347d95a2ab27adb69a4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 15 Dec 2021 12:23:00 +0000 Subject: [PATCH 5/6] Fixing issue detected by test case. --- packages/server/src/integrations/rest.ts | 2 +- .../src/integrations/tests/rest.spec.js | 21 ++++++++----------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index f87e5b7416..5fae7b4e8f 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -214,7 +214,7 @@ module RestModule { if (error) { throw "Invalid JSON for request body" } - input.body = object + input.body = string input.headers["Content-Type"] = "application/json" break } diff --git a/packages/server/src/integrations/tests/rest.spec.js b/packages/server/src/integrations/tests/rest.spec.js index 6c1d989124..d2413b4af1 100644 --- a/packages/server/src/integrations/tests/rest.spec.js +++ b/packages/server/src/integrations/tests/rest.spec.js @@ -14,6 +14,11 @@ const fetch = require("node-fetch") const RestIntegration = require("../rest") const { AuthType } = require("../rest") +const HEADERS = { + "Accept": "application/json", + "Content-Type": "application/json" +} + class TestConfiguration { constructor(config = {}) { this.integration = new RestIntegration.integration(config) @@ -35,9 +40,7 @@ describe("REST Integration", () => { const query = { path: "api", queryString: "test=1", - headers: { - Accept: "application/json", - }, + headers: HEADERS, bodyType: "json", requestBody: JSON.stringify({ name: "test", @@ -47,9 +50,7 @@ describe("REST Integration", () => { expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?test=1`, { method: "POST", body: '{"name":"test"}', - headers: { - Accept: "application/json", - }, + headers: HEADERS, }) }) @@ -86,9 +87,7 @@ describe("REST Integration", () => { expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?test=1`, { method: "PUT", body: '{"name":"test"}', - headers: { - Accept: "application/json", - }, + headers: HEADERS, }) }) @@ -107,9 +106,7 @@ describe("REST Integration", () => { const response = await config.integration.delete(query) expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?test=1`, { method: "DELETE", - headers: { - Accept: "application/json", - }, + headers: HEADERS, body: '{"name":"test"}', }) }) From ef06566eb52f8b29f7a128bb53e9dcf0fe5e0390 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 15 Dec 2021 13:09:03 +0000 Subject: [PATCH 6/6] Adding unit tests for REST bodies and response parsing. --- packages/server/src/integrations/rest.ts | 12 ++- .../src/integrations/tests/rest.spec.js | 87 +++++++++++++++++++ 2 files changed, 97 insertions(+), 2 deletions(-) diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index 5fae7b4e8f..1817f780d3 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -48,7 +48,7 @@ module RestModule { const { performance } = require("perf_hooks") const FormData = require("form-data") const { URLSearchParams } = require("url") - const xmlParser = require("xml2js").parseStringPromise + const { parseStringPromise: xmlParser, Builder: XmlBuilder } = require("xml2js") const SCHEMA: Integration = { docs: "https://github.com/node-fetch/node-fetch", @@ -185,7 +185,12 @@ module RestModule { } catch (err) { error = err } + if (!input.headers) { + input.headers = {} + } switch (bodyType) { + case BodyTypes.NONE: + break case BodyTypes.TEXT: // content type defaults to plaintext input.body = string @@ -205,8 +210,11 @@ module RestModule { input.body = form break case BodyTypes.XML: + if (object != null) { + string = (new XmlBuilder()).buildObject(object) + } input.body = string - input.headers["Content-Type"] = "text/xml" + input.headers["Content-Type"] = "application/xml" break default: case BodyTypes.JSON: diff --git a/packages/server/src/integrations/tests/rest.spec.js b/packages/server/src/integrations/tests/rest.spec.js index d2413b4af1..6cfa670622 100644 --- a/packages/server/src/integrations/tests/rest.spec.js +++ b/packages/server/src/integrations/tests/rest.spec.js @@ -111,6 +111,93 @@ describe("REST Integration", () => { }) }) + describe("request body", () => { + const input = { a: 1, b: 2 } + + it("should allow no body", () => { + const output = config.integration.addBody("none", null, {}) + expect(output.body).toBeUndefined() + expect(Object.keys(output.headers).length).toEqual(0) + }) + + it("should allow text body", () => { + const output = config.integration.addBody("text", "hello world", {}) + expect(output.body).toEqual("hello world") + // gets added by fetch + expect(Object.keys(output.headers).length).toEqual(0) + }) + + it("should allow form data", () => { + const FormData = require("form-data") + const output = config.integration.addBody("form", input, {}) + expect(output.body instanceof FormData).toEqual(true) + expect(output.body._valueLength).toEqual(2) + // gets added by fetch + expect(Object.keys(output.headers).length).toEqual(0) + }) + + it("should allow encoded form data", () => { + const { URLSearchParams } = require("url") + const output = config.integration.addBody("encoded", input, {}) + expect(output.body instanceof URLSearchParams).toEqual(true) + expect(output.body.toString()).toEqual("a=1&b=2") + // gets added by fetch + expect(Object.keys(output.headers).length).toEqual(0) + }) + + it("should allow JSON", () => { + const output = config.integration.addBody("json", input, {}) + expect(output.body).toEqual(JSON.stringify(input)) + expect(output.headers["Content-Type"]).toEqual("application/json") + }) + + it("should allow XML", () => { + const output = config.integration.addBody("xml", input, {}) + expect(output.body.includes("1")).toEqual(true) + expect(output.body.includes("2")).toEqual(true) + expect(output.headers["Content-Type"]).toEqual("application/xml") + }) + }) + + describe("response", () => { + function buildInput(json, text, header) { + return { + status: 200, + json: json ? async () => json : undefined, + text: text ? async () => text : undefined, + headers: { get: key => key === "content-length" ? 100 : header, raw: () => ({ "content-type": header }) } + } + } + + it("should be able to parse JSON response", async () => { + const input = buildInput({a: 1}, null, "application/json") + const output = await config.integration.parseResponse(input) + expect(output.data).toEqual({a: 1}) + expect(output.info.code).toEqual(200) + expect(output.info.size).toEqual("100B") + expect(output.extra.raw).toEqual(JSON.stringify({a: 1})) + expect(output.extra.headers["content-type"]).toEqual("application/json") + }) + + it("should be able to parse text response", async () => { + const text = "hello world" + const input = buildInput(null, text, "text/plain") + const output = await config.integration.parseResponse(input) + expect(output.data).toEqual(text) + expect(output.extra.raw).toEqual(text) + expect(output.extra.headers["content-type"]).toEqual("text/plain") + }) + + it("should be able to parse XML response", async () => { + const text = "12" + const input = buildInput(null, text, "application/xml") + const output = await config.integration.parseResponse(input) + expect(output.data).toEqual({a: "1", b: "2"}) + expect(output.extra.raw).toEqual(text) + expect(output.extra.headers["content-type"]).toEqual("application/xml") + }) + }) + describe("authentication", () => { const basicAuth = { _id: "c59c14bd1898a43baa08da68959b24686",