MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3785 from Budibase/feature/rest-bodies 

REST body types
This commit is contained in:
Michael Drury 2021-12-15 15:28:41 +00:00 committed by GitHub
parent 5da5acc27b ef06566eb5
commit 5f450c5bd4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 278 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
packages/builder/src/components/common/CodeMirrorEditor.svelte
View File

@ -8,6 +8,9 @@
name: "javascript",
json: true,
},
XML: {
name: "xml",
},
SQL: {
name: "sql",
},
@ -40,11 +43,12 @@
let editor
// Keep editor up to date with value
$: editor?.setOption("mode", mode)
$: editor?.setValue(value || "")
// Creates an instance of a code mirror editor
async function createEditor(mode, value) {
if (!CodeMirror || !textarea || editor) {
if (!CodeMirror || !textarea) {
return
}

1
packages/builder/src/components/integration/codemirror.js
View File

@ -4,6 +4,7 @@ import "codemirror/lib/codemirror.css"
// Modes
import "codemirror/mode/javascript/javascript"
import "codemirror/mode/sql/sql"
import "codemirror/mode/xml/xml"
import "codemirror/mode/css/css"
import "codemirror/mode/handlebars/handlebars"

2
packages/builder/src/constants/backend/index.js
View File

@ -200,6 +200,7 @@ export const RawRestBodyTypes = {
ENCODED: "encoded",
JSON: "json",
TEXT: "text",
XML: "xml",
}
export const RestBodyTypes = [
@ -207,5 +208,6 @@ export const RestBodyTypes = [
{ name: "form-data", value: "form" },
{ name: "x-www-form-encoded", value: "encoded" },
{ name: "raw (JSON)", value: "json" },
{ name: "raw (XML)", value: "xml" },
{ name: "raw (Text)", value: "text" },
]

22
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/_components/RestBodyInput.svelte
View File

@ -7,7 +7,11 @@
} from "components/common/CodeMirrorEditor.svelte"
const objectTypes = [RawRestBodyTypes.FORM, RawRestBodyTypes.ENCODED]
const textTypes = [RawRestBodyTypes.JSON, RawRestBodyTypes.TEXT]
const textTypes = [
RawRestBodyTypes.JSON,
RawRestBodyTypes.XML,
RawRestBodyTypes.TEXT,
]
export let query
export let bodyType
@ -25,6 +29,18 @@
query.fields.requestBody = ""
}
}
function editorMode(type) {
switch (type) {
case RawRestBodyTypes.JSON:
return EditorModes.JSON
case RawRestBodyTypes.XML:
return EditorModes.XML
default:
case RawRestBodyTypes.TEXT:
return EditorModes.Text
}
}
</script>
<div class="margin">
@ -41,9 +57,7 @@
{:else if textTypes.includes(bodyType)}
<CodeMirrorEditor
height={200}
mode={bodyType === RawRestBodyTypes.JSON
? EditorModes.JSON
: EditorModes.Text}
mode={editorMode(bodyType)}
value={query.fields.requestBody}
resize="vertical"
on:change={e => (query.fields.requestBody = e.detail)}

5
packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte
View File

@ -48,7 +48,7 @@
let breakQs = {},
bindings = {}
let url = ""
let saveId
let saveId, isGet
let response, schema, enabledHeaders
let datasourceType, integrationInfo, queryConfig, responseSuccess
let authConfigId
@ -58,6 +58,7 @@
$: queryConfig = integrationInfo?.query
$: url = buildUrl(url, breakQs)
$: checkQueryName(url)
$: isGet = query?.queryVerb === "read"
$: responseSuccess =
response?.info?.code >= 200 && response?.info?.code <= 206
$: authConfigs = buildAuthConfigs(datasource)
@ -262,7 +263,7 @@
<Tab title="Body">
<RadioGroup
bind:value={query.fields.bodyType}
options={bodyTypes}
options={isGet ? [bodyTypes[0]] : bodyTypes}
direction="horizontal"
getOptionLabel={option => option.name}
getOptionValue={option => option.value}

2
packages/server/package.json
View File

@ -90,6 +90,7 @@
"dotenv": "8.2.0",
"download": "8.0.0",
"fix-path": "3.0.0",
"form-data": "^4.0.0",
"fs-extra": "8.1.0",
"jimp": "0.16.1",
"joi": "17.2.1",
@ -126,6 +127,7 @@
"validate.js": "0.13.1",
"vm2": "^3.9.3",
"worker-farm": "^1.7.0",
"xml2js": "^0.4.23",
"yargs": "13.2.4",
"zlib": "1.0.5"
},

12
packages/server/src/api/controllers/automation.js
View File

@ -5,11 +5,15 @@ const { getAutomationParams, generateAutomationID } = require("../../db/utils")
const {
checkForWebhooks,
updateTestHistory,
removeDeprecated,
} = require("../../automations/utils")
const { deleteEntityMetadata } = require("../../utilities")
const { MetadataTypes } = require("../../constants")
const { setTestFlag, clearTestFlag } = require("../../utilities/redis")
const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS)
const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
/*************************
* *
* BUILDER FUNCTIONS *
@ -155,17 +159,17 @@ exports.destroy = async function (ctx) {
}
exports.getActionList = async function (ctx) {
ctx.body = actions.ACTION_DEFINITIONS
ctx.body = ACTION_DEFS
}
exports.getTriggerList = async function (ctx) {
ctx.body = triggers.TRIGGER_DEFINITIONS
ctx.body = TRIGGER_DEFS
}
module.exports.getDefinitionList = async function (ctx) {
ctx.body = {
trigger: triggers.TRIGGER_DEFINITIONS,
action: actions.ACTION_DEFINITIONS,
trigger: TRIGGER_DEFS,
action: ACTION_DEFS,
}
}

5
packages/server/src/automations/steps/outgoingWebhook.js
View File

@ -13,12 +13,11 @@ const RequestType = {
const BODY_REQUESTS = [RequestType.POST, RequestType.PUT, RequestType.PATCH]
/**
* Note, there is some functionality in this that is not currently exposed as it
* is complex and maybe better to be opinionated here.
* GET/DELETE requests cannot handle body elements so they will not be sent if configured.
* NOTE: this functionality is deprecated - it no longer should be used.
*/
exports.definition = {
deprecated: true,
name: "Outgoing webhook",
tagline: "Send a {{inputs.requestMethod}} request",
icon: "Send",

11
packages/server/src/automations/utils.js
View File

@ -7,6 +7,7 @@ const newid = require("../db/newid")
const { updateEntityMetadata } = require("../utilities")
const { MetadataTypes } = require("../constants")
const { getDeployedAppID } = require("@budibase/auth/db")
const { cloneDeep } = require("lodash/fp")
const WH_STEP_ID = definitions.WEBHOOK.stepId
const CRON_STEP_ID = definitions.CRON.stepId
@ -42,6 +43,16 @@ exports.updateTestHistory = async (appId, automation, history) => {
)
}
exports.removeDeprecated = definitions => {
const base = cloneDeep(definitions)
for (let key of Object.keys(base)) {
if (base[key].deprecated) {
delete base[key]
}
}
return base
}
// end the repetition and the job itself
exports.disableAllCrons = async appId => {
const promises = []

113
packages/server/src/integrations/rest.ts
View File

@ -6,13 +6,14 @@ import {
RestQueryFields as RestQuery,
AuthType,
BasicAuthConfig,
BearerAuthConfig
BearerAuthConfig,
} from "../definitions/datasource"
import { IntegrationBase } from "./base/IntegrationBase"
const BodyTypes = {
NONE: "none",
FORM_DATA: "form",
XML: "xml",
ENCODED: "encoded",
JSON: "json",
TEXT: "text",
@ -45,6 +46,9 @@ module RestModule {
const fetch = require("node-fetch")
const { formatBytes } = require("../utilities")
const { performance } = require("perf_hooks")
const FormData = require("form-data")
const { URLSearchParams } = require("url")
const { parseStringPromise: xmlParser, Builder: XmlBuilder } = require("xml2js")
const SCHEMA: Integration = {
docs: "https://github.com/node-fetch/node-fetch",
@ -110,15 +114,38 @@ module RestModule {
async parseResponse(response: any) {
let data, raw, headers
const contentType = response.headers.get("content-type")
if (contentType && contentType.indexOf("application/json") !== -1) {
const contentType = response.headers.get("content-type") || ""
try {
if (contentType.includes("application/json")) {
data = await response.json()
raw = JSON.stringify(data)
} else if (
contentType.includes("text/xml") ||
contentType.includes("application/xml")
) {
const rawXml = await response.text()
data =
(await xmlParser(rawXml, {
explicitArray: false,
trim: true,
explicitRoot: false,
})) || {}
// there is only one structure, its an array, return the array so it appears as rows
const keys = Object.keys(data)
if (keys.length === 1 && Array.isArray(data[keys[0]])) {
data = data[keys[0]]
}
raw = rawXml
} else {
data = await response.text()
raw = data
}
const size = formatBytes(response.headers.get("content-length") || Buffer.byteLength(raw, "utf8"))
} catch (err) {
throw "Failed to parse response body."
}
const size = formatBytes(
response.headers.get("content-length") || Buffer.byteLength(raw, "utf8")
)
const time = `${Math.round(performance.now() - this.startTimeMs)}ms`
headers = response.headers.raw()
for (let [key, value] of Object.entries(headers)) {
@ -150,6 +177,58 @@ module RestModule {
return complete
}
addBody(bodyType: string, body: string | any, input: any) {
let error, object, string
try {
string = typeof body !== "string" ? JSON.stringify(body) : body
object = typeof body === "object" ? body : JSON.parse(body)
} catch (err) {
error = err
}
if (!input.headers) {
input.headers = {}
}
switch (bodyType) {
case BodyTypes.NONE:
break
case BodyTypes.TEXT:
// content type defaults to plaintext
input.body = string
break
case BodyTypes.ENCODED:
const params = new URLSearchParams()
for (let [key, value] of Object.entries(object)) {
params.append(key, value)
}
input.body = params
break
case BodyTypes.FORM_DATA:
const form = new FormData()
for (let [key, value] of Object.entries(object)) {
form.append(key, value)
}
input.body = form
break
case BodyTypes.XML:
if (object != null) {
string = (new XmlBuilder()).buildObject(object)
}
input.body = string
input.headers["Content-Type"] = "application/xml"
break
default:
case BodyTypes.JSON:
// if JSON error, throw it
if (error) {
throw "Invalid JSON for request body"
}
input.body = string
input.headers["Content-Type"] = "application/json"
break
}
return input
}
getAuthHeaders(authConfigId: string): { [key: string]: any } {
let headers: any = {}
@ -180,7 +259,16 @@ module RestModule {
}
async _req(query: RestQuery) {
const { path = "", queryString = "", headers = {}, method = "GET", disabledHeaders, bodyType, requestBody, authConfigId } = query
const {
path = "",
queryString = "",
headers = {},
method = "GET",
disabledHeaders,
bodyType,
requestBody,
authConfigId,
} = query
const authHeaders = this.getAuthHeaders(authConfigId)
this.headers = {
@ -197,18 +285,9 @@ module RestModule {
}
}
let json
if (bodyType === BodyTypes.JSON && requestBody) {
try {
json = JSON.parse(requestBody)
} catch (err) {
throw "Invalid JSON for request body"
}
}
const input: any = { method, headers: this.headers }
if (json && typeof json === "object" && Object.keys(json).length > 0) {
input.body = JSON.stringify(json)
let input: any = { method, headers: this.headers }
if (requestBody) {
input = this.addBody(bodyType, requestBody, input)
}
this.startTimeMs = performance.now()

108
packages/server/src/integrations/tests/rest.spec.js
View File

@ -14,6 +14,11 @@ const fetch = require("node-fetch")
const RestIntegration = require("../rest")
const { AuthType } = require("../rest")
const HEADERS = {
"Accept": "application/json",
"Content-Type": "application/json"
}
class TestConfiguration {
constructor(config = {}) {
this.integration = new RestIntegration.integration(config)
@ -35,9 +40,7 @@ describe("REST Integration", () => {
const query = {
path: "api",
queryString: "test=1",
headers: {
Accept: "application/json",
},
headers: HEADERS,
bodyType: "json",
requestBody: JSON.stringify({
name: "test",
@ -47,9 +50,7 @@ describe("REST Integration", () => {
expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?test=1`, {
method: "POST",
body: '{"name":"test"}',
headers: {
Accept: "application/json",
},
headers: HEADERS,
})
})
@ -86,9 +87,7 @@ describe("REST Integration", () => {
expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?test=1`, {
method: "PUT",
body: '{"name":"test"}',
headers: {
Accept: "application/json",
},
headers: HEADERS,
})
})
@ -107,13 +106,98 @@ describe("REST Integration", () => {
const response = await config.integration.delete(query)
expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?test=1`, {
method: "DELETE",
headers: {
Accept: "application/json",
},
headers: HEADERS,
body: '{"name":"test"}',
})
})
describe("request body", () => {
const input = { a: 1, b: 2 }
it("should allow no body", () => {
const output = config.integration.addBody("none", null, {})
expect(output.body).toBeUndefined()
expect(Object.keys(output.headers).length).toEqual(0)
})
it("should allow text body", () => {
const output = config.integration.addBody("text", "hello world", {})
expect(output.body).toEqual("hello world")
// gets added by fetch
expect(Object.keys(output.headers).length).toEqual(0)
})
it("should allow form data", () => {
const FormData = require("form-data")
const output = config.integration.addBody("form", input, {})
expect(output.body instanceof FormData).toEqual(true)
expect(output.body._valueLength).toEqual(2)
// gets added by fetch
expect(Object.keys(output.headers).length).toEqual(0)
})
it("should allow encoded form data", () => {
const { URLSearchParams } = require("url")
const output = config.integration.addBody("encoded", input, {})
expect(output.body instanceof URLSearchParams).toEqual(true)
expect(output.body.toString()).toEqual("a=1&b=2")
// gets added by fetch
expect(Object.keys(output.headers).length).toEqual(0)
})
it("should allow JSON", () => {
const output = config.integration.addBody("json", input, {})
expect(output.body).toEqual(JSON.stringify(input))
expect(output.headers["Content-Type"]).toEqual("application/json")
})
it("should allow XML", () => {
const output = config.integration.addBody("xml", input, {})
expect(output.body.includes("<a>1</a>")).toEqual(true)
expect(output.body.includes("<b>2</b>")).toEqual(true)
expect(output.headers["Content-Type"]).toEqual("application/xml")
})
})
describe("response", () => {
function buildInput(json, text, header) {
return {
status: 200,
json: json ? async () => json : undefined,
text: text ? async () => text : undefined,
headers: { get: key => key === "content-length" ? 100 : header, raw: () => ({ "content-type": header }) }
}
}
it("should be able to parse JSON response", async () => {
const input = buildInput({a: 1}, null, "application/json")
const output = await config.integration.parseResponse(input)
expect(output.data).toEqual({a: 1})
expect(output.info.code).toEqual(200)
expect(output.info.size).toEqual("100B")
expect(output.extra.raw).toEqual(JSON.stringify({a: 1}))
expect(output.extra.headers["content-type"]).toEqual("application/json")
})
it("should be able to parse text response", async () => {
const text = "hello world"
const input = buildInput(null, text, "text/plain")
const output = await config.integration.parseResponse(input)
expect(output.data).toEqual(text)
expect(output.extra.raw).toEqual(text)
expect(output.extra.headers["content-type"]).toEqual("text/plain")
})
it("should be able to parse XML response", async () => {
const text = "<root><a>1</a><b>2</b></root>"
const input = buildInput(null, text, "application/xml")
const output = await config.integration.parseResponse(input)
expect(output.data).toEqual({a: "1", b: "2"})
expect(output.extra.raw).toEqual(text)
expect(output.extra.headers["content-type"]).toEqual("application/xml")
})
})
describe("authentication", () => {
const basicAuth = {
_id: "c59c14bd1898a43baa08da68959b24686",

47
packages/server/yarn.lock
View File

@ -983,10 +983,10 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/auth@^1.0.18":
version "1.0.19"
resolved "https://registry.yarnpkg.com/@budibase/auth/-/auth-1.0.19.tgz#b5a8ad51170443d2136d244f51cfe7dbcc0db116"
integrity sha512-6H1K80KX8RUseLXD307tKRc+b0B7/b2SZmAYYGq5qrUSdUotydZaZ90pt5pXVdE754duxyc8DlrwmRfri5xu+A==
"@budibase/auth@^1.0.19-alpha.1":
version "1.0.22"
resolved "https://registry.yarnpkg.com/@budibase/auth/-/auth-1.0.22.tgz#a93ea2fea46e00138ad3fa129c9ea19b056654e2"
integrity sha512-eHCNEzGl6HxYlMpfRTXBokq2ALTK5f+CDSgJmGaL/jfPc2NlzCI5NoigZUkSrdwDiYZnnWLfDDR4dArYyLlFuA==
dependencies:
"@techpass/passport-openidconnect" "^0.3.0"
aws-sdk "^2.901.0"
@ -1056,10 +1056,10 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/bbui@^1.0.19":
version "1.0.19"
resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.19.tgz#d79c99e8c0adcf24d9b83f00a15eb262ad73a7e2"
integrity sha512-GhsyqkDjHMvU1MCr7oXKIZi6NOhmkunJ6eAoob8obCLDm+LXC/1Q8ymSuJicctQpDpraaFS7zqQ6vYY9v7kpiQ==
"@budibase/bbui@^1.0.22":
version "1.0.22"
resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.22.tgz#ac3bd3a8699bd0be84aac3c5dff9d093e5b08462"
integrity sha512-8/5rXEOwkr0OcQD1fn5GpmI3d5dS1cIJBAODjTVtlZrTdacwlz5W2j3zIh+CBG0X7zhVxEze3zs2b1vDNTvK6A==
dependencies:
"@adobe/spectrum-css-workflow-icons" "^1.2.1"
"@spectrum-css/actionbutton" "^1.0.1"
@ -1106,14 +1106,14 @@
svelte-flatpickr "^3.2.3"
svelte-portal "^1.0.0"
"@budibase/client@^1.0.18":
version "1.0.19"
resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.19.tgz#50ba2ad91ac2fd57c51306b80fbab24a26ba1403"
integrity sha512-8vAsD7VkLfq9ZrD+QPXGUcj/2D3vGO++IPr0zIKGNVG5FlOLFceQ9b7itExSFWutyVAjK/e/yq56tugnf0S+Fg==
"@budibase/client@^1.0.19-alpha.1":
version "1.0.22"
resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.22.tgz#80d6c3fb2b57a050199dde4a4b3e82b221601c25"
integrity sha512-Cpao7l2lIWyJZJs8+zq1wFnQGaWRTDiRG+HkkjvqQZDkZexlo89zWPkY56NBbMT1qAXd6K3zAdRNNKVCBCtOaA==
dependencies:
"@budibase/bbui" "^1.0.19"
"@budibase/bbui" "^1.0.22"
"@budibase/standard-components" "^0.9.139"
"@budibase/string-templates" "^1.0.19"
"@budibase/string-templates" "^1.0.22"
regexparam "^1.3.0"
shortid "^2.2.15"
svelte-spa-router "^3.0.5"
@ -1163,10 +1163,10 @@
svelte-apexcharts "^1.0.2"
svelte-flatpickr "^3.1.0"
"@budibase/string-templates@^1.0.18", "@budibase/string-templates@^1.0.19":
version "1.0.19"
resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.19.tgz#4b476dfc5d317e56d84a24dffd34715cc74c7b37"
integrity sha512-MmSHF2HK3JS3goyNr3mUQi3azt5vSWlmSGlYFyw473jplRVYkmI8wXrP8gVy9mNJ4vksn3bkgFPI8Hi9RoNSbA==
"@budibase/string-templates@^1.0.19-alpha.1", "@budibase/string-templates@^1.0.22":
version "1.0.22"
resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.22.tgz#b795c61e53d541c0aa346a90d04b50dcca6ae117"
integrity sha512-1ZhxzL75kVhP44fJlCWwqmGIPjZol1eB/xi3O11xJPYQ7lfzeJcGUpksvlgbLgBlw+MKkgppK7gEoMP247E0Qw==
dependencies:
"@budibase/handlebars-helpers" "^0.11.7"
dayjs "^1.10.4"
@ -5829,6 +5829,15 @@ form-data@^3.0.0:
combined-stream "^1.0.8"
mime-types "^2.1.12"
form-data@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452"
integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==
dependencies:
asynckit "^0.4.0"
combined-stream "^1.0.8"
mime-types "^2.1.12"
formidable@^1.1.1, formidable@^1.2.0:
version "1.2.6"
resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.6.tgz#d2a51d60162bbc9b4a055d8457a7c75315d1a168"
@ -13081,7 +13090,7 @@ xml2js@0.4.19:
sax ">=0.6.0"
xmlbuilder "~9.0.1"
xml2js@^0.4.19, xml2js@^0.4.5:
xml2js@^0.4.19, xml2js@^0.4.23, xml2js@^0.4.5:
version "0.4.23"
resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"
integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==