diff --git a/.github/workflows/release-charts.yml b/.github/workflows/release-charts.yml index 933edd36b9..1ad5a54edd 100644 --- a/.github/workflows/release-charts.yml +++ b/.github/workflows/release-charts.yml @@ -12,6 +12,12 @@ jobs: with: fetch-depth: 0 + - uses: actions/setup-node@v1 + with: + node-version: 14.x + - run: yarn + - run: yarn bootstrap + - name: 'Get Previous tag' id: previoustag uses: "WyriHaximus/github-action-get-previous-tag@v1" diff --git a/charts/budibase/Chart.yaml b/charts/budibase/Chart.yaml index 602ffbf420..cec5af2914 100644 --- a/charts/budibase/Chart.yaml +++ b/charts/budibase/Chart.yaml @@ -11,8 +11,8 @@ sources: - https://github.com/Budibase/budibase - https://budibase.com type: application -version: 0.2.6 -appVersion: 1.0.10 +version: 1.0.0 +appVersion: 1.0.20 dependencies: - name: couchdb version: 3.3.4 diff --git a/lerna.json b/lerna.json index 28a2314346..09d7a43df9 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.19-alpha.2", + "version": "1.0.22", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/auth/package.json b/packages/auth/package.json index ea2479d159..74897f1b2b 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/auth", - "version": "1.0.19-alpha.2", + "version": "1.0.22", "description": "Authentication middlewares for budibase builder and apps", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 1034be22ef..d86371c554 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.19-alpha.2", + "version": "1.0.22", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/bbui/src/Form/Core/Search.svelte b/packages/bbui/src/Form/Core/Search.svelte index dd70f1eaca..e6384e242a 100644 --- a/packages/bbui/src/Form/Core/Search.svelte +++ b/packages/bbui/src/Form/Core/Search.svelte @@ -7,7 +7,7 @@ export let disabled = false export let id = null export let updateOnChange = true - + export let quiet = false const dispatch = createEventDispatcher() let focus = false @@ -41,6 +41,7 @@
diff --git a/packages/bbui/src/Form/Search.svelte b/packages/bbui/src/Form/Search.svelte index b5a6c17b4a..25dd98306b 100644 --- a/packages/bbui/src/Form/Search.svelte +++ b/packages/bbui/src/Form/Search.svelte @@ -9,6 +9,7 @@ export let placeholder = null export let disabled = false export let updateOnChange = true + export let quiet = false const dispatch = createEventDispatcher() const onChange = e => { @@ -23,6 +24,7 @@ {disabled} {value} {placeholder} + {quiet} on:change={onChange} on:click on:input diff --git a/packages/builder/cypress/integration/renameAnApplication.spec.js b/packages/builder/cypress/integration/renameAnApplication.spec.js index 95a152c017..a954faee95 100644 --- a/packages/builder/cypress/integration/renameAnApplication.spec.js +++ b/packages/builder/cypress/integration/renameAnApplication.spec.js @@ -10,7 +10,7 @@ it("should rename an unpublished application", () => { cy.get(".home-logo").click() renameApp(appRename) cy.searchForApplication(appRename) - cy.get(".appGrid").find(".wrapper").should("have.length", 1) + cy.get(".appTable").find(".title").should("have.length", 1) cy.deleteApp(appRename) }) @@ -29,7 +29,7 @@ xit("Should rename a published application", () => { cy.get(".home-logo").click() renameApp(appRename, true) cy.searchForApplication(appRename) - cy.get(".appGrid").find(".wrapper").should("have.length", 1) + cy.get(".appTable").find(".title").should("have.length", 1) }) it("Should try to rename an application to have no name", () => { @@ -38,7 +38,7 @@ it("Should try to rename an application to have no name", () => { // Close modal and confirm name has not been changed cy.get(".spectrum-Dialog-grid").contains("Cancel").click() cy.searchForApplication("Cypress Tests") - cy.get(".appGrid").find(".wrapper").should("have.length", 1) + cy.get(".appTable").find(".title").should("have.length", 1) }) xit("Should create two applications with the same name", () => { @@ -64,7 +64,7 @@ it("should validate application names", () => { cy.get(".home-logo").click() renameApp(numberName) cy.searchForApplication(numberName) - cy.get(".appGrid").find(".wrapper").should("have.length", 1) + cy.get(".appTable").find(".title").should("have.length", 1) renameApp(specialCharName) cy.get(".error").should("have.text", "App name must be letters, numbers and spaces only") }) @@ -74,14 +74,14 @@ it("should validate application names", () => { .its("body") .then(val => { if (val.length > 0) { - cy.get(".title > :nth-child(3) > .spectrum-Icon").click() + cy.get(".appTable > :nth-child(5) > :nth-child(2) > .spectrum-Icon").click() // Check for when an app is published if (published == true){ // Should not have Edit as option, will unpublish app cy.should("not.have.value", "Edit") cy.get(".spectrum-Menu").contains("Unpublish").click() cy.get(".spectrum-Dialog-grid").contains("Unpublish app").click() - cy.get(".title > :nth-child(3) > .spectrum-Icon").click() + cy.get(".appTable > :nth-child(5) > :nth-child(2) > .spectrum-Icon").click() } cy.contains("Edit").click() cy.get(".spectrum-Modal") diff --git a/packages/builder/cypress/support/commands.js b/packages/builder/cypress/support/commands.js index 0870ceac7a..e67057344a 100644 --- a/packages/builder/cypress/support/commands.js +++ b/packages/builder/cypress/support/commands.js @@ -50,7 +50,9 @@ Cypress.Commands.add("deleteApp", appName => { .its("body") .then(val => { if (val.length > 0) { - cy.get(".title > :nth-child(3) > .spectrum-Icon").click() + cy.get( + ".appTable > :nth-child(5) > :nth-child(2) > .spectrum-Icon" + ).click() cy.contains("Delete").click() cy.get(".spectrum-Modal").within(() => { cy.get("input").type(appName) diff --git a/packages/builder/package.json b/packages/builder/package.json index a4b06424d0..85e43edaaf 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.19-alpha.2", + "version": "1.0.22", "license": "GPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.19-alpha.2", - "@budibase/client": "^1.0.19-alpha.2", + "@budibase/bbui": "^1.0.22", + "@budibase/client": "^1.0.22", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^1.0.19-alpha.2", + "@budibase/string-templates": "^1.0.22", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/builder/src/components/common/CodeMirrorEditor.svelte b/packages/builder/src/components/common/CodeMirrorEditor.svelte index 3eaebde031..9d036e21ce 100644 --- a/packages/builder/src/components/common/CodeMirrorEditor.svelte +++ b/packages/builder/src/components/common/CodeMirrorEditor.svelte @@ -8,6 +8,9 @@ name: "javascript", json: true, }, + XML: { + name: "xml", + }, SQL: { name: "sql", }, @@ -40,11 +43,12 @@ let editor // Keep editor up to date with value + $: editor?.setOption("mode", mode) $: editor?.setValue(value || "") // Creates an instance of a code mirror editor async function createEditor(mode, value) { - if (!CodeMirror || !textarea || editor) { + if (!CodeMirror || !textarea) { return } diff --git a/packages/builder/src/components/integration/codemirror.js b/packages/builder/src/components/integration/codemirror.js index c4afd8488c..78ff8e15c0 100644 --- a/packages/builder/src/components/integration/codemirror.js +++ b/packages/builder/src/components/integration/codemirror.js @@ -4,6 +4,7 @@ import "codemirror/lib/codemirror.css" // Modes import "codemirror/mode/javascript/javascript" import "codemirror/mode/sql/sql" +import "codemirror/mode/xml/xml" import "codemirror/mode/css/css" import "codemirror/mode/handlebars/handlebars" diff --git a/packages/builder/src/components/start/AppRow.svelte b/packages/builder/src/components/start/AppRow.svelte index a80722733e..7466d3717d 100644 --- a/packages/builder/src/components/start/AppRow.svelte +++ b/packages/builder/src/components/start/AppRow.svelte @@ -1,5 +1,4 @@
-
-
editApp(app)}> - - {app.name} - +
+
+ +
+
editApp(app)}> + + {app.name} + +
@@ -62,6 +66,7 @@ disabled={app.lockedOther} on:click={() => editApp(app)} size="S" + quiet secondary>Open @@ -86,15 +91,11 @@ updateApp(app)} icon="Edit">Edit deleteApp(app)} icon="Delete">Delete {/if} + editIcon(app)} icon="Brush">Edit Icon
diff --git a/packages/builder/src/components/start/CreateAppModal.svelte b/packages/builder/src/components/start/CreateAppModal.svelte index 75df17e532..178588f608 100644 --- a/packages/builder/src/components/start/CreateAppModal.svelte +++ b/packages/builder/src/components/start/CreateAppModal.svelte @@ -1,13 +1,7 @@ - {#await templatesPromise} - - {:then templates} - {#if templates?.length > 0} - Select a template below, or start from scratch. - {:else} - Start your app from scratch below. - {/if} -
- {#each templates as template} -
onSelect(template)}> -
- -
- {template.name} -

{template?.category?.toUpperCase()}

-
- {/each} -
- {:catch err} -

{err}

- {/await}
onSelect(null)}>
diff --git a/packages/builder/src/stores/portal/apps.js b/packages/builder/src/stores/portal/apps.js index 48a917534f..de944c057d 100644 --- a/packages/builder/src/stores/portal/apps.js +++ b/packages/builder/src/stores/portal/apps.js @@ -65,16 +65,17 @@ export function createAppStore() { } } - async function update(appId, name) { - const response = await api.put(`/api/applications/${appId}`, { name }) + async function update(appId, value) { + console.log({ value }) + const response = await api.put(`/api/applications/${appId}`, { ...value }) if (response.status === 200) { store.update(state => { const updatedAppIndex = state.findIndex( app => app.instance._id === appId ) if (updatedAppIndex !== -1) { - const updatedApp = state[updatedAppIndex] - updatedApp.name = name + let updatedApp = state[updatedAppIndex] + updatedApp = { ...updatedApp, ...value } state.apps = state.splice(updatedAppIndex, 1, updatedApp) } return state diff --git a/packages/builder/src/stores/portal/index.js b/packages/builder/src/stores/portal/index.js index e2736cfa58..a5d33b3b15 100644 --- a/packages/builder/src/stores/portal/index.js +++ b/packages/builder/src/stores/portal/index.js @@ -5,3 +5,4 @@ export { apps } from "./apps" export { email } from "./email" export { auth } from "./auth" export { oidc } from "./oidc" +export { templates } from "./templates" diff --git a/packages/builder/src/stores/portal/templates.js b/packages/builder/src/stores/portal/templates.js new file mode 100644 index 0000000000..b82ecd70e2 --- /dev/null +++ b/packages/builder/src/stores/portal/templates.js @@ -0,0 +1,19 @@ +import { writable } from "svelte/store" +import api from "builderStore/api" + +export function templatesStore() { + const { subscribe, set } = writable([]) + + async function load() { + const response = await api.get("/api/templates?type=app") + const json = await response.json() + set(json) + } + + return { + subscribe, + load, + } +} + +export const templates = templatesStore() diff --git a/packages/cli/package.json b/packages/cli/package.json index 098b5054e3..de319745a1 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.19-alpha.2", + "version": "1.0.22", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 8b823850e7..2f40808ae0 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.19-alpha.2", + "version": "1.0.22", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.19-alpha.2", + "@budibase/bbui": "^1.0.22", "@budibase/standard-components": "^0.9.139", - "@budibase/string-templates": "^1.0.19-alpha.2", + "@budibase/string-templates": "^1.0.22", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/client/src/components/Router.svelte b/packages/client/src/components/Router.svelte index 95f52236a4..49cbc3f821 100644 --- a/packages/client/src/components/Router.svelte +++ b/packages/client/src/components/Router.svelte @@ -1,8 +1,9 @@ {#key config.id} diff --git a/packages/client/src/components/overlay/PeekScreenDisplay.svelte b/packages/client/src/components/overlay/PeekScreenDisplay.svelte index 51ff4412dc..7d3531d236 100644 --- a/packages/client/src/components/overlay/PeekScreenDisplay.svelte +++ b/packages/client/src/components/overlay/PeekScreenDisplay.svelte @@ -4,6 +4,7 @@ dataSourceStore, notificationStore, routeStore, + stateStore, } from "stores" import { Modal, ModalContent, ActionButton } from "@budibase/bbui" import { onDestroy } from "svelte" @@ -12,12 +13,13 @@ NOTIFICATION: "notification", CLOSE_SCREEN_MODAL: "close-screen-modal", INVALIDATE_DATASOURCE: "invalidate-datasource", + UPDATE_STATE: "update-state", } let iframe let listenersAttached = false - const invalidateDataSource = event => { + const proxyInvalidation = event => { const { dataSourceId } = event.detail dataSourceStore.actions.invalidateDataSource(dataSourceId) } @@ -27,14 +29,28 @@ notificationStore.actions.send(message, type, icon) } + const proxyStateUpdate = event => { + const { type, key, value, persist } = event.detail + if (type === "set") { + stateStore.actions.setValue(key, value, persist) + } else if (type === "delete") { + stateStore.actions.deleteValue(key) + } + } + function receiveMessage(message) { const handlers = { [MessageTypes.NOTIFICATION]: () => { proxyNotification(message.data) }, - [MessageTypes.CLOSE_SCREEN_MODAL]: peekStore.actions.hidePeek, + [MessageTypes.CLOSE_SCREEN_MODAL]: () => { + peekStore.actions.hidePeek() + }, [MessageTypes.INVALIDATE_DATASOURCE]: () => { - invalidateDataSource(message.data) + proxyInvalidation(message.data) + }, + [MessageTypes.UPDATE_STATE]: () => { + proxyStateUpdate(message.data) }, } diff --git a/packages/client/src/stores/dataSource.js b/packages/client/src/stores/dataSource.js index ee615b8bfb..efec755b99 100644 --- a/packages/client/src/stores/dataSource.js +++ b/packages/client/src/stores/dataSource.js @@ -1,6 +1,7 @@ import { writable, get } from "svelte/store" import { fetchTableDefinition } from "../api" import { FieldTypes } from "../constants" +import { routeStore } from "./routes" export const createDataSourceStore = () => { const store = writable([]) @@ -60,10 +61,12 @@ export const createDataSourceStore = () => { // Emit this as a window event, so parent screens which are iframing us in // can also invalidate the same datasource - window.parent.postMessage({ - type: "close-screen-modal", - detail: { dataSourceId }, - }) + if (get(routeStore).queryParams?.peek) { + window.parent.postMessage({ + type: "invalidate-datasource", + detail: { dataSourceId }, + }) + } let invalidations = [dataSourceId] diff --git a/packages/client/src/stores/peek.js b/packages/client/src/stores/peek.js index df41f5daff..83974b09f3 100644 --- a/packages/client/src/stores/peek.js +++ b/packages/client/src/stores/peek.js @@ -1,4 +1,5 @@ -import { writable } from "svelte/store" +import { writable, get } from "svelte/store" +import { stateStore } from "./state.js" const initialState = { showPeek: false, @@ -14,7 +15,10 @@ const createPeekStore = () => { let href = url let external = !url.startsWith("/") if (!external) { - href = `${window.location.href.split("#")[0]}#${url}?peek=true` + const state = get(stateStore) + const serialised = encodeURIComponent(btoa(JSON.stringify(state))) + const query = `peek=true&state=${serialised}` + href = `${window.location.href.split("#")[0]}#${url}?${query}` } store.set({ showPeek: true, diff --git a/packages/client/src/stores/state.js b/packages/client/src/stores/state.js index ce977c4333..0297b4c532 100644 --- a/packages/client/src/stores/state.js +++ b/packages/client/src/stores/state.js @@ -1,9 +1,9 @@ import { writable, get, derived } from "svelte/store" import { localStorageStore } from "builder/src/builderStore/store/localStorage" -import { appStore } from "./app" const createStateStore = () => { - const localStorageKey = `${get(appStore).appId}.state` + const appId = window["##BUDIBASE_APP_ID##"] || "app" + const localStorageKey = `${appId}.state` const persistentStore = localStorageStore(localStorageKey, {}) // Initialise the temp store to mirror the persistent store @@ -34,6 +34,9 @@ const createStateStore = () => { }) } + // Initialises the temporary state store with a certain value + const initialise = tempStore.set + // Derive the combination of both persisted and non persisted stores const store = derived( [tempStore, persistentStore], @@ -47,7 +50,7 @@ const createStateStore = () => { return { subscribe: store.subscribe, - actions: { setValue, deleteValue }, + actions: { setValue, deleteValue, initialise }, } } diff --git a/packages/client/src/utils/buttonActions.js b/packages/client/src/utils/buttonActions.js index 5ee571ae5d..9c6ae73c1f 100644 --- a/packages/client/src/utils/buttonActions.js +++ b/packages/client/src/utils/buttonActions.js @@ -116,6 +116,15 @@ const updateStateHandler = action => { } else if (type === "delete") { stateStore.actions.deleteValue(key) } + + // Emit this as an event so that parent windows which are iframing us in + // can also update their state + if (get(routeStore).queryParams?.peek) { + window.parent.postMessage({ + type: "update-state", + detail: { type, key, value, persist }, + }) + } } const handlerMap = { diff --git a/packages/server/package.json b/packages/server/package.json index af7c3029c1..97ab577241 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.19-alpha.2", + "version": "1.0.22", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -70,9 +70,9 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/auth": "^1.0.19-alpha.2", - "@budibase/client": "^1.0.19-alpha.2", - "@budibase/string-templates": "^1.0.19-alpha.2", + "@budibase/auth": "^1.0.22", + "@budibase/client": "^1.0.22", + "@budibase/string-templates": "^1.0.22", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", @@ -90,6 +90,7 @@ "dotenv": "8.2.0", "download": "8.0.0", "fix-path": "3.0.0", + "form-data": "^4.0.0", "fs-extra": "8.1.0", "jimp": "0.16.1", "joi": "17.2.1", @@ -126,6 +127,7 @@ "validate.js": "0.13.1", "vm2": "^3.9.3", "worker-farm": "^1.7.0", + "xml2js": "^0.4.23", "yargs": "13.2.4", "zlib": "1.0.5" }, diff --git a/packages/server/src/api/controllers/automation.js b/packages/server/src/api/controllers/automation.js index 841473a4ff..05337579a0 100644 --- a/packages/server/src/api/controllers/automation.js +++ b/packages/server/src/api/controllers/automation.js @@ -5,11 +5,15 @@ const { getAutomationParams, generateAutomationID } = require("../../db/utils") const { checkForWebhooks, updateTestHistory, + removeDeprecated, } = require("../../automations/utils") const { deleteEntityMetadata } = require("../../utilities") const { MetadataTypes } = require("../../constants") const { setTestFlag, clearTestFlag } = require("../../utilities/redis") +const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS) +const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS) + /************************* * * * BUILDER FUNCTIONS * @@ -155,17 +159,17 @@ exports.destroy = async function (ctx) { } exports.getActionList = async function (ctx) { - ctx.body = actions.ACTION_DEFINITIONS + ctx.body = ACTION_DEFS } exports.getTriggerList = async function (ctx) { - ctx.body = triggers.TRIGGER_DEFINITIONS + ctx.body = TRIGGER_DEFS } module.exports.getDefinitionList = async function (ctx) { ctx.body = { - trigger: triggers.TRIGGER_DEFINITIONS, - action: actions.ACTION_DEFINITIONS, + trigger: TRIGGER_DEFS, + action: ACTION_DEFS, } } diff --git a/packages/server/src/automations/steps/outgoingWebhook.js b/packages/server/src/automations/steps/outgoingWebhook.js index f0637c3351..01d1e8d6be 100644 --- a/packages/server/src/automations/steps/outgoingWebhook.js +++ b/packages/server/src/automations/steps/outgoingWebhook.js @@ -13,12 +13,11 @@ const RequestType = { const BODY_REQUESTS = [RequestType.POST, RequestType.PUT, RequestType.PATCH] /** - * Note, there is some functionality in this that is not currently exposed as it - * is complex and maybe better to be opinionated here. - * GET/DELETE requests cannot handle body elements so they will not be sent if configured. + * NOTE: this functionality is deprecated - it no longer should be used. */ exports.definition = { + deprecated: true, name: "Outgoing webhook", tagline: "Send a {{inputs.requestMethod}} request", icon: "Send", diff --git a/packages/server/src/automations/utils.js b/packages/server/src/automations/utils.js index 6e58e9aae0..d9c96963ab 100644 --- a/packages/server/src/automations/utils.js +++ b/packages/server/src/automations/utils.js @@ -7,6 +7,7 @@ const newid = require("../db/newid") const { updateEntityMetadata } = require("../utilities") const { MetadataTypes } = require("../constants") const { getDeployedAppID } = require("@budibase/auth/db") +const { cloneDeep } = require("lodash/fp") const WH_STEP_ID = definitions.WEBHOOK.stepId const CRON_STEP_ID = definitions.CRON.stepId @@ -42,6 +43,16 @@ exports.updateTestHistory = async (appId, automation, history) => { ) } +exports.removeDeprecated = definitions => { + const base = cloneDeep(definitions) + for (let key of Object.keys(base)) { + if (base[key].deprecated) { + delete base[key] + } + } + return base +} + // end the repetition and the job itself exports.disableAllCrons = async appId => { const promises = [] diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index 67668e7051..1817f780d3 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -6,13 +6,14 @@ import { RestQueryFields as RestQuery, AuthType, BasicAuthConfig, - BearerAuthConfig + BearerAuthConfig, } from "../definitions/datasource" import { IntegrationBase } from "./base/IntegrationBase" const BodyTypes = { NONE: "none", FORM_DATA: "form", + XML: "xml", ENCODED: "encoded", JSON: "json", TEXT: "text", @@ -45,6 +46,9 @@ module RestModule { const fetch = require("node-fetch") const { formatBytes } = require("../utilities") const { performance } = require("perf_hooks") + const FormData = require("form-data") + const { URLSearchParams } = require("url") + const { parseStringPromise: xmlParser, Builder: XmlBuilder } = require("xml2js") const SCHEMA: Integration = { docs: "https://github.com/node-fetch/node-fetch", @@ -110,15 +114,38 @@ module RestModule { async parseResponse(response: any) { let data, raw, headers - const contentType = response.headers.get("content-type") - if (contentType && contentType.indexOf("application/json") !== -1) { - data = await response.json() - raw = JSON.stringify(data) - } else { - data = await response.text() - raw = data + const contentType = response.headers.get("content-type") || "" + try { + if (contentType.includes("application/json")) { + data = await response.json() + raw = JSON.stringify(data) + } else if ( + contentType.includes("text/xml") || + contentType.includes("application/xml") + ) { + const rawXml = await response.text() + data = + (await xmlParser(rawXml, { + explicitArray: false, + trim: true, + explicitRoot: false, + })) || {} + // there is only one structure, its an array, return the array so it appears as rows + const keys = Object.keys(data) + if (keys.length === 1 && Array.isArray(data[keys[0]])) { + data = data[keys[0]] + } + raw = rawXml + } else { + data = await response.text() + raw = data + } + } catch (err) { + throw "Failed to parse response body." } - const size = formatBytes(response.headers.get("content-length") || Buffer.byteLength(raw, "utf8")) + const size = formatBytes( + response.headers.get("content-length") || Buffer.byteLength(raw, "utf8") + ) const time = `${Math.round(performance.now() - this.startTimeMs)}ms` headers = response.headers.raw() for (let [key, value] of Object.entries(headers)) { @@ -150,7 +177,59 @@ module RestModule { return complete } - getAuthHeaders(authConfigId: string): { [key: string]: any }{ + addBody(bodyType: string, body: string | any, input: any) { + let error, object, string + try { + string = typeof body !== "string" ? JSON.stringify(body) : body + object = typeof body === "object" ? body : JSON.parse(body) + } catch (err) { + error = err + } + if (!input.headers) { + input.headers = {} + } + switch (bodyType) { + case BodyTypes.NONE: + break + case BodyTypes.TEXT: + // content type defaults to plaintext + input.body = string + break + case BodyTypes.ENCODED: + const params = new URLSearchParams() + for (let [key, value] of Object.entries(object)) { + params.append(key, value) + } + input.body = params + break + case BodyTypes.FORM_DATA: + const form = new FormData() + for (let [key, value] of Object.entries(object)) { + form.append(key, value) + } + input.body = form + break + case BodyTypes.XML: + if (object != null) { + string = (new XmlBuilder()).buildObject(object) + } + input.body = string + input.headers["Content-Type"] = "application/xml" + break + default: + case BodyTypes.JSON: + // if JSON error, throw it + if (error) { + throw "Invalid JSON for request body" + } + input.body = string + input.headers["Content-Type"] = "application/json" + break + } + return input + } + + getAuthHeaders(authConfigId: string): { [key: string]: any } { let headers: any = {} if (this.config.authConfigs && authConfigId) { @@ -180,7 +259,16 @@ module RestModule { } async _req(query: RestQuery) { - const { path = "", queryString = "", headers = {}, method = "GET", disabledHeaders, bodyType, requestBody, authConfigId } = query + const { + path = "", + queryString = "", + headers = {}, + method = "GET", + disabledHeaders, + bodyType, + requestBody, + authConfigId, + } = query const authHeaders = this.getAuthHeaders(authConfigId) this.headers = { @@ -197,18 +285,9 @@ module RestModule { } } - let json - if (bodyType === BodyTypes.JSON && requestBody) { - try { - json = JSON.parse(requestBody) - } catch (err) { - throw "Invalid JSON for request body" - } - } - - const input: any = { method, headers: this.headers } - if (json && typeof json === "object" && Object.keys(json).length > 0) { - input.body = JSON.stringify(json) + let input: any = { method, headers: this.headers } + if (requestBody) { + input = this.addBody(bodyType, requestBody, input) } this.startTimeMs = performance.now() diff --git a/packages/server/src/integrations/tests/rest.spec.js b/packages/server/src/integrations/tests/rest.spec.js index 6c1d989124..6cfa670622 100644 --- a/packages/server/src/integrations/tests/rest.spec.js +++ b/packages/server/src/integrations/tests/rest.spec.js @@ -14,6 +14,11 @@ const fetch = require("node-fetch") const RestIntegration = require("../rest") const { AuthType } = require("../rest") +const HEADERS = { + "Accept": "application/json", + "Content-Type": "application/json" +} + class TestConfiguration { constructor(config = {}) { this.integration = new RestIntegration.integration(config) @@ -35,9 +40,7 @@ describe("REST Integration", () => { const query = { path: "api", queryString: "test=1", - headers: { - Accept: "application/json", - }, + headers: HEADERS, bodyType: "json", requestBody: JSON.stringify({ name: "test", @@ -47,9 +50,7 @@ describe("REST Integration", () => { expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?test=1`, { method: "POST", body: '{"name":"test"}', - headers: { - Accept: "application/json", - }, + headers: HEADERS, }) }) @@ -86,9 +87,7 @@ describe("REST Integration", () => { expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?test=1`, { method: "PUT", body: '{"name":"test"}', - headers: { - Accept: "application/json", - }, + headers: HEADERS, }) }) @@ -107,13 +106,98 @@ describe("REST Integration", () => { const response = await config.integration.delete(query) expect(fetch).toHaveBeenCalledWith(`${BASE_URL}/api?test=1`, { method: "DELETE", - headers: { - Accept: "application/json", - }, + headers: HEADERS, body: '{"name":"test"}', }) }) + describe("request body", () => { + const input = { a: 1, b: 2 } + + it("should allow no body", () => { + const output = config.integration.addBody("none", null, {}) + expect(output.body).toBeUndefined() + expect(Object.keys(output.headers).length).toEqual(0) + }) + + it("should allow text body", () => { + const output = config.integration.addBody("text", "hello world", {}) + expect(output.body).toEqual("hello world") + // gets added by fetch + expect(Object.keys(output.headers).length).toEqual(0) + }) + + it("should allow form data", () => { + const FormData = require("form-data") + const output = config.integration.addBody("form", input, {}) + expect(output.body instanceof FormData).toEqual(true) + expect(output.body._valueLength).toEqual(2) + // gets added by fetch + expect(Object.keys(output.headers).length).toEqual(0) + }) + + it("should allow encoded form data", () => { + const { URLSearchParams } = require("url") + const output = config.integration.addBody("encoded", input, {}) + expect(output.body instanceof URLSearchParams).toEqual(true) + expect(output.body.toString()).toEqual("a=1&b=2") + // gets added by fetch + expect(Object.keys(output.headers).length).toEqual(0) + }) + + it("should allow JSON", () => { + const output = config.integration.addBody("json", input, {}) + expect(output.body).toEqual(JSON.stringify(input)) + expect(output.headers["Content-Type"]).toEqual("application/json") + }) + + it("should allow XML", () => { + const output = config.integration.addBody("xml", input, {}) + expect(output.body.includes("1")).toEqual(true) + expect(output.body.includes("2")).toEqual(true) + expect(output.headers["Content-Type"]).toEqual("application/xml") + }) + }) + + describe("response", () => { + function buildInput(json, text, header) { + return { + status: 200, + json: json ? async () => json : undefined, + text: text ? async () => text : undefined, + headers: { get: key => key === "content-length" ? 100 : header, raw: () => ({ "content-type": header }) } + } + } + + it("should be able to parse JSON response", async () => { + const input = buildInput({a: 1}, null, "application/json") + const output = await config.integration.parseResponse(input) + expect(output.data).toEqual({a: 1}) + expect(output.info.code).toEqual(200) + expect(output.info.size).toEqual("100B") + expect(output.extra.raw).toEqual(JSON.stringify({a: 1})) + expect(output.extra.headers["content-type"]).toEqual("application/json") + }) + + it("should be able to parse text response", async () => { + const text = "hello world" + const input = buildInput(null, text, "text/plain") + const output = await config.integration.parseResponse(input) + expect(output.data).toEqual(text) + expect(output.extra.raw).toEqual(text) + expect(output.extra.headers["content-type"]).toEqual("text/plain") + }) + + it("should be able to parse XML response", async () => { + const text = "12" + const input = buildInput(null, text, "application/xml") + const output = await config.integration.parseResponse(input) + expect(output.data).toEqual({a: "1", b: "2"}) + expect(output.extra.raw).toEqual(text) + expect(output.extra.headers["content-type"]).toEqual("application/xml") + }) + }) + describe("authentication", () => { const basicAuth = { _id: "c59c14bd1898a43baa08da68959b24686", diff --git a/packages/server/src/threads/automation.js b/packages/server/src/threads/automation.js index 11ee28dbe8..fc9e2a21fc 100644 --- a/packages/server/src/threads/automation.js +++ b/packages/server/src/threads/automation.js @@ -1,6 +1,5 @@ -// when thread starts, make sure it is recorded +require("./utils").threadSetup() const env = require("../environment") -env.setInThread() const actions = require("../automations/actions") const automationUtils = require("../automations/automationUtils") const AutomationEmitter = require("../events/AutomationEmitter") diff --git a/packages/server/src/threads/query.js b/packages/server/src/threads/query.js index 9645bfcd47..f8d713e6a2 100644 --- a/packages/server/src/threads/query.js +++ b/packages/server/src/threads/query.js @@ -1,6 +1,4 @@ -// when thread starts, make sure it is recorded -const env = require("../environment") -env.setInThread() +require("./utils").threadSetup() const ScriptRunner = require("../utilities/scriptRunner") const { integrations } = require("../integrations") diff --git a/packages/server/src/threads/utils.js b/packages/server/src/threads/utils.js new file mode 100644 index 0000000000..48b40d7002 --- /dev/null +++ b/packages/server/src/threads/utils.js @@ -0,0 +1,13 @@ +const env = require("../environment") +const CouchDB = require("../db") +const { init } = require("@budibase/auth") + +exports.threadSetup = () => { + // don't run this if not threading + if (env.isTest() || env.DISABLE_THREADING) { + return + } + // when thread starts, make sure it is recorded + env.setInThread() + init(CouchDB) +} diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index ff2f63d531..8e2c9db352 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -983,10 +983,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/auth@^1.0.18": - version "1.0.19" - resolved "https://registry.yarnpkg.com/@budibase/auth/-/auth-1.0.19.tgz#b5a8ad51170443d2136d244f51cfe7dbcc0db116" - integrity sha512-6H1K80KX8RUseLXD307tKRc+b0B7/b2SZmAYYGq5qrUSdUotydZaZ90pt5pXVdE754duxyc8DlrwmRfri5xu+A== +"@budibase/auth@^1.0.19-alpha.1": + version "1.0.22" + resolved "https://registry.yarnpkg.com/@budibase/auth/-/auth-1.0.22.tgz#a93ea2fea46e00138ad3fa129c9ea19b056654e2" + integrity sha512-eHCNEzGl6HxYlMpfRTXBokq2ALTK5f+CDSgJmGaL/jfPc2NlzCI5NoigZUkSrdwDiYZnnWLfDDR4dArYyLlFuA== dependencies: "@techpass/passport-openidconnect" "^0.3.0" aws-sdk "^2.901.0" @@ -1056,10 +1056,10 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/bbui@^1.0.19": - version "1.0.19" - resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.19.tgz#d79c99e8c0adcf24d9b83f00a15eb262ad73a7e2" - integrity sha512-GhsyqkDjHMvU1MCr7oXKIZi6NOhmkunJ6eAoob8obCLDm+LXC/1Q8ymSuJicctQpDpraaFS7zqQ6vYY9v7kpiQ== +"@budibase/bbui@^1.0.22": + version "1.0.22" + resolved "https://registry.yarnpkg.com/@budibase/bbui/-/bbui-1.0.22.tgz#ac3bd3a8699bd0be84aac3c5dff9d093e5b08462" + integrity sha512-8/5rXEOwkr0OcQD1fn5GpmI3d5dS1cIJBAODjTVtlZrTdacwlz5W2j3zIh+CBG0X7zhVxEze3zs2b1vDNTvK6A== dependencies: "@adobe/spectrum-css-workflow-icons" "^1.2.1" "@spectrum-css/actionbutton" "^1.0.1" @@ -1106,14 +1106,14 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/client@^1.0.18": - version "1.0.19" - resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.19.tgz#50ba2ad91ac2fd57c51306b80fbab24a26ba1403" - integrity sha512-8vAsD7VkLfq9ZrD+QPXGUcj/2D3vGO++IPr0zIKGNVG5FlOLFceQ9b7itExSFWutyVAjK/e/yq56tugnf0S+Fg== +"@budibase/client@^1.0.19-alpha.1": + version "1.0.22" + resolved "https://registry.yarnpkg.com/@budibase/client/-/client-1.0.22.tgz#80d6c3fb2b57a050199dde4a4b3e82b221601c25" + integrity sha512-Cpao7l2lIWyJZJs8+zq1wFnQGaWRTDiRG+HkkjvqQZDkZexlo89zWPkY56NBbMT1qAXd6K3zAdRNNKVCBCtOaA== dependencies: - "@budibase/bbui" "^1.0.19" + "@budibase/bbui" "^1.0.22" "@budibase/standard-components" "^0.9.139" - "@budibase/string-templates" "^1.0.19" + "@budibase/string-templates" "^1.0.22" regexparam "^1.3.0" shortid "^2.2.15" svelte-spa-router "^3.0.5" @@ -1163,10 +1163,10 @@ svelte-apexcharts "^1.0.2" svelte-flatpickr "^3.1.0" -"@budibase/string-templates@^1.0.18", "@budibase/string-templates@^1.0.19": - version "1.0.19" - resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.19.tgz#4b476dfc5d317e56d84a24dffd34715cc74c7b37" - integrity sha512-MmSHF2HK3JS3goyNr3mUQi3azt5vSWlmSGlYFyw473jplRVYkmI8wXrP8gVy9mNJ4vksn3bkgFPI8Hi9RoNSbA== +"@budibase/string-templates@^1.0.19-alpha.1", "@budibase/string-templates@^1.0.22": + version "1.0.22" + resolved "https://registry.yarnpkg.com/@budibase/string-templates/-/string-templates-1.0.22.tgz#b795c61e53d541c0aa346a90d04b50dcca6ae117" + integrity sha512-1ZhxzL75kVhP44fJlCWwqmGIPjZol1eB/xi3O11xJPYQ7lfzeJcGUpksvlgbLgBlw+MKkgppK7gEoMP247E0Qw== dependencies: "@budibase/handlebars-helpers" "^0.11.7" dayjs "^1.10.4" @@ -5829,6 +5829,15 @@ form-data@^3.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" +form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + formidable@^1.1.1, formidable@^1.2.0: version "1.2.6" resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.2.6.tgz#d2a51d60162bbc9b4a055d8457a7c75315d1a168" @@ -13081,7 +13090,7 @@ xml2js@0.4.19: sax ">=0.6.0" xmlbuilder "~9.0.1" -xml2js@^0.4.19, xml2js@^0.4.5: +xml2js@^0.4.19, xml2js@^0.4.23, xml2js@^0.4.5: version "0.4.23" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 3f0b0e4f63..005d6304fe 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.19-alpha.2", + "version": "1.0.22", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index 83f5a647f2..f1e09ebf60 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.19-alpha.2", + "version": "1.0.22", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -29,8 +29,8 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/auth": "^1.0.19-alpha.2", - "@budibase/string-templates": "^1.0.19-alpha.2", + "@budibase/auth": "^1.0.22", + "@budibase/string-templates": "^1.0.22", "@koa/router": "^8.0.0", "@sentry/node": "^6.0.0", "@techpass/passport-openidconnect": "^0.3.0",