MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2992 from Budibase/feature/query-transformers 

Query transformers
This commit is contained in:
Michael Drury 2021-10-14 11:23:28 +01:00 committed by GitHub
parent 29659361fe b6939397d1
commit f1e5483e89
9 changed files with 929 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
packages/builder/src/components/common/CodeMirrorEditor.svelte
View File

@ -1,4 +1,6 @@
<script context="module"> <script context="module">
import { Label } from "@budibase/bbui"
export const EditorModes = { export const EditorModes = {
JS: { JS: {
name: "javascript", name: "javascript",
@ -26,8 +28,10 @@
export let mode = EditorModes.JS export let mode = EditorModes.JS
export let value = "" export let value = ""
export let height = 300 export let height = 300
export let resize = "none"
export let readonly = false export let readonly = false
export let hints = [] export let hints = []
export let label
const dispatch = createEventDispatcher() const dispatch = createEventDispatcher()
let textarea let textarea
@ -110,19 +114,28 @@
}) })
</script> </script>
<div style={`--code-mirror-height: ${height}px`}> {#if label}
<div style="margin-bottom: var(--spacing-s)">
<Label small>{label}</Label>
</div>
{/if}
<div
style={`--code-mirror-height: ${height}px; --code-mirror-resize: ${resize}`}
>
<textarea tabindex="0" bind:this={textarea} readonly {value} /> <textarea tabindex="0" bind:this={textarea} readonly {value} />
</div> </div>
<style> <style>
div :global(.CodeMirror) { div :global(.CodeMirror) {
height: var(--code-mirror-height); height: var(--code-mirror-height);
min-height: var(--code-mirror-height);
font-family: monospace; font-family: monospace;
line-height: 1.3; line-height: 1.3;
border: var(--spectrum-alias-border-size-thin) solid; border: var(--spectrum-alias-border-size-thin) solid;
border-color: var(--spectrum-alias-border-color); border-color: var(--spectrum-alias-border-color);
overflow: hidden;
border-radius: var(--border-radius-s); border-radius: var(--border-radius-s);
resize: var(--code-mirror-resize);
overflow: hidden;
} }
/* Override default active line highlight colour in dark theme */ /* Override default active line highlight colour in dark theme */

39
packages/builder/src/components/integration/QueryViewer.svelte
View File

@ -21,12 +21,15 @@
import ParameterBuilder from "components/integration/QueryParameterBuilder.svelte" import ParameterBuilder from "components/integration/QueryParameterBuilder.svelte"
import { datasources, integrations, queries } from "stores/backend" import { datasources, integrations, queries } from "stores/backend"
import { capitalise } from "../../helpers" import { capitalise } from "../../helpers"
import CodeMirrorEditor from "components/common/CodeMirrorEditor.svelte"
export let query export let query
export let fields = [] export let fields = []
let parameters let parameters
let data = [] let data = []
const transformerDocs =
"https://docs.budibase.com/building-apps/data/transformers"
const typeOptions = [ const typeOptions = [
{ label: "Text", value: "STRING" }, { label: "Text", value: "STRING" },
{ label: "Number", value: "NUMBER" }, { label: "Number", value: "NUMBER" },
@ -52,6 +55,11 @@
$: readQuery = query.queryVerb === "read" || query.readable $: readQuery = query.queryVerb === "read" || query.readable
$: queryInvalid = !query.name || (readQuery && data.length === 0) $: queryInvalid = !query.name || (readQuery && data.length === 0)
// seed the transformer
if (query && !query.transformer) {
query.transformer = "return data"
}
function newField() { function newField() {
fields = [...fields, {}] fields = [...fields, {}]
} }
@ -74,6 +82,7 @@
const response = await api.post(`/api/queries/preview`, { const response = await api.post(`/api/queries/preview`, {
fields: query.fields, fields: query.fields,
queryVerb: query.queryVerb, queryVerb: query.queryVerb,
transformer: query.transformer,
parameters: query.parameters.reduce( parameters: query.parameters.reduce(
(acc, next) => ({ (acc, next) => ({
...acc, ...acc,
@ -160,12 +169,34 @@
<IntegrationQueryEditor <IntegrationQueryEditor
{datasource} {datasource}
{query} {query}
height={300} height={200}
schema={queryConfig[query.queryVerb]} schema={queryConfig[query.queryVerb]}
bind:parameters bind:parameters
/> />
<Divider /> <Divider />
</div> </div>
<div class="config">
<div class="help-heading">
<Heading size="S">Transformer</Heading>
<Icon
on:click={() => window.open(transformerDocs)}
hoverable
name="Help"
size="L"
/>
</div>
<Body size="S"
>Add a JavaScript function to transform the query result.</Body
>
<CodeMirrorEditor
height={200}
label="Transformer"
value={query.transformer}
resize="vertical"
on:change={e => (query.transformer = e.detail)}
/>
<Divider />
</div>
<div class="viewer-controls"> <div class="viewer-controls">
<Heading size="S">Results</Heading> <Heading size="S">Results</Heading>
<ButtonGroup> <ButtonGroup>
@ -220,6 +251,7 @@
display: grid; display: grid;
grid-gap: var(--spacing-s); grid-gap: var(--spacing-s);
} }
.config-field { .config-field {
display: grid; display: grid;
grid-template-columns: 20% 1fr; grid-template-columns: 20% 1fr;
@ -227,6 +259,11 @@
align-items: center; align-items: center;
} }
.help-heading {
display: flex;
justify-content: space-between;
}
.field { .field {
display: grid; display: grid;
grid-template-columns: 1fr 1fr 5%; grid-template-columns: 1fr 1fr 5%;

1
packages/server/package.json
View File

@ -119,6 +119,7 @@
"to-json-schema": "0.2.5", "to-json-schema": "0.2.5",
"uuid": "3.3.2", "uuid": "3.3.2",
"validate.js": "0.13.1", "validate.js": "0.13.1",
"vm2": "^3.9.3",
"yargs": "13.2.4", "yargs": "13.2.4",
"zlib": "1.0.5" "zlib": "1.0.5"
}, },

57
packages/server/src/api/controllers/query.js
View File

@ -4,6 +4,7 @@ const { generateQueryID, getQueryParams } = require("../../db/utils")
const { integrations } = require("../../integrations") const { integrations } = require("../../integrations")
const { BaseQueryVerbs } = require("../../constants") const { BaseQueryVerbs } = require("../../constants")
const env = require("../../environment") const env = require("../../environment")
const ScriptRunner = require("../../utilities/scriptRunner")
// simple function to append "readable" to all read queries // simple function to append "readable" to all read queries
function enrichQueries(input) { function enrichQueries(input) {
@ -28,12 +29,39 @@ function formatResponse(resp) {
resp = { response: resp } resp = { response: resp }
} }
} }
if (!Array.isArray(resp)) {
resp = [resp]
}
return resp return resp
} }
async function runAndTransform(
integration,
queryVerb,
enrichedQuery,
transformer
) {
let rows = formatResponse(await integration[queryVerb](enrichedQuery))
// transform as required
if (transformer) {
const runner = new ScriptRunner(transformer, { data: rows })
rows = runner.execute()
}
// needs to an array for next step
if (!Array.isArray(rows)) {
rows = [rows]
}
// map into JSON if just raw primitive here
if (rows.find(row => typeof row !== "object")) {
rows = rows.map(value => ({ value }))
}
// get all the potential fields in the schema
let keys = rows.flatMap(Object.keys)
return { rows, keys }
}
exports.fetch = async function (ctx) { exports.fetch = async function (ctx) {
const db = new CouchDB(ctx.appId) const db = new CouchDB(ctx.appId)
@ -122,15 +150,16 @@ exports.preview = async function (ctx) {
ctx.throw(400, "Integration type does not exist.") ctx.throw(400, "Integration type does not exist.")
} }
const { fields, parameters, queryVerb } = ctx.request.body const { fields, parameters, queryVerb, transformer } = ctx.request.body
const enrichedQuery = await enrichQueryFields(fields, parameters) const enrichedQuery = await enrichQueryFields(fields, parameters)
const integration = new Integration(datasource.config) const integration = new Integration(datasource.config)
const rows = formatResponse(await integration[queryVerb](enrichedQuery))
// get all the potential fields in the schema const { rows, keys } = await runAndTransform(
const keys = rows.flatMap(Object.keys) integration,
queryVerb,
enrichedQuery,
transformer
)
ctx.body = { ctx.body = {
rows, rows,
@ -158,10 +187,16 @@ exports.execute = async function (ctx) {
query.fields, query.fields,
ctx.request.body.parameters ctx.request.body.parameters
) )
const integration = new Integration(datasource.config) const integration = new Integration(datasource.config)
// call the relevant CRUD method on the integration class // call the relevant CRUD method on the integration class
ctx.body = formatResponse(await integration[query.queryVerb](enrichedQuery)) const { rows } = await runAndTransform(
integration,
query.queryVerb,
enrichedQuery,
query.transformer
)
ctx.body = rows
// cleanup // cleanup
if (integration.end) { if (integration.end) {
integration.end() integration.end()

23
packages/server/src/api/controllers/script.js
View File

@ -1,24 +1,9 @@
const fetch = require("node-fetch") const ScriptRunner = require("../../utilities/scriptRunner")
const vm = require("vm")
class ScriptExecutor {
constructor(body) {
const code = `let fn = () => {\n${body.script}\n}; out = fn();`
this.script = new vm.Script(code)
this.context = vm.createContext(body.context)
this.context.fetch = fetch
}
execute() {
this.script.runInContext(this.context)
return this.context.out
}
}
exports.execute = async function (ctx) { exports.execute = async function (ctx) {
const executor = new ScriptExecutor(ctx.request.body) const { script, context } = ctx.request.body
const runner = new ScriptRunner(script, context)
ctx.body = executor.execute() ctx.body = runner.execute()
} }
exports.save = async function (ctx) { exports.save = async function (ctx) {

4
packages/server/src/api/routes/query.js
View File

@ -31,7 +31,8 @@ function generateQueryValidation() {
})), })),
queryVerb: Joi.string().allow().required(), queryVerb: Joi.string().allow().required(),
extra: Joi.object().optional(), extra: Joi.object().optional(),
schema: Joi.object({}).required().unknown(true) schema: Joi.object({}).required().unknown(true),
transformer: Joi.string().optional(),
})) }))
} }
@ -42,6 +43,7 @@ function generateQueryPreviewValidation() {
queryVerb: Joi.string().allow().required(), queryVerb: Joi.string().allow().required(),
extra: Joi.object().optional(), extra: Joi.object().optional(),
datasourceId: Joi.string().required(), datasourceId: Joi.string().required(),
transformer: Joi.string().optional(),
parameters: Joi.object({}).required().unknown(true) parameters: Joi.object({}).required().unknown(true)
})) }))
} }

13
packages/server/src/automations/actions.js
View File

@ -3,7 +3,6 @@ const createRow = require("./steps/createRow")
const updateRow = require("./steps/updateRow") const updateRow = require("./steps/updateRow")
const deleteRow = require("./steps/deleteRow") const deleteRow = require("./steps/deleteRow")
const executeScript = require("./steps/executeScript") const executeScript = require("./steps/executeScript")
const bash = require("./steps/bash")
const executeQuery = require("./steps/executeQuery") const executeQuery = require("./steps/executeQuery")
const outgoingWebhook = require("./steps/outgoingWebhook") const outgoingWebhook = require("./steps/outgoingWebhook")
const serverLog = require("./steps/serverLog") const serverLog = require("./steps/serverLog")
@ -14,6 +13,7 @@ const integromat = require("./steps/integromat")
let filter = require("./steps/filter") let filter = require("./steps/filter")
let delay = require("./steps/delay") let delay = require("./steps/delay")
let queryRow = require("./steps/queryRows") let queryRow = require("./steps/queryRows")
const env = require("../environment")
const ACTION_IMPLS = { const ACTION_IMPLS = {
SEND_EMAIL_SMTP: sendSmtpEmail.run, SEND_EMAIL_SMTP: sendSmtpEmail.run,
@ -22,7 +22,6 @@ const ACTION_IMPLS = {
DELETE_ROW: deleteRow.run, DELETE_ROW: deleteRow.run,
OUTGOING_WEBHOOK: outgoingWebhook.run, OUTGOING_WEBHOOK: outgoingWebhook.run,
EXECUTE_SCRIPT: executeScript.run, EXECUTE_SCRIPT: executeScript.run,
EXECUTE_BASH: bash.run,
EXECUTE_QUERY: executeQuery.run, EXECUTE_QUERY: executeQuery.run,
SERVER_LOG: serverLog.run, SERVER_LOG: serverLog.run,
DELAY: delay.run, DELAY: delay.run,
@ -42,7 +41,6 @@ const ACTION_DEFINITIONS = {
OUTGOING_WEBHOOK: outgoingWebhook.definition, OUTGOING_WEBHOOK: outgoingWebhook.definition,
EXECUTE_SCRIPT: executeScript.definition, EXECUTE_SCRIPT: executeScript.definition,
EXECUTE_QUERY: executeQuery.definition, EXECUTE_QUERY: executeQuery.definition,
EXECUTE_BASH: bash.definition,
SERVER_LOG: serverLog.definition, SERVER_LOG: serverLog.definition,
DELAY: delay.definition, DELAY: delay.definition,
FILTER: filter.definition, FILTER: filter.definition,
@ -54,6 +52,15 @@ const ACTION_DEFINITIONS = {
integromat: integromat.definition, integromat: integromat.definition,
} }
// don't add the bash script/definitions unless in self host
// the fact this isn't included in any definitions means it cannot be
// ran at all
if (env.SELF_HOSTED) {
const bash = require("./steps/bash")
ACTION_IMPLS["EXECUTE_BASH"] = bash.run
ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
}
/* istanbul ignore next */ /* istanbul ignore next */
exports.getAction = async function (actionName) { exports.getAction = async function (actionName) {
if (ACTION_IMPLS[actionName] != null) { if (ACTION_IMPLS[actionName] != null) {

21
packages/server/src/utilities/scriptRunner.js Normal file
View File

@ -0,0 +1,21 @@
const fetch = require("node-fetch")
const { VM, VMScript } = require("vm2")
class ScriptRunner {
constructor(script, context) {
const code = `let fn = () => {\n${script}\n}; results.out = fn();`
this.vm = new VM()
this.results = { out: "" }
this.vm.setGlobals(context)
this.vm.setGlobal("fetch", fetch)
this.vm.setGlobal("results", this.results)
this.script = new VMScript(code)
}
execute() {
this.vm.run(this.script)
return this.results.out
}
}
module.exports = ScriptRunner

807
packages/server/yarn.lock
View File

File diff suppressed because it is too large Load Diff