MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adding the basics of the query transformers to the frontend and to the backend, as well as switching to vm2 for script running.

This commit is contained in:
mike12345567 2021-10-12 18:45:13 +01:00
parent c01263e8f6
commit b46a945fc4
8 changed files with 891 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
packages/builder/src/components/common/CodeMirrorEditor.svelte
View File

@ -1,4 +1,6 @@
<script context="module"> <script context="module">
import { Label } from "@budibase/bbui"
export const EditorModes = { export const EditorModes = {
JS: { JS: {
name: "javascript", name: "javascript",
@ -28,6 +30,7 @@
export let height = 300 export let height = 300
export let readonly = false export let readonly = false
export let hints = [] export let hints = []
export let label
const dispatch = createEventDispatcher() const dispatch = createEventDispatcher()
let textarea let textarea
@ -110,6 +113,11 @@
}) })
</script> </script>
{#if label}
<div style="margin-bottom: var(--spacing-s)">
<Label small>{label}</Label>
</div>
{/if}
<div style={`--code-mirror-height: ${height}px`}> <div style={`--code-mirror-height: ${height}px`}>
<textarea tabindex="0" bind:this={textarea} readonly {value} /> <textarea tabindex="0" bind:this={textarea} readonly {value} />
</div> </div>

22
packages/builder/src/components/integration/QueryViewer.svelte
View File

@ -21,6 +21,7 @@
import ParameterBuilder from "components/integration/QueryParameterBuilder.svelte" import ParameterBuilder from "components/integration/QueryParameterBuilder.svelte"
import { datasources, integrations, queries } from "stores/backend" import { datasources, integrations, queries } from "stores/backend"
import { capitalise } from "../../helpers" import { capitalise } from "../../helpers"
import CodeMirrorEditor from "components/common/CodeMirrorEditor.svelte"
export let query export let query
export let fields = [] export let fields = []
@ -52,6 +53,11 @@
$: readQuery = query.queryVerb === "read" || query.readable $: readQuery = query.queryVerb === "read" || query.readable
$: queryInvalid = !query.name || (readQuery && data.length === 0) $: queryInvalid = !query.name || (readQuery && data.length === 0)
// seed the transformer
if (query && !query.transformer) {
query.transformer = "return data"
}
function newField() { function newField() {
fields = [...fields, {}] fields = [...fields, {}]
} }
@ -74,6 +80,7 @@
const response = await api.post(`/api/queries/preview`, { const response = await api.post(`/api/queries/preview`, {
fields: query.fields, fields: query.fields,
queryVerb: query.queryVerb, queryVerb: query.queryVerb,
transformer: query.transformer,
parameters: query.parameters.reduce( parameters: query.parameters.reduce(
(acc, next) => ({ (acc, next) => ({
...acc, ...acc,
@ -160,12 +167,25 @@
<IntegrationQueryEditor <IntegrationQueryEditor
{datasource} {datasource}
{query} {query}
height={300} height={200}
schema={queryConfig[query.queryVerb]} schema={queryConfig[query.queryVerb]}
bind:parameters bind:parameters
/> />
<Divider /> <Divider />
</div> </div>
<div class="config">
<Heading size="S">Transformer</Heading>
<Body size="S"
>Add a Javascript function to transform the query result.</Body
>
<CodeMirrorEditor
height={200}
label="Transformer"
value={query.transformer}
on:change={e => (query.transformer = e.detail)}
/>
<Divider />
</div>
<div class="viewer-controls"> <div class="viewer-controls">
<Heading size="S">Results</Heading> <Heading size="S">Results</Heading>
<ButtonGroup> <ButtonGroup>

1
packages/server/package.json
View File

@ -119,6 +119,7 @@
"to-json-schema": "0.2.5", "to-json-schema": "0.2.5",
"uuid": "3.3.2", "uuid": "3.3.2",
"validate.js": "0.13.1", "validate.js": "0.13.1",
"vm2": "^3.9.3",
"yargs": "13.2.4", "yargs": "13.2.4",
"zlib": "1.0.5" "zlib": "1.0.5"
}, },

50
packages/server/src/api/controllers/query.js
View File

@ -4,6 +4,7 @@ const { generateQueryID, getQueryParams } = require("../../db/utils")
const { integrations } = require("../../integrations") const { integrations } = require("../../integrations")
const { BaseQueryVerbs } = require("../../constants") const { BaseQueryVerbs } = require("../../constants")
const env = require("../../environment") const env = require("../../environment")
const ScriptRunner = require("../../utilities/scriptRunner")
// simple function to append "readable" to all read queries // simple function to append "readable" to all read queries
function enrichQueries(input) { function enrichQueries(input) {
@ -34,6 +35,32 @@ function formatResponse(resp) {
return resp return resp
} }
async function runAndTransform(
integration,
queryVerb,
enrichedQuery,
transformer
) {
let rows = formatResponse(await integration[queryVerb](enrichedQuery))
// transform as required
if (transformer) {
const runner = new ScriptRunner(transformer, { data: rows })
rows = runner.execute()
}
// get all the potential fields in the schema
let keys = rows.flatMap(Object.keys)
// map into JSON if just raw primitive here
if (keys.length === 0) {
rows = rows.map(value => ({ value }))
keys = ["value"]
}
return { rows, keys }
}
exports.fetch = async function (ctx) { exports.fetch = async function (ctx) {
const db = new CouchDB(ctx.appId) const db = new CouchDB(ctx.appId)
@ -122,15 +149,16 @@ exports.preview = async function (ctx) {
ctx.throw(400, "Integration type does not exist.") ctx.throw(400, "Integration type does not exist.")
} }
const { fields, parameters, queryVerb } = ctx.request.body const { fields, parameters, queryVerb, transformer } = ctx.request.body
const enrichedQuery = await enrichQueryFields(fields, parameters) const enrichedQuery = await enrichQueryFields(fields, parameters)
const integration = new Integration(datasource.config) const integration = new Integration(datasource.config)
const rows = formatResponse(await integration[queryVerb](enrichedQuery))
// get all the potential fields in the schema const { rows, keys } = await runAndTransform(
const keys = rows.flatMap(Object.keys) integration,
queryVerb,
enrichedQuery,
transformer
)
ctx.body = { ctx.body = {
rows, rows,
@ -158,10 +186,16 @@ exports.execute = async function (ctx) {
query.fields, query.fields,
ctx.request.body.parameters ctx.request.body.parameters
) )
const integration = new Integration(datasource.config) const integration = new Integration(datasource.config)
// call the relevant CRUD method on the integration class // call the relevant CRUD method on the integration class
ctx.body = formatResponse(await integration[query.queryVerb](enrichedQuery)) const { rows } = await runAndTransform(
integration,
query.queryVerb,
enrichedQuery,
query.transformer
)
ctx.body = rows
// cleanup // cleanup
if (integration.end) { if (integration.end) {
integration.end() integration.end()

23
packages/server/src/api/controllers/script.js
View File

@ -1,24 +1,9 @@
const fetch = require("node-fetch") const ScriptRunner = require("../../utilities/scriptRunner")
const vm = require("vm")
class ScriptExecutor {
constructor(body) {
const code = `let fn = () => {\n${body.script}\n}; out = fn();`
this.script = new vm.Script(code)
this.context = vm.createContext(body.context)
this.context.fetch = fetch
}
execute() {
this.script.runInContext(this.context)
return this.context.out
}
}
exports.execute = async function (ctx) { exports.execute = async function (ctx) {
const executor = new ScriptExecutor(ctx.request.body) const { script, context } = ctx.request.body
const runner = new ScriptRunner(script, context)
ctx.body = executor.execute() ctx.body = runner.execute()
} }
exports.save = async function (ctx) { exports.save = async function (ctx) {

4
packages/server/src/api/routes/query.js
View File

@ -31,7 +31,8 @@ function generateQueryValidation() {
})), })),
queryVerb: Joi.string().allow().required(), queryVerb: Joi.string().allow().required(),
extra: Joi.object().optional(), extra: Joi.object().optional(),
schema: Joi.object({}).required().unknown(true) schema: Joi.object({}).required().unknown(true),
transformer: Joi.string().optional(),
})) }))
} }
@ -42,6 +43,7 @@ function generateQueryPreviewValidation() {
queryVerb: Joi.string().allow().required(), queryVerb: Joi.string().allow().required(),
extra: Joi.object().optional(), extra: Joi.object().optional(),
datasourceId: Joi.string().required(), datasourceId: Joi.string().required(),
transformer: Joi.string().optional(),
parameters: Joi.object({}).required().unknown(true) parameters: Joi.object({}).required().unknown(true)
})) }))
} }

21
packages/server/src/utilities/scriptRunner.js Normal file
View File

@ -0,0 +1,21 @@
const fetch = require("node-fetch")
const { VM, VMScript } = require("vm2")
class ScriptRunner {
constructor(script, context) {
const code = `let fn = () => {\n${script}\n}; results.out = fn();`
this.vm = new VM()
this.results = { out: "" }
this.vm.setGlobals(context)
this.vm.setGlobal("fetch", fetch)
this.vm.setGlobal("results", this.results)
this.script = new VMScript(code)
}
execute() {
this.vm.run(this.script)
return this.results.out
}
}
module.exports = ScriptRunner

807
packages/server/yarn.lock
View File

File diff suppressed because it is too large Load Diff