Adding the basics of the query transformers to the frontend and to the backend, as well as switching to vm2 for script running.

2021-10-12 18:45:13 +01:00 · 2021-10-12 18:45:13 +01:00 · b46a945fc4
parent c01263e8f6
commit b46a945fc4
8 changed files with 891 additions and 45 deletions
--- a/packages/builder/src/components/common/CodeMirrorEditor.svelte
+++ b/packages/builder/src/components/common/CodeMirrorEditor.svelte
@ -1,4 +1,6 @@
 <script context="module">
+  import { Label } from "@budibase/bbui"
+
  export const EditorModes = {
    JS: {
      name: "javascript",
@ -28,6 +30,7 @@
  export let height = 300
  export let readonly = false
  export let hints = []
+  export let label

  const dispatch = createEventDispatcher()
  let textarea
@ -110,6 +113,11 @@
  })
 </script>

+{#if label}
+  <div style="margin-bottom: var(--spacing-s)">
+    <Label small>{label}</Label>
+  </div>
+{/if}
 <div style={`--code-mirror-height: ${height}px`}>
  <textarea tabindex="0" bind:this={textarea} readonly {value} />
 </div>
--- a/packages/builder/src/components/integration/QueryViewer.svelte
+++ b/packages/builder/src/components/integration/QueryViewer.svelte
@ -21,6 +21,7 @@
  import ParameterBuilder from "components/integration/QueryParameterBuilder.svelte"
  import { datasources, integrations, queries } from "stores/backend"
  import { capitalise } from "../../helpers"
+  import CodeMirrorEditor from "components/common/CodeMirrorEditor.svelte"

  export let query
  export let fields = []
@ -52,6 +53,11 @@
  $: readQuery = query.queryVerb === "read" || query.readable
  $: queryInvalid = !query.name || (readQuery && data.length === 0)

+  // seed the transformer
+  if (query && !query.transformer) {
+    query.transformer = "return data"
+  }
+
  function newField() {
    fields = [...fields, {}]
  }
@ -74,6 +80,7 @@
      const response = await api.post(`/api/queries/preview`, {
        fields: query.fields,
        queryVerb: query.queryVerb,
+        transformer: query.transformer,
        parameters: query.parameters.reduce(
          (acc, next) => ({
            ...acc,
@ -160,12 +167,25 @@
      <IntegrationQueryEditor
        {datasource}
        {query}
-        height={300}
+        height={200}
        schema={queryConfig[query.queryVerb]}
        bind:parameters
      />
      <Divider />
    </div>
+    <div class="config">
+      <Heading size="S">Transformer</Heading>
+      <Body size="S"
+        >Add a Javascript function to transform the query result.</Body
+      >
+      <CodeMirrorEditor
+        height={200}
+        label="Transformer"
+        value={query.transformer}
+        on:change={e => (query.transformer = e.detail)}
+      />
+      <Divider />
+    </div>
    <div class="viewer-controls">
      <Heading size="S">Results</Heading>
      <ButtonGroup>
--- a/packages/server/package.json
+++ b/packages/server/package.json
@ -119,6 +119,7 @@
    "to-json-schema": "0.2.5",
    "uuid": "3.3.2",
    "validate.js": "0.13.1",
+    "vm2": "^3.9.3",
    "yargs": "13.2.4",
    "zlib": "1.0.5"
  },
--- a/packages/server/src/api/controllers/query.js
+++ b/packages/server/src/api/controllers/query.js
@ -4,6 +4,7 @@ const { generateQueryID, getQueryParams } = require("../../db/utils")
 const { integrations } = require("../../integrations")
 const { BaseQueryVerbs } = require("../../constants")
 const env = require("../../environment")
+const ScriptRunner = require("../../utilities/scriptRunner")

 // simple function to append "readable" to all read queries
 function enrichQueries(input) {
@ -34,6 +35,32 @@ function formatResponse(resp) {
  return resp
 }

+async function runAndTransform(
+  integration,
+  queryVerb,
+  enrichedQuery,
+  transformer
+) {
+  let rows = formatResponse(await integration[queryVerb](enrichedQuery))
+
+  // transform as required
+  if (transformer) {
+    const runner = new ScriptRunner(transformer, { data: rows })
+    rows = runner.execute()
+  }
+
+  // get all the potential fields in the schema
+  let keys = rows.flatMap(Object.keys)
+
+  // map into JSON if just raw primitive here
+  if (keys.length === 0) {
+    rows = rows.map(value => ({ value }))
+    keys = ["value"]
+  }
+
+  return { rows, keys }
+}
+
 exports.fetch = async function (ctx) {
  const db = new CouchDB(ctx.appId)

@ -122,15 +149,16 @@ exports.preview = async function (ctx) {
    ctx.throw(400, "Integration type does not exist.")
  }

-  const { fields, parameters, queryVerb } = ctx.request.body
-
+  const { fields, parameters, queryVerb, transformer } = ctx.request.body
  const enrichedQuery = await enrichQueryFields(fields, parameters)
-
  const integration = new Integration(datasource.config)
-  const rows = formatResponse(await integration[queryVerb](enrichedQuery))

-  // get all the potential fields in the schema
-  const keys = rows.flatMap(Object.keys)
+  const { rows, keys } = await runAndTransform(
+    integration,
+    queryVerb,
+    enrichedQuery,
+    transformer
+  )

  ctx.body = {
    rows,
@ -158,10 +186,16 @@ exports.execute = async function (ctx) {
    query.fields,
    ctx.request.body.parameters
  )
-
  const integration = new Integration(datasource.config)
+
  // call the relevant CRUD method on the integration class
-  ctx.body = formatResponse(await integration[query.queryVerb](enrichedQuery))
+  const { rows } = await runAndTransform(
+    integration,
+    query.queryVerb,
+    enrichedQuery,
+    query.transformer
+  )
+  ctx.body = rows
  // cleanup
  if (integration.end) {
    integration.end()
--- a/packages/server/src/api/controllers/script.js
+++ b/packages/server/src/api/controllers/script.js
@ -1,24 +1,9 @@
-const fetch = require("node-fetch")
-const vm = require("vm")
-
-class ScriptExecutor {
-  constructor(body) {
-    const code = `let fn = () => {\n${body.script}\n}; out = fn();`
-    this.script = new vm.Script(code)
-    this.context = vm.createContext(body.context)
-    this.context.fetch = fetch
-  }
-
-  execute() {
-    this.script.runInContext(this.context)
-    return this.context.out
-  }
-}
+const ScriptRunner = require("../../utilities/scriptRunner")

 exports.execute = async function (ctx) {
-  const executor = new ScriptExecutor(ctx.request.body)
-
-  ctx.body = executor.execute()
+  const { script, context } = ctx.request.body
+  const runner = new ScriptRunner(script, context)
+  ctx.body = runner.execute()
 }

 exports.save = async function (ctx) {
--- a/packages/server/src/api/routes/query.js
+++ b/packages/server/src/api/routes/query.js
@ -31,7 +31,8 @@ function generateQueryValidation() {
    })),
    queryVerb: Joi.string().allow().required(),
    extra: Joi.object().optional(),
-    schema: Joi.object({}).required().unknown(true)
+    schema: Joi.object({}).required().unknown(true),
+    transformer: Joi.string().optional(),
  }))
 }

@ -42,6 +43,7 @@ function generateQueryPreviewValidation() {
    queryVerb: Joi.string().allow().required(),
    extra: Joi.object().optional(),
    datasourceId: Joi.string().required(),
+    transformer: Joi.string().optional(),
    parameters: Joi.object({}).required().unknown(true)
  }))
 }
--- a/packages/server/src/utilities/scriptRunner.js
+++ b/packages/server/src/utilities/scriptRunner.js
@ -0,0 +1,21 @@
+const fetch = require("node-fetch")
+const { VM, VMScript } = require("vm2")
+
+class ScriptRunner {
+  constructor(script, context) {
+    const code = `let fn = () => {\n${script}\n}; results.out = fn();`
+    this.vm = new VM()
+    this.results = { out: "" }
+    this.vm.setGlobals(context)
+    this.vm.setGlobal("fetch", fetch)
+    this.vm.setGlobal("results", this.results)
+    this.script = new VMScript(code)
+  }
+
+  execute() {
+    this.vm.run(this.script)
+    return this.results.out
+  }
+}
+
+module.exports = ScriptRunner
--- a/packages/server/yarn.lock
+++ b/packages/server/yarn.lock