Addomg a fix for #4370 - allow queries to contain newlines, they will always be escaped.

packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/actions/ExecuteQuery.svelte

@@ -52,7 +52,7 @@
       {/if}
 
       {#if query?.parameters?.length > 0}
-        <div>
+        <div class="params">
           <BindingBuilder
             bind:customParams={parameters.queryParams}
             queryBindings={query.parameters}
@@ -70,3 +70,11 @@
     {/if}
   {/if}
 </Layout>
+
+<style>
+  .params {
+    display: flex;
+    flex-direction: column;
+    gap: var(--spacing-l);
+  }
+</style>

packages/server/src/api/routes/tests/query.spec.js

@@ -230,7 +230,6 @@ describe("/queries", () => {
   })
 
   describe("variables", () => {
-
     async function preview(datasource, fields) {
       return config.previewQuery(request, config, datasource, fields)
     }

packages/server/src/threads/query.js

@@ -161,10 +161,16 @@ class QueryRunner {
       const responses = await Promise.all(dynamics)
       for (let i = 0; i < foundVars.length; i++) {
         const variable = foundVars[i]
-        parameters[variable.name] = processStringSync(variable.value, {
+        parameters[variable.name] = processStringSync(
+          variable.value,
+          {
             data: responses[i].rows,
             info: responses[i].extra,
-        })
+          },
+          {
+            escapeNewlines: true,
+          }
+        )
         // make sure its known that this uses dynamic variables in case it fails
         this.hasDynamicVariables = true
       }
@@ -188,6 +194,7 @@ class QueryRunner {
         enrichedQuery[key] = processStringSync(fields[key], parameters, {
           noEscaping: true,
           noHelpers: true,
+          escapeNewlines: true,
         })
       } else {
         enrichedQuery[key] = fields[key]

packages/string-templates/src/helpers/index.js

@@ -21,7 +21,7 @@ const HELPERS = [
   // javascript helper
   new Helper(HelperFunctionNames.JS, processJS, false),
   // this help is applied to all statements
-  new Helper(HelperFunctionNames.ALL, value => {
+  new Helper(HelperFunctionNames.ALL, (value, { __opts }) => {
     if (
       value != null &&
       typeof value === "object" &&
@@ -36,7 +36,11 @@ const HELPERS = [
     if (value && value.string) {
       value = value.string
     }
-    let text = new SafeString(value.replace(/&/g, "&"))
+    let text = value
+    if (__opts && __opts.escapeNewlines) {
+      text = value.replace(/\n/g, "\\n")
+    }
+    text = new SafeString(text.replace(/&/g, "&"))
     if (text == null || typeof text !== "string") {
       return text
     }
@@ -62,10 +66,14 @@ module.exports.HelperNames = () => {
   )
 }
 
-module.exports.registerAll = handlebars => {
+module.exports.registerMinimum = handlebars => {
   for (let helper of HELPERS) {
     helper.register(handlebars)
   }
+}
+
+module.exports.registerAll = handlebars => {
+  module.exports.registerMinimum(handlebars)
   // register imported helpers
   externalHandlebars.registerAll(handlebars)
 }

packages/string-templates/src/index.js

@@ -1,5 +1,5 @@
 const handlebars = require("handlebars")
-const { registerAll } = require("./helpers/index")
+const { registerAll, registerMinimum } = require("./helpers/index")
 const processors = require("./processors")
 const { atob, btoa } = require("./utilities")
 const manifest = require("../manifest.json")
@@ -8,6 +8,7 @@ const { FIND_HBS_REGEX, FIND_DOUBLE_HBS_REGEX } = require("./utilities")
 const hbsInstance = handlebars.create()
 registerAll(hbsInstance)
 const hbsInstanceNoHelpers = handlebars.create()
+registerMinimum(hbsInstanceNoHelpers)
 const defaultOpts = { noHelpers: false, noEscaping: false }
 
 /**
@@ -105,9 +106,7 @@ module.exports.processStringSync = (string, context, opts) => {
     throw "Cannot process non-string types."
   }
   try {
-    // finalising adds a helper, can't do this with no helpers
+    string = processors.preprocess(string, opts)
-    const shouldFinalise = !opts.noHelpers
-    string = processors.preprocess(string, shouldFinalise)
     // this does not throw an error when template can't be fulfilled, have to try correct beforehand
     const instance = opts.noHelpers ? hbsInstanceNoHelpers : hbsInstance
     const templateString =
@@ -119,8 +118,10 @@ module.exports.processStringSync = (string, context, opts) => {
     return processors.postprocess(
       template({
         now: new Date(now).toISOString(),
+        __opts: opts,
         ...context,
-      })
+      }),
+      { escapeNewlines: opts ? opts.escapeNewlines : false }
     )
   } catch (err) {
     return input

packages/string-templates/src/processors/index.js

@@ -2,7 +2,7 @@ const { FIND_HBS_REGEX } = require("../utilities")
 const preprocessor = require("./preprocessor")
 const postprocessor = require("./postprocessor")
 
-function process(output, processors) {
+function process(output, processors, opts) {
   for (let processor of processors) {
     // if a literal statement has occurred stop
     if (typeof output !== "string") {
@@ -15,24 +15,18 @@ function process(output, processors) {
       continue
     }
     for (let match of matches) {
-      output = processor.process(output, match)
+      output = processor.process(output, match, opts)
     }
   }
   return output
 }
 
-module.exports.preprocess = (string, finalise = true) => {
+module.exports.preprocess = (string, opts) => {
   let processors = preprocessor.processors
-  // the pre-processor finalisation stops handlebars from ever throwing an error
+  return process(string, processors, opts)
-  // might want to pre-process for other benefits but still want to see errors
-  if (!finalise) {
-    processors = processors.filter(
-      processor => processor.name !== preprocessor.PreprocessorNames.FINALISE
-    )
-  }
-  return process(string, processors)
 }
 
 module.exports.postprocess = string => {
-  return process(string, postprocessor.processors)
+  let processors = postprocessor.processors
+  return process(string, processors)
 }

packages/string-templates/src/processors/postprocessor.js

@@ -16,6 +16,8 @@ class Postprocessor {
   }
 }
 
+module.exports.PostProcessorNames = PostProcessorNames
+
 module.exports.processors = [
   new Postprocessor(PostProcessorNames.CONVERT_LITERALS, statement => {
     if (typeof statement !== "string" || !statement.includes(LITERAL_MARKER)) {

packages/string-templates/src/processors/preprocessor.js

@@ -16,8 +16,8 @@ class Preprocessor {
     this.fn = fn
   }
 
-  process(fullString, statement) {
+  process(fullString, statement, opts) {
-    const output = this.fn(statement)
+    const output = this.fn(statement, opts)
     const idx = fullString.indexOf(statement)
     return swapStrings(fullString, idx, statement.length, output)
   }
@@ -48,7 +48,8 @@ module.exports.processors = [
     return statement
   }),
 
-  new Preprocessor(PreprocessorNames.FINALISE, statement => {
+  new Preprocessor(PreprocessorNames.FINALISE, (statement, opts) => {
+    const noHelpers = opts && opts.noHelpers
     let insideStatement = statement.slice(2, statement.length - 2)
     if (insideStatement.charAt(0) === " ") {
       insideStatement = insideStatement.slice(1)
@@ -63,7 +64,10 @@ module.exports.processors = [
         return statement
       }
     }
-    if (HelperNames().some(option => option.includes(possibleHelper))) {
+    if (
+      !noHelpers &&
+      HelperNames().some(option => option.includes(possibleHelper))
+    ) {
       insideStatement = `(${insideStatement})`
     }
     return `{{ all ${insideStatement} }}`

packages/string-templates/test/escapes.spec.js

@@ -59,3 +59,33 @@ describe("attempt some complex problems", () => {
     expect(output).toBe("nulltest")
   })
 })
+
+describe("check behaviour with newlines", () => {
+  const context = {
+    binding: `Hello
+      there`
+  }
+  it("should escape new line to \\n with double brace", async () => {
+    const hbs = JSON.stringify({
+      body: "{{ binding }}"
+    })
+    const output = await processString(hbs, context, { escapeNewlines: true })
+    expect(JSON.parse(output).body).toBe(context.binding)
+  })
+
+  it("should work the same with triple brace", async () => {
+    const hbs = JSON.stringify({
+      body: "{{{ binding }}}"
+    })
+    const output = await processString(hbs, context, { escapeNewlines: true })
+    expect(JSON.parse(output).body).toBe(context.binding)
+  })
+
+  it("should still work with helpers disabled", async () => {
+    const hbs = JSON.stringify({
+      body: "{{ binding }}"
+    })
+    const output = await processString(hbs, context, { escapeNewlines: true, noHelpers: true })
+    expect(JSON.parse(output).body).toBe(context.binding)
+  })
+})