Merge pull request #6986 from Budibase/labday/transpiler

JS helpers, development only HBS -> JS conversion

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -8,6 +8,7 @@
     Tab,
     Body,
     Layout,
+    Button,
   } from "@budibase/bbui"
   import { createEventDispatcher, onMount } from "svelte"
   import {
@@ -15,10 +16,15 @@
     decodeJSBinding,
     encodeJSBinding,
   } from "@budibase/string-templates"
-  import { readableToRuntimeBinding } from "builderStore/dataBinding"
+  import {
+    readableToRuntimeBinding,
+    runtimeToReadableBinding,
+  } from "builderStore/dataBinding"
   import { handlebarsCompletions } from "constants/completions"
   import { addHBSBinding, addJSBinding } from "./utils"
   import CodeMirrorEditor from "components/common/CodeMirrorEditor.svelte"
+  import { convertToJS } from "@budibase/string-templates"
+  import { admin } from "stores/portal"
 
   const dispatch = createEventDispatcher()
 
@@ -62,15 +68,24 @@
     }
   }
 
-  // Adds a HBS helper to the expression
-  const addHelper = helper => {
-    hbsValue = addHBSBinding(hbsValue, getCaretPosition(), helper.text)
-    updateValue(hbsValue)
+  // Adds a JS/HBS helper to the expression
+  const addHelper = (helper, js) => {
+    let tempVal
+    const pos = getCaretPosition()
+    if (js) {
+      const decoded = decodeJSBinding(jsValue)
+      tempVal = jsValue = encodeJSBinding(
+        addJSBinding(decoded, pos, helper.text, { helper: true })
+      )
+    } else {
+      tempVal = hbsValue = addHBSBinding(hbsValue, pos, helper.text)
+    }
+    updateValue(tempVal)
   }
 
   // Adds a data binding to the expression
-  const addBinding = binding => {
-    if (usingJS) {
+  const addBinding = (binding, { forceJS } = {}) => {
+    if (usingJS || forceJS) {
       let js = decodeJSBinding(jsValue)
       js = addJSBinding(js, getCaretPosition(), binding.readableBinding)
       jsValue = encodeJSBinding(js)
@@ -100,6 +115,26 @@
     updateValue(jsValue)
   }
 
+  const convert = () => {
+    const runtime = readableToRuntimeBinding(bindings, hbsValue)
+    const runtimeJs = encodeJSBinding(convertToJS(runtime))
+    jsValue = runtimeToReadableBinding(bindings, runtimeJs)
+    hbsValue = null
+    mode = "JavaScript"
+    addBinding("", { forceJS: true })
+  }
+
+  const getHelperExample = (helper, js) => {
+    let example = helper.example || ""
+    if (js) {
+      example = convertToJS(example).split("\n")[0].split("= ")[1]
+      if (example === "null;") {
+        example = ""
+      }
+    }
+    return example || ""
+  }
+
   onMount(() => {
     valid = isValid(readableToRuntimeBinding(bindings, value))
   })
@@ -135,18 +170,21 @@
           </section>
         {/if}
       {/each}
-      {#if filteredHelpers?.length && !usingJS}
+      {#if filteredHelpers?.length}
         <section>
           <div class="heading">Helpers</div>
           <ul>
             {#each filteredHelpers as helper}
-              <li on:click={() => addHelper(helper)}>
+              <li on:click={() => addHelper(helper, usingJS)}>
                 <div class="helper">
                   <div class="helper__name">{helper.displayText}</div>
                   <div class="helper__description">
                     {@html helper.description}
                   </div>
-                  <pre class="helper__example">{helper.example || ""}</pre>
+                  <pre class="helper__example">{getHelperExample(
+                      helper,
+                      usingJS
+                    )}</pre>
                 </div>
               </li>
             {/each}
@@ -172,6 +210,11 @@
               for more details.
             </p>
           {/if}
+          {#if $admin.isDev}
+            <div class="convert">
+              <Button secondary on:click={convert}>Convert to JS</Button>
+            </div>
+          {/if}
         </div>
       </Tab>
       {#if allowJS}
@@ -306,4 +349,8 @@
     color: var(--red);
     text-decoration: underline;
   }
+
+  .convert {
+    padding-top: var(--spacing-m);
+  }
 </style>

packages/builder/src/components/common/bindings/utils.js

@@ -18,10 +18,14 @@ export function addHBSBinding(value, caretPos, binding) {
   return value
 }
 
-export function addJSBinding(value, caretPos, binding) {
+export function addJSBinding(value, caretPos, binding, { helper } = {}) {
   binding = typeof binding === "string" ? binding : binding.path
   value = value == null ? "" : value
+  if (!helper) {
     binding = `$("${binding}")`
+  } else {
+    binding = `helper.${binding}()`
+  }
   if (caretPos.start) {
     value =
       value.substring(0, caretPos.start) +

packages/string-templates/manifest.json

@@ -74,7 +74,7 @@
         "b"
       ],
       "numArgs": 2,
-      "example": "{{ product 10 5 }} -> 50",
+      "example": "{{ multiply 10 5 }} -> 50",
       "description": "<p>Return the product of <code>a</code> times <code>b</code>.</p>\n"
     },
     "plus": {

packages/string-templates/scripts/gen-collection-info.js

@@ -108,6 +108,10 @@ function getCommentInfo(file, func) {
   if (examples.length > 0) {
     docs.example = examples.join(" ")
   }
+  // hacky example fix
+  if (docs.example && docs.example.includes("product")) {
+    docs.example = docs.example.replace("product", "multiply")
+  }
   docs.description = blocks[0].trim()
   return docs
 }
@@ -166,7 +170,7 @@ function run() {
   // convert all markdown to HTML
   for (let collection of Object.values(outputJSON)) {
     for (let helper of Object.values(collection)) {
-      helper.description = marked(helper.description)
+      helper.description = marked.parse(helper.description)
     }
   }
   fs.writeFileSync(FILENAME, JSON.stringify(outputJSON, null, 2))

packages/string-templates/src/conversion/index.js

@@ -0,0 +1,129 @@
+const { getHelperList } = require("../helpers")
+
+function getLayers(fullBlock) {
+  let layers = []
+  while (fullBlock.length) {
+    const start = fullBlock.lastIndexOf("("),
+      end = fullBlock.indexOf(")")
+    let layer
+    if (start === -1 || end === -1) {
+      layer = fullBlock.trim()
+      fullBlock = ""
+    } else {
+      const untrimmed = fullBlock.substring(start, end + 1)
+      layer = untrimmed.substring(1, untrimmed.length - 1).trim()
+      fullBlock =
+        fullBlock.slice(0, start) +
+        fullBlock.slice(start + untrimmed.length + 1, fullBlock.length)
+    }
+    layers.push(layer)
+  }
+  return layers
+}
+
+function getVariable(variableName) {
+  if (!variableName || typeof variableName !== "string") {
+    return variableName
+  }
+  // it is an array
+  const arrayOrObject = [",", "{", ":"]
+  let contains = false
+  arrayOrObject.forEach(char => {
+    if (variableName.includes(char)) {
+      contains = true
+    }
+  })
+  if (variableName.startsWith("[") && contains) {
+    return variableName
+  }
+  // it is just a number
+  if (!isNaN(parseFloat(variableName))) {
+    return variableName
+  }
+  if (variableName.startsWith("'") || variableName.startsWith('"')) {
+    return variableName
+  }
+  // extract variable
+  return `$("${variableName}")`
+}
+
+function buildList(parts, value) {
+  function build() {
+    return parts
+      .map(part => (part.startsWith("helper") ? part : getVariable(part)))
+      .join(", ")
+  }
+  if (!value) {
+    return parts.length > 1 ? `${build()}` : build()
+  } else {
+    return parts.length === 0 ? value : `${value}, ${build()}`
+  }
+}
+
+function splitBySpace(layer) {
+  const parts = []
+  let started = null,
+    endChar = null,
+    last = 0
+  function add(str) {
+    const startsWith = ["]"]
+    while (startsWith.indexOf(str.substring(0, 1)) !== -1) {
+      str = str.substring(1, str.length)
+    }
+    if (str.length > 0) {
+      parts.push(str.trim())
+    }
+  }
+  const continuationChars = ["[", "'", '"']
+  for (let index = 0; index < layer.length; index++) {
+    const char = layer[index]
+    if (continuationChars.indexOf(char) !== -1 && started == null) {
+      started = index
+      endChar = char === "[" ? "]" : char
+    } else if (
+      char === endChar &&
+      started != null &&
+      layer[index + 1] !== "."
+    ) {
+      add(layer.substring(started, index + 1))
+      started = null
+      endChar = null
+      last = index + 1
+    } else if (started == null && char === " ") {
+      add(layer.substring(last, index))
+      last = index
+    }
+  }
+  if (
+    (!layer.startsWith("[") || parts.length === 0) &&
+    last !== layer.length - 1
+  ) {
+    add(layer.substring(last, layer.length))
+  }
+  return parts
+}
+
+module.exports.convertHBSBlock = (block, blockNumber) => {
+  const braceLength = block[2] === "{" ? 3 : 2
+  block = block.substring(braceLength, block.length - braceLength).trim()
+  const layers = getLayers(block)
+
+  let value = null
+  const list = getHelperList()
+  for (let layer of layers) {
+    const parts = splitBySpace(layer)
+    if (value || parts.length > 1) {
+      // first of layer should always be the helper
+      const helper = parts.splice(0, 1)
+      if (list[helper]) {
+        value = `helpers.${helper}(${buildList(parts, value)})`
+      }
+    }
+    // no helpers
+    else {
+      value = getVariable(parts[0])
+    }
+  }
+  // split by space will remove square brackets
+  return { variable: `var${blockNumber}`, value }
+}

packages/string-templates/src/helpers/external.js

@@ -23,6 +23,9 @@ const ADDED_HELPERS = {
   duration: duration,
 }
 
+exports.externalCollections = EXTERNAL_FUNCTION_COLLECTIONS
+exports.addedHelpers = ADDED_HELPERS
+
 exports.registerAll = handlebars => {
   for (let [name, helper] of Object.entries(ADDED_HELPERS)) {
     handlebars.registerHelper(name, helper)

packages/string-templates/src/helpers/index.js

@@ -7,6 +7,7 @@ const {
   HelperFunctionBuiltin,
   LITERAL_MARKER,
 } = require("./constants")
+const { getHelperList } = require("./list")
 
 const HTML_SWAPS = {
   "<": "&lt;",
@@ -91,3 +92,5 @@ module.exports.unregisterAll = handlebars => {
   // unregister all imported helpers
   externalHandlebars.unregisterAll(handlebars)
 }
+
+module.exports.getHelperList = getHelperList

packages/string-templates/src/helpers/javascript.js

@@ -1,6 +1,7 @@
 const { atob } = require("../utilities")
 const { cloneDeep } = require("lodash/fp")
 const { LITERAL_MARKER } = require("../helpers/constants")
+const { getHelperList } = require("./list")
 
 // The method of executing JS scripts depends on the bundle being built.
 // This setter is used in the entrypoint (either index.cjs or index.mjs).
@@ -45,6 +46,7 @@ module.exports.processJS = (handlebars, context) => {
     // app context.
     const sandboxContext = {
       $: path => getContextValue(path, cloneDeep(context)),
+      helpers: getHelperList(),
     }
 
     // Create a sandbox with our context and run the JS

packages/string-templates/src/helpers/list.js

@@ -0,0 +1,19 @@
+const externalHandlebars = require("./external")
+const helperList = require("@budibase/handlebars-helpers")
+
+module.exports.getHelperList = () => {
+  let constructed = []
+  for (let collection of externalHandlebars.externalCollections) {
+    constructed.push(helperList[collection]())
+  }
+  const fullMap = {}
+  for (let collection of constructed) {
+    for (let [key, func] of Object.entries(collection)) {
+      fullMap[key] = func
+    }
+  }
+  for (let key of Object.keys(externalHandlebars.addedHelpers)) {
+    fullMap[key] = externalHandlebars.addedHelpers[key]
+  }
+  return fullMap
+}

packages/string-templates/src/index.cjs

@@ -19,6 +19,7 @@ module.exports.doesContainStrings = templates.doesContainStrings
 module.exports.doesContainString = templates.doesContainString
 module.exports.disableEscaping = templates.disableEscaping
 module.exports.findHBSBlocks = templates.findHBSBlocks
+module.exports.convertToJS = templates.convertToJS
 
 /**
  * Use vm2 to run JS scripts in a node env

packages/string-templates/src/index.js

@@ -8,6 +8,7 @@ const {
   FIND_ANY_HBS_REGEX,
   findDoubleHbsInstances,
 } = require("./utilities")
+const { convertHBSBlock } = require("./conversion")
 
 const hbsInstance = handlebars.create()
 registerAll(hbsInstance)
@@ -342,3 +343,31 @@ module.exports.findHBSBlocks = string => {
 module.exports.doesContainString = (template, string) => {
   return exports.doesContainStrings(template, [string])
 }
+
+module.exports.convertToJS = hbs => {
+  const blocks = exports.findHBSBlocks(hbs)
+  let js = "return `",
+    prevBlock = null
+  const variables = {}
+  if (blocks.length === 0) {
+    js += hbs
+  }
+  let count = 1
+  for (let block of blocks) {
+    let stringPart = hbs
+    if (prevBlock) {
+      stringPart = stringPart.split(prevBlock)[1]
+    }
+    stringPart = stringPart.split(block)[0]
+    prevBlock = block
+    const { variable, value } = convertHBSBlock(block, count++)
+    variables[variable] = value
+    js += `${stringPart.split()}\${${variable}}`
+  }
+  let varBlock = ""
+  for (let [variable, value] of Object.entries(variables)) {
+    varBlock += `const ${variable} = ${value};\n`
+  }
+  js += "`;"
+  return `${varBlock}${js}`
+}

packages/string-templates/src/index.mjs

@@ -19,6 +19,7 @@ export const doesContainStrings = templates.doesContainStrings
 export const doesContainString = templates.doesContainString
 export const disableEscaping = templates.disableEscaping
 export const findHBSBlocks = templates.findHBSBlocks
+export const convertToJS = templates.convertToJS
 
 /**
  * Use polyfilled vm to run JS scripts in a browser Env

packages/string-templates/test/hbsToJs.spec.js

@@ -0,0 +1,132 @@
+const {
+  convertToJS
+} = require("../src/index.cjs")
+
+function checkLines(response, lines) {
+  const toCheck = response.split("\n")
+  let count = 0
+  for (let line of lines) {
+    expect(toCheck[count++]).toBe(line)
+  }
+}
+
+describe("Test that the string processing works correctly", () => {
+  it("should convert string without HBS", () => {
+    const response = convertToJS("Hello my name is Michael")
+    expect(response).toBe("return `Hello my name is Michael`;")
+  })
+
+  it("basic example with square brackets", () => {
+    const response = convertToJS("{{ [query] }}")
+    checkLines(response, [
+      "const var1 = $(\"[query]\");",
+      "return `${var1}`;",
+    ])
+  })
+
+  it("handle properties", () => {
+    const response = convertToJS("{{ [query].id }}")
+    checkLines(response, [
+      "const var1 = $(\"[query].id\");",
+      "return `${var1}`;",
+    ])
+  })
+
+  it("should convert some basic HBS strings", () => {
+    const response = convertToJS("Hello {{ name }}, welcome to {{ company }}!")
+    checkLines(response, [
+      "const var1 = $(\"name\");",
+      "const var2 = $(\"company\");",
+      "return `Hello ${var1}, welcome to ${var2}`;",
+    ])
+  })
+
+  it("should handle many square brackets in helpers", () => {
+    const response = convertToJS("Hello {{ avg [user].[_id] [user].[_rev] }}")
+    checkLines(response, [
+      "const var1 = helpers.avg($(\"[user].[_id]\"), $(\"[user].[_rev]\"));",
+      "return `Hello ${var1}`;",
+    ])
+  })
+
+  it("should handle one of the examples (after)", () => {
+    const response = convertToJS("{{ after [1, 2, 3] 1}}")
+    checkLines(response, [
+      "const var1 = helpers.after([1, 2, 3], 1);",
+      "return `${var1}`;",
+    ])
+  })
+
+  it("should handle one of the examples (equalsLength)", () => {
+    const response = convertToJS("{{equalsLength '[1,2,3]' 3}}")
+    checkLines(response, [
+      "const var1 = helpers.equalsLength('[1,2,3]', 3);",
+      "return `${var1}`;"
+    ])
+  })
+
+  it("should handle one of the examples (pluck)", () => {
+    const response = convertToJS("{{pluck [{ 'name': 'Bob' }] 'name' }}")
+    checkLines(response, [
+      "const var1 = helpers.pluck([{ 'name': 'Bob' }], 'name');",
+      "return `${var1}`;",
+    ])
+  })
+
+  it("should handle sorting an array", () => {
+    const response = convertToJS("{{ sort ['b', 'a', 'c'] }}")
+    checkLines(response, [
+      "const var1 = helpers.sort(['b', 'a', 'c']);",
+      "return `${var1}`;",
+    ])
+  })
+
+  it("should handle a helper block", () => {
+    const response = convertToJS("This is the average: {{ avg array }}")
+    checkLines(response, [
+      "const var1 = helpers.avg($(\"array\"));",
+      "return `This is the average: ${var1}`;",
+    ])
+  })
+
+  it("should handle multi-variable helper", () => {
+    const response = convertToJS("This is the average: {{ join ( avg val1 val2 val3 ) }}")
+    checkLines(response, [
+      "const var1 = helpers.join(helpers.avg($(\"val1\"), $(\"val2\"), $(\"val3\")));",
+      "return `This is the average: ${var1}`;",
+    ])
+  })
+
+  it("should handle a complex statement", () => {
+    const response = convertToJS("This is the average: {{ join ( avg val1 val2 val3 ) val4 }}")
+    checkLines(response, [
+      "const var1 = helpers.join(helpers.avg($(\"val1\"), $(\"val2\"), $(\"val3\")), $(\"val4\"));",
+      "return `This is the average: ${var1}`;",
+    ])
+  })
+
+  it("should handle square brackets", () => {
+    const response = convertToJS("This is: {{ [val thing] }}")
+    checkLines(response, [
+      "const var1 = $(\"[val thing]\");",
+      "return `This is: ${var1}`;",
+    ])
+  })
+
+  it("should handle square brackets with properties", () => {
+    const response = convertToJS("{{ [user].[_id] }}")
+    checkLines(response, [
+      "const var1 = $(\"[user].[_id]\");",
+      "return `${var1}`;",
+    ])
+  })
+
+  it("should handle multiple complex statements", () => {
+    const response = convertToJS("average: {{ avg ( abs val1 ) val2 }} add: {{ add 1 2 }}")
+    checkLines(response, [
+      "const var1 = helpers.avg(helpers.abs($(\"val1\")), $(\"val2\"));",
+      "const var2 = helpers.add(1, 2);",
+      "return `average: ${var1} add: ${var2}`;",
+    ])
+  })
+})