Merge branch 'feature/handlebars-migration' of github.com:Budibase/budibase into form-builder

packages/builder/package.json

@@ -66,6 +66,7 @@
     "@budibase/bbui": "^1.54.0",
     "@budibase/client": "^0.5.3",
     "@budibase/colorpicker": "^1.0.1",
+    "@budibase/string-templates": "^0.5.3",
     "@budibase/svelte-ag-grid": "^0.0.16",
     "@sentry/browser": "5.19.1",
     "@svelteschool/svelte-forms": "^0.7.0",
@@ -75,7 +76,6 @@
     "deepmerge": "^4.2.2",
     "fast-sort": "^2.2.0",
     "lodash": "^4.17.13",
-    "mustache": "^4.0.1",
     "posthog-js": "1.4.5",
     "remixicon": "^2.5.0",
     "shortid": "^2.2.15",

packages/builder/src/builderStore/dataBinding.js

@@ -3,8 +3,8 @@ import { get } from "svelte/store"
 import { backendUiStore, store } from "builderStore"
 import { findAllMatchingComponents, findComponentPath } from "./storeUtils"
 
-// Regex to match mustache variables, for replacing bindings
-const CAPTURE_VAR_INSIDE_MUSTACHE = /{{([^}]+)}}/g
+// Regex to match all instances of template strings
+const CAPTURE_VAR_INSIDE_TEMPLATE = /{{([^}]+)}}/g
 
 /**
  * Gets all bindable data context fields and instance fields.
@@ -173,7 +173,7 @@ export function readableToRuntimeBinding(bindableProperties, textWithBindings) {
   if (typeof textWithBindings !== "string") {
     return textWithBindings
   }
-  const boundValues = textWithBindings.match(CAPTURE_VAR_INSIDE_MUSTACHE) || []
+  const boundValues = textWithBindings.match(CAPTURE_VAR_INSIDE_TEMPLATE) || []
   let result = textWithBindings
   boundValues.forEach(boundValue => {
     const binding = bindableProperties.find(({ readableBinding }) => {
@@ -193,7 +193,7 @@ export function runtimeToReadableBinding(bindableProperties, textWithBindings) {
   if (typeof textWithBindings !== "string") {
     return textWithBindings
   }
-  const boundValues = textWithBindings.match(CAPTURE_VAR_INSIDE_MUSTACHE) || []
+  const boundValues = textWithBindings.match(CAPTURE_VAR_INSIDE_TEMPLATE) || []
   let result = textWithBindings
   boundValues.forEach(boundValue => {
     const binding = bindableProperties.find(({ runtimeBinding }) => {

packages/builder/src/builderStore/uuid.js

@@ -1,6 +1,6 @@
 export function uuid() {
   // always want to make this start with a letter, as this makes it
-  // easier to use with mustache bindings in the client
+  // easier to use with template string bindings in the client
   return "cxxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx".replace(/[xy]/g, c => {
     const r = (Math.random() * 16) | 0,
       v = c == "x" ? r : (r & 0x3) | 0x8

packages/builder/src/components/automation/AutomationBuilder/FlowChart/AutomationBlockTagline.svelte

@@ -1,5 +1,5 @@
 <script>
-  import mustache from "mustache"
+  import { processStringSync } from "@budibase/string-templates"
   import { get } from "lodash/fp"
   import { backendUiStore } from "builderStore"
 
@@ -54,8 +54,8 @@
       }
     })
 
-    // Fill in bindings with mustache
-    return mustache.render(formattedTagline, { inputs })
+    // Fill in bindings with templating library
+    return processStringSync(formattedTagline, { inputs })
   }
 </script>
 

packages/builder/yarn.lock

@@ -5324,11 +5324,6 @@ ms@2.1.2, ms@^2.1.1:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
   integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
 
-mustache@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/mustache/-/mustache-4.0.1.tgz#d99beb031701ad433338e7ea65e0489416c854a2"
-  integrity sha512-yL5VE97+OXn4+Er3THSmTdCFCtx5hHWzrolvH+JObZnUYwuaG7XV+Ch4fR2cIrcYI0tFHxS7iyFYl14bW8y2sA==
-
 nan@^2.12.1:
   version "2.14.2"
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.2.tgz#f5376400695168f4cc694ac9393d0c9585eeea19"

packages/client/package.json

@@ -9,8 +9,8 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
+    "@budibase/string-templates": "^0.5.3",
     "deep-equal": "^2.0.1",
-    "mustache": "^4.0.1",
     "regexparam": "^1.3.0",
     "svelte-spa-router": "^3.0.5"
   },

packages/client/src/components/Component.svelte

@@ -7,7 +7,9 @@
   import { bindingStore, builderStore } from "../store"
 
   export let definition = {}
-  let componentProps = {}
+
+  let enrichedProps
+  let componentProps
 
   // Get contexts
   const dataContext = getContext("data")
@@ -21,7 +23,7 @@
   $: constructor = getComponentConstructor(definition._component)
   $: children = definition._children
   $: id = definition._id
-  $: enrichedProps = enrichProps(definition, $dataContext, $bindingStore)
+  $: enrichComponentProps(definition, $dataContext, $bindingStore)
   $: updateProps(enrichedProps)
   $: styles = definition._styles
 
@@ -37,6 +39,12 @@
   // Most props are deeply compared so that svelte will only trigger reactive
   // statements on props that have actually changed.
   const updateProps = props => {
+    if (!props) {
+      return
+    }
+    if (!componentProps) {
+      componentProps = {}
+    }
     Object.keys(props).forEach(key => {
       if (!propsAreSame(props[key], componentProps[key])) {
         componentProps[key] = props[key]
@@ -54,6 +62,11 @@
     return ComponentLibrary[name]
   }
 
+  // Enriches any string component props using handlebars
+  const enrichComponentProps = async (definition, context, bindingStore) => {
+    enrichedProps = await enrichProps(definition, context, bindingStore)
+  }
+
   // Returns a unique key to let svelte know when to remount components.
   // If a component is selected we want to remount it every time any props
   // change.
@@ -63,7 +76,7 @@
   }
 </script>
 
-{#if constructor}
+{#if constructor && componentProps}
   <svelte:component this={constructor} {...componentProps}>
     {#if children && children.length}
       {#each children as child (getChildKey(child._id))}

packages/client/src/utils/buttonActions.js

@@ -6,11 +6,11 @@ import { saveRow, deleteRow, executeQuery, triggerAutomation } from "../api"
 const saveRowHandler = async (action, context) => {
   const { fields, providerId } = action.parameters
   if (providerId) {
-    let draft = context[`${action.parameters.providerId}_draft`]
+    let draft = context[`${providerId}_draft`]
     if (fields) {
-      Object.entries(fields).forEach(([key, entry]) => {
-        draft[key] = enrichDataBinding(entry.value, context)
-      })
+      for (let [key, entry] of Object.entries(fields)) {
+        draft[key] = await enrichDataBinding(entry.value, context)
+      }
     }
     await saveRow(draft)
   }
@@ -19,25 +19,28 @@ const saveRowHandler = async (action, context) => {
 const deleteRowHandler = async (action, context) => {
   const { tableId, revId, rowId } = action.parameters
   if (tableId && revId && rowId) {
+    const [enrichTable, enrichRow, enrichRev] = await Promise.all([
+      enrichDataBinding(tableId, context),
+      enrichDataBinding(rowId, context),
+      enrichDataBinding(revId, context),
+    ])
     await deleteRow({
-      tableId: enrichDataBinding(tableId, context),
-      rowId: enrichDataBinding(rowId, context),
-      revId: enrichDataBinding(revId, context),
+      tableId: enrichTable,
+      rowId: enrichRow,
+      revId: enrichRev,
     })
   }
 }
 
 const triggerAutomationHandler = async (action, context) => {
+  const { fields } = action.parameters()
+  if (fields) {
     const params = {}
-  if (action.parameters.fields) {
-    for (let field in action.parameters.fields) {
-      params[field] = enrichDataBinding(
-        action.parameters.fields[field].value,
-        context
-      )
-    }
+    for (let field in fields) {
+      params[field] = await enrichDataBinding(fields[field].value, context)
     }
     await triggerAutomation(action.parameters.automationId, params)
+  }
 }
 
 const navigationHandler = action => {

packages/client/src/utils/componentProps.js

@@ -21,7 +21,7 @@ export const propsAreSame = (a, b) => {
  * Enriches component props.
  * Data bindings are enriched, and button actions are enriched.
  */
-export const enrichProps = (props, dataContexts, dataBindings) => {
+export const enrichProps = async (props, dataContexts, dataBindings) => {
   // Exclude all private props that start with an underscore
   let validProps = {}
   Object.entries(props)
@@ -40,7 +40,7 @@ export const enrichProps = (props, dataContexts, dataBindings) => {
   }
 
   // Enrich all data bindings in top level props
-  let enrichedProps = enrichDataBindings(validProps, context)
+  let enrichedProps = await enrichDataBindings(validProps, context)
 
   // Enrich button actions if they exist
   if (props._component.endsWith("/button") && enrichedProps.onClick) {

packages/client/src/utils/enrichDataBinding.js

@@ -1,48 +1,30 @@
 import { cloneDeep } from "lodash/fp"
-import mustache from "mustache"
+import { processString, processObject } from "@budibase/string-templates"
 
-// this is a much more liberal version of mustache's escape function
-// ...just ignoring < and > to prevent tags from user input
-// original version here https://github.com/janl/mustache.js/blob/4b7908f5c9fec469a11cfaed2f2bed23c84e1c5c/mustache.js#L78
-const entityMap = {
-  "<": "&lt;",
-  ">": "&gt;",
-}
-mustache.escape = text => {
-  if (text == null || typeof text !== "string") {
-    return text
-  }
-  return text.replace(/[<>]/g, function fromEntityMap(s) {
-    return entityMap[s] || s
-  })
-}
-
-// Regex to test inputs with to see if they are likely candidates for mustache
-const looksLikeMustache = /{{.*}}/
+// Regex to test inputs with to see if they are likely candidates for template strings
+const looksLikeTemplate = /{{.*}}/
 
 /**
  * Enriches a given input with a row from the database.
  */
-export const enrichDataBinding = (input, context) => {
+export const enrichDataBinding = async (input, context) => {
   // Only accept string inputs
   if (!input || typeof input !== "string") {
     return input
   }
-  // Do a fast regex check if this looks like a mustache string
-  if (!looksLikeMustache.test(input)) {
+  // Do a fast regex check if this looks like a template string
+  if (!looksLikeTemplate.test(input)) {
     return input
   }
-  return mustache.render(input, context)
+  return processString(input, context)
 }
 
 /**
  * Recursively enriches all props in a props object and returns the new props.
  * Props are deeply cloned so that no mutation is done to the source object.
  */
-export const enrichDataBindings = (props, context) => {
-  let clonedProps = cloneDeep(props)
-  recursiveEnrich(clonedProps, context)
-  return clonedProps
+export const enrichDataBindings = async (props, context) => {
+  return await processObject(cloneDeep(props), context)
 }
 
 /**

packages/client/yarn.lock

@@ -1362,11 +1362,6 @@ minimatch@^3.0.4:
   dependencies:
     brace-expansion "^1.1.7"
 
-mustache@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/mustache/-/mustache-4.0.1.tgz#d99beb031701ad433338e7ea65e0489416c854a2"
-  integrity sha512-yL5VE97+OXn4+Er3THSmTdCFCtx5hHWzrolvH+JObZnUYwuaG7XV+Ch4fR2cIrcYI0tFHxS7iyFYl14bW8y2sA==
-
 nwsapi@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.0.tgz#204879a9e3d068ff2a55139c2c772780681a38b7"

packages/server/package.json

@@ -51,6 +51,7 @@
   "dependencies": {
     "@elastic/elasticsearch": "^7.10.0",
     "@budibase/client": "^0.5.3",
+    "@budibase/string-templates": "^0.5.3",
     "@koa/router": "^8.0.0",
     "@sendgrid/mail": "^7.1.1",
     "@sentry/node": "^5.19.2",
@@ -67,7 +68,6 @@
     "electron-util": "^0.14.2",
     "fix-path": "^3.0.0",
     "fs-extra": "^8.1.0",
-    "handlebars": "^4.7.6",
     "jimp": "^0.16.1",
     "joi": "^17.2.1",
     "jsonschema": "^1.4.0",
@@ -82,7 +82,6 @@
     "lodash": "^4.17.13",
     "mongodb": "^3.6.3",
     "mssql": "^6.2.3",
-    "mustache": "^4.0.1",
     "node-fetch": "^2.6.0",
     "open": "^7.3.0",
     "pg": "^8.5.1",

packages/server/src/api/controllers/application.js

@@ -29,7 +29,7 @@ const {
   createLoginScreen,
 } = require("../../constants/screens")
 const { cloneDeep } = require("lodash/fp")
-const { recurseMustache } = require("../../utilities/mustache")
+const { processObject } = require("@budibase/string-templates")
 const { getAllApps } = require("../../utilities")
 const { USERS_TABLE_SCHEMA } = require("../../constants")
 const {
@@ -232,8 +232,7 @@ const createEmptyAppPackage = async (ctx, app) => {
   let screensAndLayouts = []
   for (let layout of BASE_LAYOUTS) {
     const cloned = cloneDeep(layout)
-    cloned.title = app.name
-    screensAndLayouts.push(recurseMustache(cloned, app))
+    screensAndLayouts.push(await processObject(cloned, app))
   }
 
   const homeScreen = createHomeScreen(app)

packages/server/src/api/controllers/static/index.js

@@ -7,7 +7,7 @@ const fs = require("fs-extra")
 const uuid = require("uuid")
 const AWS = require("aws-sdk")
 const { prepareUpload } = require("../deploy/utils")
-const handlebars = require("handlebars")
+const { processString } = require("@budibase/string-templates")
 const {
   budibaseAppsDir,
   budibaseTempDir,
@@ -176,11 +176,8 @@ exports.serveApp = async function(ctx) {
     objectStoreUrl: objectStoreUrl(),
   })
 
-  const template = handlebars.compile(
-    fs.readFileSync(`${__dirname}/templates/app.hbs`, "utf8")
-  )
-
-  ctx.body = template({
+  const appHbs = fs.readFileSync(`${__dirname}/templates/app.hbs`, "utf8")
+  ctx.body = await processString(appHbs, {
     head,
     body: html,
     style: css.code,

packages/server/src/automations/automationUtils.js

@@ -1,10 +1,11 @@
 const CouchDB = require("../db")
 
 /**
- * When values are input to the system generally they will be of type string as this is required for mustache. This can
- * generate some odd scenarios as the Schema of the automation requires a number but the builder might supply a string
- * with mustache syntax to get the number from the rest of the context. To support this the server has to make sure that
- * the post mustache statement can be cast into the correct type, this function does this for numbers and booleans.
+ * When values are input to the system generally they will be of type string as this is required for template strings.
+ * This can generate some odd scenarios as the Schema of the automation requires a number but the builder might supply
+ * a string with template syntax to get the number from the rest of the context. To support this the server has to
+ * make sure that the post template statement can be cast into the correct type, this function does this for numbers
+ * and booleans.
  *
  * @param {object} inputs An object of inputs, please note this will not recurse down into any objects within, it simply
  * cleanses the top level inputs, however it can be used by recursively calling it deeper into the object structures if
@@ -54,7 +55,7 @@ module.exports.cleanInputValues = (inputs, schema) => {
  *
  * @param {string} appId The instance which the Table/Table is contained under.
  * @param {string} tableId The ID of the Table/Table which the schema is to be retrieved for.
- * @param {object} row The input row structure which requires clean-up after having been through mustache statements.
+ * @param {object} row The input row structure which requires clean-up after having been through template statements.
  * @returns {Promise<Object>} The cleaned up rows object, will should now have all the required primitive types.
  */
 module.exports.cleanUpRow = async (appId, tableId, row) => {
@@ -66,11 +67,11 @@ module.exports.cleanUpRow = async (appId, tableId, row) => {
 
 /**
  * A utility function for the cleanUpRow, which can be used if only the row ID is known (not the table ID) to clean
- * up a row after mustache statements have been replaced. This is specifically useful for the update row action.
+ * up a row after template statements have been replaced. This is specifically useful for the update row action.
  *
  * @param {string} appId The instance which the Table/Table is contained under.
  * @param {string} rowId The ID of the row from which the tableId will be extracted, to get the Table/Table schema.
- * @param {object} row The input row structure which requires clean-up after having been through mustache statements.
+ * @param {object} row The input row structure which requires clean-up after having been through template statements.
  * @returns {Promise<Object>} The cleaned up rows object, which will now have all the required primitive types.
  */
 module.exports.cleanUpRowById = async (appId, rowId, row) => {

packages/server/src/automations/thread.js

@@ -1,13 +1,8 @@
-const handlebars = require("handlebars")
 const actions = require("./actions")
 const logic = require("./logic")
 const automationUtils = require("./automationUtils")
 const AutomationEmitter = require("../events/AutomationEmitter")
-const { recurseMustache } = require("../utilities/mustache")
-
-handlebars.registerHelper("object", value => {
-  return new handlebars.SafeString(JSON.stringify(value))
-})
+const { processObject } = require("@budibase/string-templates")
 
 const FILTER_STEP_ID = logic.BUILTIN_DEFINITIONS.FILTER.stepId
 
@@ -24,7 +19,7 @@ class Orchestrator {
     // remove from context
     delete triggerOutput.appId
     delete triggerOutput.metadata
-    // step zero is never used as the mustache is zero indexed for customer facing
+    // step zero is never used as the template string is zero indexed for customer facing
     this._context = { steps: [{}], trigger: triggerOutput }
     this._automation = automation
     // create an emitter which has the chain count for this automation run in it, so it can block
@@ -49,7 +44,7 @@ class Orchestrator {
     let automation = this._automation
     for (let step of automation.definition.steps) {
       let stepFn = await this.getStepFunctionality(step.type, step.stepId)
-      step.inputs = recurseMustache(step.inputs, this._context)
+      step.inputs = await processObject(step.inputs, this._context)
       step.inputs = automationUtils.cleanInputValues(
         step.inputs,
         step.schema.inputs

packages/server/src/events/index.js

@@ -9,7 +9,7 @@ const { rowEmission, tableEmission } = require("./utils")
 
 /**
  * Extending the standard emitter to some syntactic sugar and standardisation to the emitted event.
- * This is specifically quite important for mustache used in automations.
+ * This is specifically quite important for template strings used in automations.
  */
 class BudibaseEmitter extends EventEmitter {
   emitRow(eventName, appId, row, table = null) {

packages/server/src/utilities/initialiseBudibase.js

@@ -1,6 +1,6 @@
 const { existsSync, readFile, writeFile, ensureDir } = require("fs-extra")
 const { join, resolve } = require("./centralPath")
-const handlebars = require("handlebars")
+const { processString } = require("@budibase/string-templates")
 const uuid = require("uuid")
 
 module.exports = async opts => {
@@ -31,8 +31,7 @@ const createDevEnvFile = async opts => {
       }
     )
     opts.cookieKey1 = opts.cookieKey1 || uuid.v4()
-    const envTemplate = handlebars.compile(template)
-    const config = envTemplate(opts)
+    const config = await processString(template, opts)
     await writeFile(destConfigFile, config, { flag: "w+" })
   }
 }

packages/server/src/utilities/mustache.js

@@ -1,73 +0,0 @@
-const handlebars = require("handlebars")
-
-handlebars.registerHelper("object", value => {
-  return new handlebars.SafeString(JSON.stringify(value))
-})
-
-/**
- * When running mustache statements to execute on the context of the automation it possible user's may input mustache
- * in a few different forms, some of which are invalid but are logically valid. An example of this would be the mustache
- * statement "{{steps[0].revision}}" here it is obvious the user is attempting to access an array or object using array
- * like operators. These are not supported by Mustache and therefore the statement will fail. This function will clean up
- * the mustache statement so it instead reads as "{{steps.0.revision}}" which is valid and will work. It may also be expanded
- * to include any other mustache statement cleanup that has been deemed necessary for the system.
- *
- * @param {string} string The string which *may* contain mustache statements, it is OK if it does not contain any.
- * @returns {string} The string that was input with cleaned up mustache statements as required.
- */
-function cleanMustache(string) {
-  let charToReplace = {
-    "[": ".",
-    "]": "",
-  }
-  let regex = new RegExp(/{{[^}}]*}}/g)
-  let matches = string.match(regex)
-  if (matches == null) {
-    return string
-  }
-  for (let match of matches) {
-    let baseIdx = string.indexOf(match)
-    for (let key of Object.keys(charToReplace)) {
-      let idxChar = match.indexOf(key)
-      if (idxChar !== -1) {
-        string =
-          string.slice(baseIdx, baseIdx + idxChar) +
-          charToReplace[key] +
-          string.slice(baseIdx + idxChar + 1)
-      }
-    }
-  }
-  return string
-}
-
-/**
- * Given an input object this will recurse through all props to try and update
- * any handlebars/mustache statements within.
- * @param {object|array} inputs The input structure which is to be recursed, it is important to note that
- * if the structure contains any cycles then this will fail.
- * @param {object} context The context that handlebars should fill data from.
- * @returns {object|array} The structure input, as fully updated as possible.
- */
-function recurseMustache(inputs, context) {
-  // JSON stringify will fail if there are any cycles, stops infinite recursion
-  try {
-    JSON.stringify(inputs)
-  } catch (err) {
-    throw "Unable to process inputs to JSON, cannot recurse"
-  }
-  for (let key of Object.keys(inputs)) {
-    let val = inputs[key]
-    if (typeof val === "string") {
-      val = cleanMustache(inputs[key])
-      const template = handlebars.compile(val)
-      inputs[key] = template(context)
-    }
-    // this covers objects and arrays
-    else if (typeof val === "object") {
-      inputs[key] = recurseMustache(inputs[key], context)
-    }
-  }
-  return inputs
-}
-
-exports.recurseMustache = recurseMustache

packages/string-templates/.eslintrc

@@ -0,0 +1,12 @@
+{
+  "globals": {
+    "emit": true,
+    "key": true
+  },
+  "env": {
+    "node": true
+  },
+  "extends": ["eslint:recommended"],
+  "rules": {
+  }
+}

packages/string-templates/.gitignore

@@ -0,0 +1,2 @@
+dist/
+node_modules/

packages/string-templates/jest.config.js

@@ -0,0 +1,194 @@
+/*
+ * For a detailed explanation regarding each configuration property, visit:
+ * https://jestjs.io/docs/en/configuration.html
+ */
+
+module.exports = {
+  // All imported modules in your tests should be mocked automatically
+  // automock: false,
+
+  // Stop running tests after `n` failures
+  // bail: 0,
+
+  // The directory where Jest should store its cached dependency information
+  // cacheDirectory: "/tmp/jest_rs",
+
+  // Automatically clear mock calls and instances between every test
+  clearMocks: true,
+
+  // Indicates whether the coverage information should be collected while executing the test
+  // collectCoverage: false,
+
+  // An array of glob patterns indicating a set of files for which coverage information should be collected
+  // collectCoverageFrom: undefined,
+
+  // The directory where Jest should output its coverage files
+  coverageDirectory: "coverage",
+
+  // An array of regexp pattern strings used to skip coverage collection
+  // coveragePathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // Indicates which provider should be used to instrument code for coverage
+  coverageProvider: "v8",
+
+  // A list of reporter names that Jest uses when writing coverage reports
+  // coverageReporters: [
+  //   "json",
+  //   "text",
+  //   "lcov",
+  //   "clover"
+  // ],
+
+  // An object that configures minimum threshold enforcement for coverage results
+  // coverageThreshold: undefined,
+
+  // A path to a custom dependency extractor
+  // dependencyExtractor: undefined,
+
+  // Make calling deprecated APIs throw helpful error messages
+  // errorOnDeprecated: false,
+
+  // Force coverage collection from ignored files using an array of glob patterns
+  // forceCoverageMatch: [],
+
+  // A path to a module which exports an async function that is triggered once before all test suites
+  // globalSetup: undefined,
+
+  // A path to a module which exports an async function that is triggered once after all test suites
+  // globalTeardown: undefined,
+
+  // A set of global variables that need to be available in all test environments
+  // globals: {},
+
+  // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
+  // maxWorkers: "50%",
+
+  // An array of directory names to be searched recursively up from the requiring module's location
+  // moduleDirectories: [
+  //   "node_modules"
+  // ],
+
+  // An array of file extensions your modules use
+  // moduleFileExtensions: [
+  //   "js",
+  //   "json",
+  //   "jsx",
+  //   "ts",
+  //   "tsx",
+  //   "node"
+  // ],
+
+  // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
+  // moduleNameMapper: {},
+
+  // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
+  // modulePathIgnorePatterns: [],
+
+  // Activates notifications for test results
+  // notify: false,
+
+  // An enum that specifies notification mode. Requires { notify: true }
+  // notifyMode: "failure-change",
+
+  // A preset that is used as a base for Jest's configuration
+  // preset: undefined,
+
+  // Run tests from one or more projects
+  // projects: undefined,
+
+  // Use this configuration option to add custom reporters to Jest
+  // reporters: undefined,
+
+  // Automatically reset mock state between every test
+  // resetMocks: false,
+
+  // Reset the module registry before running each individual test
+  // resetModules: false,
+
+  // A path to a custom resolver
+  // resolver: undefined,
+
+  // Automatically restore mock state between every test
+  // restoreMocks: false,
+
+  // The root directory that Jest should scan for tests and modules within
+  // rootDir: undefined,
+
+  // A list of paths to directories that Jest should use to search for files in
+  // roots: [
+  //   "<rootDir>"
+  // ],
+
+  // Allows you to use a custom runner instead of Jest's default test runner
+  // runner: "jest-runner",
+
+  // The paths to modules that run some code to configure or set up the testing environment before each test
+  // setupFiles: [],
+
+  // A list of paths to modules that run some code to configure or set up the testing framework before each test
+  // setupFilesAfterEnv: [],
+
+  // The number of seconds after which a test is considered as slow and reported as such in the results.
+  // slowTestThreshold: 5,
+
+  // A list of paths to snapshot serializer modules Jest should use for snapshot testing
+  // snapshotSerializers: [],
+
+  // The test environment that will be used for testing
+  testEnvironment: "node",
+
+  // Options that will be passed to the testEnvironment
+  // testEnvironmentOptions: {},
+
+  // Adds a location field to test results
+  // testLocationInResults: false,
+
+  // The glob patterns Jest uses to detect test files
+  // testMatch: [
+  //   "**/__tests__/**/*.[jt]s?(x)",
+  //   "**/?(*.)+(spec|test).[tj]s?(x)"
+  // ],
+
+  // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
+  // testPathIgnorePatterns: [
+  //   "/node_modules/"
+  // ],
+
+  // The regexp pattern or array of patterns that Jest uses to detect test files
+  // testRegex: [],
+
+  // This option allows the use of a custom results processor
+  // testResultsProcessor: undefined,
+
+  // This option allows use of a custom test runner
+  // testRunner: "jasmine2",
+
+  // This option sets the URL for the jsdom environment. It is reflected in properties such as location.href
+  // testURL: "http://localhost",
+
+  // Setting this value to "fake" allows the use of fake timers for functions such as "setTimeout"
+  // timers: "real",
+
+  // A map from regular expressions to paths to transformers
+  // transform: undefined,
+
+  // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
+  // transformIgnorePatterns: [
+  //   "/node_modules/",
+  //   "\\.pnp\\.[^\\/]+$"
+  // ],
+
+  // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
+  // unmockedModulePathPatterns: undefined,
+
+  // Indicates whether each individual test should be reported during the run
+  // verbose: undefined,
+
+  // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
+  // watchPathIgnorePatterns: [],
+
+  // Whether to use watchman for file crawling
+  // watchman: true,
+};

packages/string-templates/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@budibase/string-templates",
+  "version": "0.5.3",
+  "description": "Handlebars wrapper for Budibase templating.",
+  "main": "dist/bundle.js",
+  "module": "dist/bundle.js",
+  "license": "AGPL-3.0",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "rollup -c",
+    "dev:builder": "tsc && rollup -cw",
+    "test": "jest"
+  },
+  "dependencies": {
+    "handlebars": "^4.7.6"
+  },
+  "devDependencies": {
+    "rollup": "^2.36.2",
+    "rollup-plugin-commonjs": "^10.1.0",
+    "rollup-plugin-node-builtins": "^2.1.2",
+    "rollup-plugin-node-globals": "^1.4.0",
+    "rollup-plugin-node-resolve": "^5.2.0",
+    "typescript": "^4.1.3",
+    "jest": "^26.6.3"
+  }
+}

packages/string-templates/rollup.config.js

@@ -0,0 +1,24 @@
+import commonjs from "rollup-plugin-commonjs"
+import nodeResolve from "rollup-plugin-node-resolve"
+import globals from "rollup-plugin-node-globals"
+import builtins from "rollup-plugin-node-builtins"
+
+export default {
+  input: "src/index.js",
+  output: {
+    file: "dist/bundle.js",
+    format: "umd",
+    name: "string-templates",
+    exports: "named",
+    globals: {
+      "fs": "fs",
+    },
+  },
+  external: ["fs"],
+  plugins: [
+    nodeResolve({ preferBuiltins: false }),
+    commonjs(),
+    globals(),
+    builtins(),
+  ],
+}

packages/string-templates/src/cleaning.js

@@ -0,0 +1,91 @@
+const { HelperFunctions } = require("./helpers/index")
+
+const HBS_CLEANING_REGEX = /{{[^}}]*}}/g
+const ALPHA_NUMERIC_REGEX = /^[A-Za-z0-9]+$/g
+
+function isAlphaNumeric(char) {
+  return char.match(ALPHA_NUMERIC_REGEX)
+}
+
+function swapStrings(string, start, length, swap) {
+  return string.slice(0, start) + swap + string.slice(start + length)
+}
+
+function handleCleaner(string, match, fn) {
+  const output = fn(match)
+  const idx = string.indexOf(match)
+  return swapStrings(string, idx, match.length, output)
+}
+
+function swapToDotNotation(statement) {
+  let startBraceIdx = statement.indexOf("[")
+  let lastIdx = 0
+  while (startBraceIdx !== -1) {
+    // if the character previous to the literal specifier is alpha-numeric this should happen
+    if (isAlphaNumeric(statement.charAt(startBraceIdx - 1))) {
+      statement = swapStrings(statement, startBraceIdx + lastIdx, 1, ".[")
+    }
+    lastIdx = startBraceIdx + 1
+    startBraceIdx = statement.substring(lastIdx + 1).indexOf("[")
+  }
+  return statement
+}
+
+function handleSpacesInProperties(statement) {
+  // exclude helpers and brackets, regex will only find double brackets
+  const exclusions = HelperFunctions.concat(["{{", "}}"])
+  // find all the parts split by spaces
+  const splitBySpaces = statement.split(" ")
+  // remove the excluded elements
+  const propertyParts = splitBySpaces.filter(part => exclusions.indexOf(part) === -1)
+  // rebuild to get the full property
+  const fullProperty = propertyParts.join(" ")
+  // now work out the dot notation layers and split them up
+  const propertyLayers = fullProperty.split(".")
+  // find the layers which need to be wrapped and wrap them
+  for (let layer of propertyLayers) {
+    if (layer.indexOf(" ") !== -1) {
+      statement = swapStrings(statement, statement.indexOf(layer), layer.length, `[${layer}]`)
+    }
+  }
+  // remove the edge case of double brackets being entered (in-case user already has specified)
+  return statement.replace(/\[\[/g, "[").replace(/]]/g, "]")
+}
+
+function finalise(statement) {
+  let insideStatement = statement.slice(2, statement.length - 2)
+  if (insideStatement.charAt(0) === " ") {
+    insideStatement = insideStatement.slice(1)
+  }
+  if (insideStatement.charAt(insideStatement.length - 1) === " ") {
+    insideStatement = insideStatement.slice(0, insideStatement.length - 1)
+  }
+  return `{{ all (${insideStatement}) }}`
+}
+
+/**
+ * When running handlebars statements to execute on the context of the automation it possible user's may input handlebars
+ * in a few different forms, some of which are invalid but are logically valid. An example of this would be the handlebars
+ * statement "{{steps[0].revision}}" here it is obvious the user is attempting to access an array or object using array
+ * like operators. These are not supported by handlebars and therefore the statement will fail. This function will clean up
+ * the handlebars statement so it instead reads as "{{steps.0.revision}}" which is valid and will work. It may also be expanded
+ * to include any other handlebars statement cleanup that has been deemed necessary for the system.
+ *
+ * @param {string} string The string which *may* contain handlebars statements, it is OK if it does not contain any.
+ * @returns {string} The string that was input with cleaned up handlebars statements as required.
+ */
+module.exports.cleanHandlebars = (string) => {
+  let cleaners = [swapToDotNotation, handleSpacesInProperties, finalise]
+  for (let cleaner of cleaners) {
+    // re-run search each time incase previous cleaner update/removed a match
+    let regex = new RegExp(HBS_CLEANING_REGEX)
+    let matches = string.match(regex)
+    if (matches == null) {
+      continue
+    }
+    for (let match of matches) {
+      string = handleCleaner(string, match, cleaner)
+    }
+  }
+  return string
+}

packages/string-templates/src/helpers/Helper.js

@@ -0,0 +1,19 @@
+class Helper {
+  constructor(name, fn) {
+    this.name = name
+    this.fn = fn
+  }
+
+  register(handlebars) {
+    // wrap the function so that no helper can cause handlebars to break
+    handlebars.registerHelper(this.name, value => {
+      return this.fn(value) || value
+    })
+  }
+
+  unregister(handlebars) {
+    handlebars.unregisterHelper(this.name)
+  }
+}
+
+module.exports = Helper

packages/string-templates/src/helpers/index.js

@@ -0,0 +1,52 @@
+const Helper = require("./Helper")
+const { SafeString } = require("handlebars")
+
+const HTML_SWAPS = {
+  "<": "&lt;",
+  ">": "&gt;",
+}
+
+const HelperFunctionBuiltin = [
+  "#if",
+  "#unless",
+  "#each",
+  "#with",
+  "lookup",
+  "log"
+]
+
+const HelperFunctionNames = {
+  OBJECT: "object",
+  ALL: "all",
+}
+
+const HELPERS = [
+  // external helpers
+  new Helper(HelperFunctionNames.OBJECT, value => {
+    return new SafeString(JSON.stringify(value))
+  }),
+  // this help is applied to all statements
+  new Helper(HelperFunctionNames.ALL, value => {
+    let text = new SafeString(unescape(value).replace(/&amp;/g, '&'))
+    if (text == null || typeof text !== "string") {
+      return text
+    }
+    return text.replace(/[<>]/g, tag => {
+      return HTML_SWAPS[tag] || tag
+    })
+  })
+]
+
+module.exports.HelperFunctions = Object.values(HelperFunctionNames).concat(HelperFunctionBuiltin)
+
+module.exports.registerAll = handlebars => {
+  for (let helper of HELPERS) {
+    helper.register(handlebars)
+  }
+}
+
+module.exports.unregisterAll = handlebars => {
+  for (let helper of HELPERS) {
+    helper.unregister(handlebars)
+  }
+}

packages/string-templates/src/index.js

@@ -0,0 +1,102 @@
+const handlebars = require("handlebars")
+const { registerAll } = require("./helpers/index")
+const { cleanHandlebars } = require("./cleaning")
+
+const hbsInstance = handlebars.create()
+registerAll(hbsInstance)
+
+/**
+ * utility function to check if the object is valid
+ */
+function testObject(object) {
+  // JSON stringify will fail if there are any cycles, stops infinite recursion
+  try {
+    JSON.stringify(object)
+  } catch (err) {
+    throw "Unable to process inputs to JSON, cannot recurse"
+  }
+}
+
+/**
+ * Given an input object this will recurse through all props to try and update any handlebars statements within.
+ * @param {object|array} object The input structure which is to be recursed, it is important to note that
+ * if the structure contains any cycles then this will fail.
+ * @param {object} context The context that handlebars should fill data from.
+ * @returns {Promise<object|array>} The structure input, as fully updated as possible.
+ */
+module.exports.processObject = async (object, context) => {
+  testObject(object)
+  for (let key of Object.keys(object)) {
+    let val = object[key]
+    if (typeof val === "string") {
+      object[key] = await module.exports.processString(object[key], context)
+    } else if (typeof val === "object") {
+      object[key] = await module.exports.processObject(object[key], context)
+    }
+  }
+  return object
+}
+
+/**
+ * This will process a single handlebars containing string. If the string passed in has no valid handlebars statements
+ * then nothing will occur.
+ * @param {string} string The template string which is the filled from the context object.
+ * @param {object} context An object of information which will be used to enrich the string.
+ * @returns {Promise<string>} The enriched string, all templates should have been replaced if they can be.
+ */
+module.exports.processString = async (string, context) => {
+  // TODO: carry out any async calls before carrying out async call
+  return module.exports.processStringSync(string, context)
+}
+
+/**
+ * Given an input object this will recurse through all props to try and update any handlebars statements within. This is
+ * a pure sync call and therefore does not have the full functionality of the async call.
+ * @param {object|array} object The input structure which is to be recursed, it is important to note that
+ * if the structure contains any cycles then this will fail.
+ * @param {object} context The context that handlebars should fill data from.
+ * @returns {object|array} The structure input, as fully updated as possible.
+ */
+module.exports.processObjectSync = (object, context) => {
+  testObject(object)
+  for (let key of Object.keys(object)) {
+    let val = object[key]
+    if (typeof val === "string") {
+      object[key] = module.exports.processStringSync(object[key], context)
+    } else if (typeof val === "object") {
+      object[key] = module.exports.processObjectSync(object[key], context)
+    }
+  }
+  return object
+}
+
+/**
+ * This will process a single handlebars containing string. If the string passed in has no valid handlebars statements
+ * then nothing will occur. This is a pure sync call and therefore does not have the full functionality of the async call.
+ * @param {string} string The template string which is the filled from the context object.
+ * @param {object} context An object of information which will be used to enrich the string.
+ * @returns {string} The enriched string, all templates should have been replaced if they can be.
+ */
+module.exports.processStringSync = (string, context) => {
+  if (typeof string !== "string") {
+    throw "Cannot process non-string types."
+  }
+  console.log(string)
+  console.log(context)
+  let template
+  string = cleanHandlebars(string)
+  console.log(string)
+  // this does not throw an error when template can't be fulfilled, have to try correct beforehand
+  template = hbsInstance.compile(string)
+  return template(context)
+}
+
+/**
+ * Errors can occur if a user of this library attempts to use a helper that has not been added to the system, these errors
+ * can be captured to alert the user of the mistake.
+ * @param {function} handler a function which will be called every time an error occurs when processing a handlebars
+ * statement.
+ */
+module.exports.errorEvents = handler => {
+  hbsInstance.registerHelper("helperMissing", handler)
+}

packages/string-templates/test/basic.spec.js

@@ -0,0 +1,69 @@
+const {
+  processObject,
+  processString,
+} = require("../src/index")
+
+describe("Test that the string processing works correctly", () => {
+  it("should process a basic template string", async () => {
+    const output = await processString("templating is {{ adjective }}", {
+      adjective: "easy"
+    })
+    expect(output).toBe("templating is easy")
+  })
+
+  it("should fail gracefully when wrong type passed in", async () => {
+    let error = null
+    try {
+      await processString(null, null)
+    } catch (err) {
+      error = err
+    }
+    expect(error).not.toBeNull()
+  })
+})
+
+describe("Test that the object processing works correctly", () => {
+  it("should be able to process an object with some template strings", async () => {
+    const output = await processObject({
+      first: "thing is {{ adjective }}",
+      second: "thing is bad",
+      third: "we are {{ adjective }} {{ noun }}",
+    }, {
+      adjective: "easy",
+      noun: "people",
+    })
+    expect(output.first).toBe("thing is easy")
+    expect(output.second).toBe("thing is bad")
+    expect(output.third).toBe("we are easy people")
+  })
+
+  it("should be able to handle arrays of string templates", async () => {
+    const output = await processObject(["first {{ noun }}", "second {{ noun }}"], {
+      noun: "person"
+    })
+    expect(output[0]).toBe("first person")
+    expect(output[1]).toBe("second person")
+  })
+
+  it("should fail gracefully when object passed in has cycles", async () => {
+    let error = null
+    try {
+      const innerObj = { a: "thing {{ a }}" }
+      innerObj.b = innerObj
+      await processObject(innerObj, { a: 1 })
+    } catch (err) {
+      error = err
+    }
+    expect(error).not.toBeNull()
+  })
+
+  it("should fail gracefully when wrong type is passed in", async () => {
+    let error = null
+    try {
+      await processObject(null, null)
+    } catch (err) {
+      error = err
+    }
+    expect(error).not.toBeNull()
+  })
+})

packages/string-templates/test/escapes.spec.js

@@ -0,0 +1,35 @@
+const {
+  processString,
+} = require("../src/index")
+
+describe("Handling context properties with spaces in their name", () => {
+  it("should allow through literal specifiers", async () => {
+    const output = await processString("test {{ [test thing] }}", {
+      "test thing": 1
+    })
+    expect(output).toBe("test 1")
+  })
+
+  it("should convert to dot notation where required", async () => {
+    const output = await processString("test {{ test[0] }}", {
+      test: [2]
+    })
+    expect(output).toBe("test 2")
+  })
+
+  it("should be able to handle a property with a space in its name", async () => {
+    const output = await processString("hello my name is {{ person name }}", {
+      "person name": "Mike",
+    })
+    expect(output).toBe("hello my name is Mike")
+  })
+
+  it("should be able to handle an object with layers that requires escaping", async () => {
+    const output = await processString("testcase {{ testing.test case }}", {
+      testing: {
+        "test case": 1
+      }
+    })
+    expect(output).toBe("testcase 1")
+  })
+})

packages/string-templates/test/helpers.spec.js

@@ -0,0 +1,12 @@
+const {
+  processString,
+} = require("../src/index")
+
+describe("test the custom helpers we have applied", () => {
+  it("should be able to use the object helper", async () => {
+    const output = await processString("object is {{ object obj }}", {
+      obj: { a: 1 },
+    })
+    expect(output).toBe("object is {\"a\":1}")
+  })
+})

packages/string-templates/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "include": ["src/**/*"],
+
+  "compilerOptions": {
+    "allowJs": true,
+    "declaration": true,
+    "emitDeclarationOnly": true,
+    "outDir": "dist"
+  }
+}

packages/worker/.eslintrc

@@ -0,0 +1,12 @@
+{
+  "globals": {
+    "emit": true,
+    "key": true
+  },
+  "env": {
+    "node": true
+  },
+  "extends": ["eslint:recommended"],
+  "rules": {
+  }
+}

scripts/symlinkDev.js

@@ -35,6 +35,10 @@ const SYMLINK_PATHS = [
     symlink: `${devDir}/budibase-client.js.map`,
     destination: resolve("packages/client/dist/budibase-client.js.map"),
   },
+  {
+    symlink: `${devDir}/@budibase/string-templates`,
+    destination: resolve("packages/string-templates"),
+  },
 ]
 
 SYMLINK_PATHS.forEach(sym => {