Merge pull request #3002 from Budibase/js-binding-drawer

JavaScript bindings

packages/bbui/src/Tabs/Tab.svelte

@@ -8,11 +8,19 @@
   const selected = getContext("tab")
   let tab
   let tabInfo
+
   const setTabInfo = () => {
+    // If the tabs are being rendered inside a component which uses
+    // a svelte transition to enter, then this initial getBoundingClientRect
+    // will return an incorrect position.
+    // We just need to get this off the main thread to fix this, by using
+    // a 0ms timeout.
+    setTimeout(() => {
       tabInfo = tab.getBoundingClientRect()
       if ($selected.title === title) {
         $selected.info = tabInfo
       }
+    }, 0)
   }
 
   onMount(() => {

packages/builder/src/builderStore/dataBinding.js

@@ -7,11 +7,17 @@ import {
 } from "./storeUtils"
 import { store } from "builderStore"
 import { queries as queriesStores, tables as tablesStore } from "stores/backend"
-import { makePropSafe } from "@budibase/string-templates"
+import {
+  makePropSafe,
+  isJSBinding,
+  decodeJSBinding,
+  encodeJSBinding,
+} from "@budibase/string-templates"
 import { TableNames } from "../constants"
 
 // Regex to match all instances of template strings
 const CAPTURE_VAR_INSIDE_TEMPLATE = /{{([^}]+)}}/g
+const CAPTURE_VAR_INSIDE_JS = /\$\("([^")]+)"\)/g
 const CAPTURE_HBS_TEMPLATE = /{{[\S\s]*?}}/g
 
 /**
@@ -430,6 +436,15 @@ function replaceBetween(string, start, end, replacement) {
  * utility function for the readableToRuntimeBinding and runtimeToReadableBinding.
  */
 function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
+  // Decide from base64 if using JS
+  const isJS = isJSBinding(textWithBindings)
+  if (isJS) {
+    textWithBindings = decodeJSBinding(textWithBindings)
+  }
+
+  // Determine correct regex to find bindings to replace
+  const regex = isJS ? CAPTURE_VAR_INSIDE_JS : CAPTURE_VAR_INSIDE_TEMPLATE
+
   const convertFrom =
     convertTo === "runtimeBinding" ? "readableBinding" : "runtimeBinding"
   if (typeof textWithBindings !== "string") {
@@ -441,7 +456,7 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
     .sort((a, b) => {
       return b.length - a.length
     })
-  const boundValues = textWithBindings.match(CAPTURE_VAR_INSIDE_TEMPLATE) || []
+  const boundValues = textWithBindings.match(regex) || []
   let result = textWithBindings
   for (let boundValue of boundValues) {
     let newBoundValue = boundValue
@@ -449,7 +464,7 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
     // in the search, working from longest to shortest so always use best match first
     let searchString = newBoundValue
     for (let from of convertFromProps) {
-      if (shouldReplaceBinding(newBoundValue, from, convertTo)) {
+      if (isJS || shouldReplaceBinding(newBoundValue, from, convertTo)) {
         const binding = bindableProperties.find(el => el[convertFrom] === from)
         let idx
         do {
@@ -472,6 +487,12 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
     }
     result = result.replace(boundValue, newBoundValue)
   }
+
+  // Re-encode to base64 if using JS
+  if (isJS) {
+    result = encodeJSBinding(result)
+  }
+
   return result
 }
 

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -194,6 +194,7 @@
             value={inputData[key]}
             on:change={e => onChange(e, key)}
             {bindings}
+            allowJS={false}
           />
         {/if}
       {:else if value.customType === "query"}
@@ -259,6 +260,7 @@
               value={inputData[key]}
               on:change={e => onChange(e, key)}
               {bindings}
+              allowJS={false}
             />
           </div>
         {/if}

packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte

@@ -39,6 +39,7 @@
         type="string"
         {bindings}
         fillWidth={true}
+        allowJS={false}
       />
     {/each}
   </div>

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -110,6 +110,7 @@
               type="string"
               {bindings}
               fillWidth={true}
+              allowJS={false}
             />
           {/if}
         {/if}

packages/builder/src/components/common/CodeMirrorEditor.svelte

@@ -0,0 +1,159 @@
+<script context="module">
+  import { Label } from "@budibase/bbui"
+
+  export const EditorModes = {
+    JS: {
+      name: "javascript",
+      json: false,
+    },
+    JSON: {
+      name: "javascript",
+      json: true,
+    },
+    SQL: {
+      name: "sql",
+    },
+    Handlebars: {
+      name: "handlebars",
+      base: "text/html",
+    },
+  }
+</script>
+
+<script>
+  import CodeMirror from "components/integration/codemirror"
+  import { themeStore } from "builderStore"
+  import { createEventDispatcher, onMount } from "svelte"
+
+  export let mode = EditorModes.JS
+  export let value = ""
+  export let height = 300
+  export let resize = "none"
+  export let readonly = false
+  export let hints = []
+  export let label
+
+  const dispatch = createEventDispatcher()
+  let textarea
+  let editor
+
+  // Keep editor up to date with value
+  $: editor?.setValue(value || "")
+
+  // Creates an instance of a code mirror editor
+  async function createEditor(mode, value) {
+    if (!CodeMirror || !textarea || editor) {
+      return
+    }
+
+    // Configure CM options
+    const lightTheme = $themeStore.theme.includes("light")
+    const options = {
+      mode,
+      value: value || "",
+      readOnly: readonly,
+      theme: lightTheme ? "default" : "tomorrow-night-eighties",
+
+      // Style
+      lineNumbers: true,
+      lineWrapping: true,
+      indentWithTabs: true,
+      indentUnit: 2,
+      tabSize: 2,
+
+      // QOL addons
+      extraKeys: { "Ctrl-Space": "autocomplete" },
+      styleActiveLine: { nonEmpty: true },
+      autoCloseBrackets: true,
+      matchBrackets: true,
+    }
+
+    // Register hints plugin if desired
+    if (hints?.length) {
+      CodeMirror.registerHelper("hint", "dictionaryHint", function (editor) {
+        const cursor = editor.getCursor()
+        return {
+          list: hints,
+          from: CodeMirror.Pos(cursor.line, cursor.ch),
+          to: CodeMirror.Pos(cursor.line, cursor.ch),
+        }
+      })
+      CodeMirror.commands.autocomplete = function (cm) {
+        CodeMirror.showHint(cm, CodeMirror.hint.dictionaryHint)
+      }
+    }
+
+    // Construct CM instance
+    editor = CodeMirror.fromTextArea(textarea, options)
+
+    // Use a blur handler to update the value
+    editor.on("blur", instance => {
+      dispatch("change", instance.getValue())
+    })
+  }
+
+  // Export a function to expose caret position
+  export const getCaretPosition = () => {
+    const cursor = editor.getCursor()
+    return {
+      start: cursor.ch,
+      end: cursor.ch,
+    }
+  }
+
+  onMount(() => {
+    // Create the editor with initial value
+    createEditor(mode, value)
+
+    // Clean up editor on unmount
+    return () => {
+      if (editor) {
+        editor.toTextArea()
+      }
+    }
+  })
+</script>
+
+{#if label}
+  <div style="margin-bottom: var(--spacing-s)">
+    <Label small>{label}</Label>
+  </div>
+{/if}
+<div
+  style={`--code-mirror-height: ${height}px; --code-mirror-resize: ${resize}`}
+>
+  <textarea tabindex="0" bind:this={textarea} readonly {value} />
+</div>
+
+<style>
+  div :global(.CodeMirror) {
+    height: var(--code-mirror-height);
+    min-height: var(--code-mirror-height);
+    font-family: monospace;
+    line-height: 1.3;
+    border: var(--spectrum-alias-border-size-thin) solid;
+    border-color: var(--spectrum-alias-border-color);
+    border-radius: var(--border-radius-s);
+    resize: var(--code-mirror-resize);
+    overflow: hidden;
+  }
+
+  /* Override default active line highlight colour in dark theme */
+  div
+    :global(.CodeMirror-focused.cm-s-tomorrow-night-eighties
+      .CodeMirror-activeline-background) {
+    background: rgba(255, 255, 255, 0.075);
+  }
+
+  /* Remove active line styling when not focused */
+  div
+    :global(.CodeMirror:not(.CodeMirror-focused)
+      .CodeMirror-activeline-background) {
+    background: unset;
+  }
+
+  /* Add a spectrum themed border when focused */
+  div :global(.CodeMirror-focused) {
+    border-color: var(--spectrum-alias-border-color-mouse-focus);
+  }
+</style>

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -1,32 +1,98 @@
 <script>
   import groupBy from "lodash/fp/groupBy"
-  import { Search, TextArea, DrawerContent } from "@budibase/bbui"
+  import {
-  import { createEventDispatcher } from "svelte"
+    Search,
-  import { isValid } from "@budibase/string-templates"
+    TextArea,
+    DrawerContent,
+    Tabs,
+    Tab,
+    Body,
+    Layout,
+  } from "@budibase/bbui"
+  import { createEventDispatcher, onMount } from "svelte"
+  import {
+    isValid,
+    decodeJSBinding,
+    encodeJSBinding,
+  } from "@budibase/string-templates"
   import { readableToRuntimeBinding } from "builderStore/dataBinding"
   import { handlebarsCompletions } from "constants/completions"
-  import { addToText } from "./utils"
+  import { addHBSBinding, addJSBinding } from "./utils"
+  import CodeMirrorEditor from "components/common/CodeMirrorEditor.svelte"
 
   const dispatch = createEventDispatcher()
 
   export let bindableProperties
   export let value = ""
   export let valid
+  export let allowJS = false
 
   let helpers = handlebarsCompletions()
   let getCaretPosition
   let search = ""
+  let initialValueJS = value?.startsWith("{{ js ")
+  let mode = initialValueJS ? "JavaScript" : "Handlebars"
+  let jsValue = initialValueJS ? value : null
+  let hbsValue = initialValueJS ? null : value
 
-  $: valid = isValid(readableToRuntimeBinding(bindableProperties, value))
+  $: usingJS = mode === "JavaScript"
-  $: dispatch("change", value)
   $: ({ context } = groupBy("type", bindableProperties))
   $: searchRgx = new RegExp(search, "ig")
-  $: filteredColumns = context?.filter(context => {
+  $: filteredBindings = context?.filter(context => {
     return context.readableBinding.match(searchRgx)
   })
   $: filteredHelpers = helpers?.filter(helper => {
     return helper.label.match(searchRgx) || helper.description.match(searchRgx)
   })
+
+  const updateValue = value => {
+    valid = isValid(readableToRuntimeBinding(bindableProperties, value))
+    if (valid) {
+      dispatch("change", value)
+    }
+  }
+
+  // Adds a HBS helper to the expression
+  const addHelper = helper => {
+    hbsValue = addHBSBinding(value, getCaretPosition(), helper.text)
+    updateValue(hbsValue)
+  }
+
+  // Adds a data binding to the expression
+  const addBinding = binding => {
+    if (usingJS) {
+      let js = decodeJSBinding(jsValue)
+      js = addJSBinding(js, getCaretPosition(), binding.readableBinding)
+      jsValue = encodeJSBinding(js)
+      updateValue(jsValue)
+    } else {
+      hbsValue = addHBSBinding(
+        hbsValue,
+        getCaretPosition(),
+        binding.readableBinding
+      )
+      updateValue(hbsValue)
+    }
+  }
+
+  const onChangeMode = e => {
+    mode = e.detail
+    updateValue(mode === "JavaScript" ? jsValue : hbsValue)
+  }
+
+  const onChangeHBSValue = e => {
+    hbsValue = e.detail
+    updateValue(hbsValue)
+  }
+
+  const onChangeJSValue = e => {
+    jsValue = encodeJSBinding(e.detail)
+    updateValue(jsValue)
+  }
+
+  onMount(() => {
+    valid = isValid(readableToRuntimeBinding(bindableProperties, value))
+  })
 </script>
 
 <DrawerContent>
@@ -36,32 +102,24 @@
         <div class="heading">Search</div>
         <Search placeholder="Search" bind:value={search} />
       </section>
-      {#if filteredColumns?.length}
+      {#if filteredBindings?.length}
         <section>
           <div class="heading">Bindable Values</div>
           <ul>
-            {#each filteredColumns as { readableBinding }}
+            {#each filteredBindings as binding}
-              <li
+              <li on:click={() => addBinding(binding)}>
-                on:click={() => {
+                {binding.readableBinding}
-                  value = addToText(value, getCaretPosition(), readableBinding)
-                }}
-              >
-                {readableBinding}
               </li>
             {/each}
           </ul>
         </section>
       {/if}
-      {#if filteredHelpers?.length}
+      {#if filteredHelpers?.length && !usingJS}
         <section>
           <div class="heading">Helpers</div>
           <ul>
             {#each filteredHelpers as helper}
-              <li
+              <li on:click={() => addHelper(helper)}>
-                on:click={() => {
-                  value = addToText(value, getCaretPosition(), helper.text)
-                }}
-              >
                 <div class="helper">
                   <div class="helper__name">{helper.displayText}</div>
                   <div class="helper__description">
@@ -77,9 +135,13 @@
     </div>
   </svelte:fragment>
   <div class="main">
+    <Tabs selected={mode} on:select={onChangeMode}>
+      <Tab title="Handlebars">
+        <div class="main-content">
           <TextArea
             bind:getCaretPosition
-      bind:value
+            value={hbsValue}
+            on:change={onChangeHBSValue}
             placeholder="Add text, or click the objects on the left to add them to the textbox."
           />
           {#if !valid}
@@ -90,11 +152,39 @@
             </p>
           {/if}
         </div>
+      </Tab>
+      {#if allowJS}
+        <Tab title="JavaScript">
+          <div class="main-content">
+            <Layout noPadding gap="XS">
+              <CodeMirrorEditor
+                bind:getCaretPosition
+                height={200}
+                value={decodeJSBinding(jsValue)}
+                on:change={onChangeJSValue}
+                hints={context?.map(x => `$("${x.readableBinding}")`)}
+              />
+              <Body size="S">
+                JavaScript expressions are executed as functions, so ensure that
+                your expression returns a value.
+              </Body>
+            </Layout>
+          </div>
+        </Tab>
+      {/if}
+    </Tabs>
+  </div>
 </DrawerContent>
 
 <style>
   .main :global(textarea) {
-    min-height: 150px !important;
+    min-height: 202px !important;
+  }
+  .main {
+    margin: calc(-1 * var(--spacing-xl));
+  }
+  .main-content {
+    padding: var(--spacing-s) var(--spacing-xl);
   }
 
   .container {

packages/builder/src/components/common/bindings/DrawerBindableCombobox.svelte

@@ -6,6 +6,7 @@
   } from "builderStore/dataBinding"
   import BindingPanel from "components/common/bindings/BindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
+  import { isJSBinding } from "@budibase/string-templates"
 
   export let panel = BindingPanel
   export let value = ""
@@ -15,11 +16,14 @@
   export let label
   export let disabled = false
   export let options
+  export let allowJS = true
 
   const dispatch = createEventDispatcher()
   let bindingDrawer
+
   $: readableValue = runtimeToReadableBinding(bindings, value)
   $: tempValue = readableValue
+  $: isJS = isJSBinding(value)
 
   const handleClose = () => {
     onChange(tempValue)
@@ -35,7 +39,7 @@
   <Combobox
     {label}
     {disabled}
-    value={readableValue}
+    value={isJS ? "(JavaScript function)" : readableValue}
     on:change={event => onChange(event.detail)}
     {placeholder}
     {options}
@@ -58,6 +62,7 @@
     close={handleClose}
     on:change={event => (tempValue = event.detail)}
     bindableProperties={bindings}
+    {allowJS}
   />
 </Drawer>
 

packages/builder/src/components/common/bindings/DrawerBindableInput.svelte

@@ -6,6 +6,7 @@
   } from "builderStore/dataBinding"
   import BindingPanel from "components/common/bindings/BindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
+  import { isJSBinding } from "@budibase/string-templates"
 
   export let panel = BindingPanel
   export let value = ""
@@ -15,12 +16,15 @@
   export let label
   export let disabled = false
   export let fillWidth
+  export let allowJS = true
 
   const dispatch = createEventDispatcher()
   let bindingDrawer
   let valid = true
+
   $: readableValue = runtimeToReadableBinding(bindings, value)
   $: tempValue = readableValue
+  $: isJS = isJSBinding(value)
 
   const saveBinding = () => {
     onChange(tempValue)
@@ -36,7 +40,7 @@
   <Input
     {label}
     {disabled}
-    value={readableValue}
+    value={isJS ? "(JavaScript function)" : readableValue}
     on:change={event => onChange(event.detail)}
     {placeholder}
   />
@@ -60,6 +64,7 @@
     value={readableValue}
     on:change={event => (tempValue = event.detail)}
     bindableProperties={bindings}
+    {allowJS}
   />
 </Drawer>
 

packages/builder/src/components/common/bindings/ServerBindingPanel.svelte

@@ -5,7 +5,7 @@
   import { isValid } from "@budibase/string-templates"
   import { handlebarsCompletions } from "constants/completions"
   import { readableToRuntimeBinding } from "builderStore/dataBinding"
-  import { addToText } from "./utils"
+  import { addHBSBinding } from "./utils"
 
   const dispatch = createEventDispatcher()
 
@@ -47,7 +47,7 @@
               {#each bindings as binding}
                 <li
                   on:click={() => {
-                    value = addToText(value, getCaretPosition(), binding)
+                    value = addHBSBinding(value, getCaretPosition(), binding)
                   }}
                 >
                   <span class="binding__label">{binding.label}</span>
@@ -71,7 +71,7 @@
             {#each filteredHelpers as helper}
               <li
                 on:click={() => {
-                  value = addToText(value, getCaretPosition(), helper.text)
+                  value = addHBSBinding(value, getCaretPosition(), helper.text)
                 }}
               >
                 <div class="helper">

packages/builder/src/components/common/bindings/utils.js

@@ -1,4 +1,4 @@
-export function addToText(value, caretPos, binding) {
+export function addHBSBinding(value, caretPos, binding) {
   binding = typeof binding === "string" ? binding : binding.path
   value = value == null ? "" : value
   if (!value.includes("{{") && !value.includes("}}")) {
@@ -14,3 +14,18 @@ export function addToText(value, caretPos, binding) {
   }
   return value
 }
+
+export function addJSBinding(value, caretPos, binding) {
+  binding = typeof binding === "string" ? binding : binding.path
+  value = value == null ? "" : value
+  binding = `$("${binding}")`
+  if (caretPos.start) {
+    value =
+      value.substring(0, caretPos.start) +
+      binding +
+      value.substring(caretPos.end, value.length)
+  } else {
+    value += binding
+  }
+  return value
+}

packages/builder/src/components/design/PropertiesPanel/PropertyControls/Input.svelte

@@ -0,0 +1,15 @@
+<script>
+  import { Input } from "@budibase/bbui"
+  import { isJSBinding } from "@budibase/string-templates"
+
+  export let value
+
+  $: isJS = isJSBinding(value)
+</script>
+
+<Input
+  {...$$props}
+  value={isJS ? "(JavaScript function)" : value}
+  readonly={isJS}
+  on:change
+/>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/PropertyControl.svelte

@@ -105,6 +105,7 @@
           value={safeValue}
           on:change={e => (tempValue = e.detail)}
           bindableProperties={bindings}
+          allowJS
         />
       </Drawer>
     {/if}

packages/builder/src/components/design/PropertiesPanel/PropertyControls/componentSettings.js

@@ -1,4 +1,4 @@
-import { Checkbox, Input, Select, Stepper } from "@budibase/bbui"
+import { Checkbox, Select, Stepper } from "@budibase/bbui"
 import DataSourceSelect from "./DataSourceSelect.svelte"
 import DataProviderSelect from "./DataProviderSelect.svelte"
 import EventsEditor from "./EventsEditor"
@@ -15,6 +15,7 @@ import URLSelect from "./URLSelect.svelte"
 import OptionsEditor from "./OptionsEditor/OptionsEditor.svelte"
 import FormFieldSelect from "./FormFieldSelect.svelte"
 import ValidationEditor from "./ValidationEditor/ValidationEditor.svelte"
+import Input from "./Input.svelte"
 
 const componentMap = {
   text: Input,

packages/builder/src/components/integration/QueryViewer.svelte

@@ -21,12 +21,15 @@
   import ParameterBuilder from "components/integration/QueryParameterBuilder.svelte"
   import { datasources, integrations, queries } from "stores/backend"
   import { capitalise } from "../../helpers"
+  import CodeMirrorEditor from "components/common/CodeMirrorEditor.svelte"
 
   export let query
   export let fields = []
 
   let parameters
   let data = []
+  const transformerDocs =
+    "https://docs.budibase.com/building-apps/data/transformers"
   const typeOptions = [
     { label: "Text", value: "STRING" },
     { label: "Number", value: "NUMBER" },
@@ -52,6 +55,11 @@
   $: readQuery = query.queryVerb === "read" || query.readable
   $: queryInvalid = !query.name || (readQuery && data.length === 0)
 
+  // seed the transformer
+  if (query && !query.transformer) {
+    query.transformer = "return data"
+  }
+
   function newField() {
     fields = [...fields, {}]
   }
@@ -74,6 +82,7 @@
       const response = await api.post(`/api/queries/preview`, {
         fields: query.fields,
         queryVerb: query.queryVerb,
+        transformer: query.transformer,
         parameters: query.parameters.reduce(
           (acc, next) => ({
             ...acc,
@@ -160,12 +169,34 @@
       <IntegrationQueryEditor
         {datasource}
         {query}
-        height={300}
+        height={200}
         schema={queryConfig[query.queryVerb]}
         bind:parameters
       />
       <Divider />
     </div>
+    <div class="config">
+      <div class="help-heading">
+        <Heading size="S">Transformer</Heading>
+        <Icon
+          on:click={() => window.open(transformerDocs)}
+          hoverable
+          name="Help"
+          size="L"
+        />
+      </div>
+      <Body size="S"
+        >Add a JavaScript function to transform the query result.</Body
+      >
+      <CodeMirrorEditor
+        height={200}
+        label="Transformer"
+        value={query.transformer}
+        resize="vertical"
+        on:change={e => (query.transformer = e.detail)}
+      />
+      <Divider />
+    </div>
     <div class="viewer-controls">
       <Heading size="S">Results</Heading>
       <ButtonGroup>
@@ -220,6 +251,7 @@
     display: grid;
     grid-gap: var(--spacing-s);
   }
+
   .config-field {
     display: grid;
     grid-template-columns: 20% 1fr;
@@ -227,6 +259,11 @@
     align-items: center;
   }
 
+  .help-heading {
+    display: flex;
+    justify-content: space-between;
+  }
+
   .field {
     display: grid;
     grid-template-columns: 1fr 1fr 5%;

packages/builder/src/components/integration/codemirror.js

@@ -1,12 +1,22 @@
 import CodeMirror from "codemirror"
 import "codemirror/lib/codemirror.css"
-import "codemirror/theme/tomorrow-night-eighties.css"
+
-import "codemirror/addon/hint/show-hint.css"
+// Modes
-import "codemirror/theme/neo.css"
+import "codemirror/mode/javascript/javascript"
 import "codemirror/mode/sql/sql"
 import "codemirror/mode/css/css"
 import "codemirror/mode/handlebars/handlebars"
-import "codemirror/mode/javascript/javascript"
+
+// Hints
 import "codemirror/addon/hint/show-hint"
+import "codemirror/addon/hint/show-hint.css"
+
+// Theming
+import "codemirror/theme/tomorrow-night-eighties.css"
+
+// Functional addons
+import "codemirror/addon/selection/active-line"
+import "codemirror/addon/edit/closebrackets"
+import "codemirror/addon/edit/matchbrackets"
 
 export default CodeMirror

packages/builder/src/helpers/fetchTableData.js

@@ -48,7 +48,6 @@ export const fetchTableData = opts => {
   const fetchPage = async bookmark => {
     lastBookmark = bookmark
     const { tableId, limit, sortColumn, sortOrder, paginate } = options
-    store.update($store => ({ ...$store, loading: true }))
     const res = await API.post(`/api/${options.tableId}/search`, {
       tableId,
       query,
@@ -59,7 +58,6 @@ export const fetchTableData = opts => {
       paginate,
       bookmark,
     })
-    store.update($store => ({ ...$store, loading: false, loaded: true }))
     return await res.json()
   }
 
@@ -103,7 +101,7 @@ export const fetchTableData = opts => {
     if (!schema) {
       return
     }
-    store.update($store => ({ ...$store, schema }))
+    store.update($store => ({ ...$store, schema, loading: true }))
 
     // Work out what sort type to use
     if (!sortColumn || !schema[sortColumn]) {
@@ -135,6 +133,7 @@ export const fetchTableData = opts => {
     }
 
     // Fetch next page
+    store.update($store => ({ ...$store, loading: true }))
     const page = await fetchPage(state.bookmarks[state.pageNumber + 1])
 
     // Update state
@@ -148,6 +147,7 @@ export const fetchTableData = opts => {
         pageNumber: pageNumber + 1,
         rows: page.rows,
         bookmarks,
+        loading: false,
       }
     })
   }
@@ -160,6 +160,7 @@ export const fetchTableData = opts => {
     }
 
     // Fetch previous page
+    store.update($store => ({ ...$store, loading: true }))
     const page = await fetchPage(state.bookmarks[state.pageNumber - 1])
 
     // Update state
@@ -168,6 +169,7 @@ export const fetchTableData = opts => {
         ...$store,
         pageNumber: $store.pageNumber - 1,
         rows: page.rows,
+        loading: false,
       }
     })
   }

packages/client/src/components/CustomThemeWrapper.svelte

@@ -16,7 +16,10 @@
     /*  Buttons */
     --spectrum-semantic-cta-color-background-default: var(--primaryColor);
     --spectrum-semantic-cta-color-background-hover: var(--primaryColorHover);
+    --spectrum-button-primary-s-border-radius: var(--buttonBorderRadius);
     --spectrum-button-primary-m-border-radius: var(--buttonBorderRadius);
+    --spectrum-button-primary-l-border-radius: var(--buttonBorderRadius);
+    --spectrum-button-primary-xl-border-radius: var(--buttonBorderRadius);
 
     /* Loading spinners */
     --spectrum-progresscircle-medium-track-fill-color: var(--primaryColor);

packages/server/package.json

@@ -119,6 +119,7 @@
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
+    "vm2": "^3.9.3",
     "yargs": "13.2.4",
     "zlib": "1.0.5"
   },

packages/server/src/api/controllers/query.js

@@ -4,6 +4,7 @@ const { generateQueryID, getQueryParams } = require("../../db/utils")
 const { integrations } = require("../../integrations")
 const { BaseQueryVerbs } = require("../../constants")
 const env = require("../../environment")
+const ScriptRunner = require("../../utilities/scriptRunner")
 
 // simple function to append "readable" to all read queries
 function enrichQueries(input) {
@@ -28,12 +29,39 @@ function formatResponse(resp) {
       resp = { response: resp }
     }
   }
-  if (!Array.isArray(resp)) {
-    resp = [resp]
-  }
   return resp
 }
 
+async function runAndTransform(
+  integration,
+  queryVerb,
+  enrichedQuery,
+  transformer
+) {
+  let rows = formatResponse(await integration[queryVerb](enrichedQuery))
+
+  // transform as required
+  if (transformer) {
+    const runner = new ScriptRunner(transformer, { data: rows })
+    rows = runner.execute()
+  }
+
+  // needs to an array for next step
+  if (!Array.isArray(rows)) {
+    rows = [rows]
+  }
+
+  // map into JSON if just raw primitive here
+  if (rows.find(row => typeof row !== "object")) {
+    rows = rows.map(value => ({ value }))
+  }
+
+  // get all the potential fields in the schema
+  let keys = rows.flatMap(Object.keys)
+
+  return { rows, keys }
+}
+
 exports.fetch = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
@@ -122,15 +150,16 @@ exports.preview = async function (ctx) {
     ctx.throw(400, "Integration type does not exist.")
   }
 
-  const { fields, parameters, queryVerb } = ctx.request.body
+  const { fields, parameters, queryVerb, transformer } = ctx.request.body
-
   const enrichedQuery = await enrichQueryFields(fields, parameters)
-
   const integration = new Integration(datasource.config)
-  const rows = formatResponse(await integration[queryVerb](enrichedQuery))
 
-  // get all the potential fields in the schema
+  const { rows, keys } = await runAndTransform(
-  const keys = rows.flatMap(Object.keys)
+    integration,
+    queryVerb,
+    enrichedQuery,
+    transformer
+  )
 
   ctx.body = {
     rows,
@@ -158,10 +187,16 @@ exports.execute = async function (ctx) {
     query.fields,
     ctx.request.body.parameters
   )
-
   const integration = new Integration(datasource.config)
+
   // call the relevant CRUD method on the integration class
-  ctx.body = formatResponse(await integration[query.queryVerb](enrichedQuery))
+  const { rows } = await runAndTransform(
+    integration,
+    query.queryVerb,
+    enrichedQuery,
+    query.transformer
+  )
+  ctx.body = rows
   // cleanup
   if (integration.end) {
     integration.end()

packages/server/src/api/controllers/script.js

@@ -1,24 +1,9 @@
-const fetch = require("node-fetch")
+const ScriptRunner = require("../../utilities/scriptRunner")
-const vm = require("vm")
-
-class ScriptExecutor {
-  constructor(body) {
-    const code = `let fn = () => {\n${body.script}\n}; out = fn();`
-    this.script = new vm.Script(code)
-    this.context = vm.createContext(body.context)
-    this.context.fetch = fetch
-  }
-
-  execute() {
-    this.script.runInContext(this.context)
-    return this.context.out
-  }
-}
 
 exports.execute = async function (ctx) {
-  const executor = new ScriptExecutor(ctx.request.body)
+  const { script, context } = ctx.request.body
-
+  const runner = new ScriptRunner(script, context)
-  ctx.body = executor.execute()
+  ctx.body = runner.execute()
 }
 
 exports.save = async function (ctx) {

packages/server/src/api/routes/query.js

@@ -31,7 +31,8 @@ function generateQueryValidation() {
     })),
     queryVerb: Joi.string().allow().required(),
     extra: Joi.object().optional(),
-    schema: Joi.object({}).required().unknown(true)
+    schema: Joi.object({}).required().unknown(true),
+    transformer: Joi.string().optional(),
   }))
 }
 
@@ -42,6 +43,7 @@ function generateQueryPreviewValidation() {
     queryVerb: Joi.string().allow().required(),
     extra: Joi.object().optional(),
     datasourceId: Joi.string().required(),
+    transformer: Joi.string().optional(),
     parameters: Joi.object({}).required().unknown(true)
   }))
 }

packages/server/src/automations/actions.js

@@ -3,7 +3,6 @@ const createRow = require("./steps/createRow")
 const updateRow = require("./steps/updateRow")
 const deleteRow = require("./steps/deleteRow")
 const executeScript = require("./steps/executeScript")
-const bash = require("./steps/bash")
 const executeQuery = require("./steps/executeQuery")
 const outgoingWebhook = require("./steps/outgoingWebhook")
 const serverLog = require("./steps/serverLog")
@@ -14,6 +13,7 @@ const integromat = require("./steps/integromat")
 let filter = require("./steps/filter")
 let delay = require("./steps/delay")
 let queryRow = require("./steps/queryRows")
+const env = require("../environment")
 
 const ACTION_IMPLS = {
   SEND_EMAIL_SMTP: sendSmtpEmail.run,
@@ -22,7 +22,6 @@ const ACTION_IMPLS = {
   DELETE_ROW: deleteRow.run,
   OUTGOING_WEBHOOK: outgoingWebhook.run,
   EXECUTE_SCRIPT: executeScript.run,
-  EXECUTE_BASH: bash.run,
   EXECUTE_QUERY: executeQuery.run,
   SERVER_LOG: serverLog.run,
   DELAY: delay.run,
@@ -42,7 +41,6 @@ const ACTION_DEFINITIONS = {
   OUTGOING_WEBHOOK: outgoingWebhook.definition,
   EXECUTE_SCRIPT: executeScript.definition,
   EXECUTE_QUERY: executeQuery.definition,
-  EXECUTE_BASH: bash.definition,
   SERVER_LOG: serverLog.definition,
   DELAY: delay.definition,
   FILTER: filter.definition,
@@ -54,6 +52,15 @@ const ACTION_DEFINITIONS = {
   integromat: integromat.definition,
 }
 
+// don't add the bash script/definitions unless in self host
+// the fact this isn't included in any definitions means it cannot be
+// ran at all
+if (env.SELF_HOSTED) {
+  const bash = require("./steps/bash")
+  ACTION_IMPLS["EXECUTE_BASH"] = bash.run
+  ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
+}
+
 /* istanbul ignore next */
 exports.getAction = async function (actionName) {
   if (ACTION_IMPLS[actionName] != null) {

packages/server/src/utilities/scriptRunner.js

@@ -0,0 +1,21 @@
+const fetch = require("node-fetch")
+const { VM, VMScript } = require("vm2")
+
+class ScriptRunner {
+  constructor(script, context) {
+    const code = `let fn = () => {\n${script}\n}; results.out = fn();`
+    this.vm = new VM()
+    this.results = { out: "" }
+    this.vm.setGlobals(context)
+    this.vm.setGlobal("fetch", fetch)
+    this.vm.setGlobal("results", this.results)
+    this.script = new VMScript(code)
+  }
+
+  execute() {
+    this.vm.run(this.script)
+    return this.results.out
+  }
+}
+
+module.exports = ScriptRunner

packages/server/yarn.lock

@@ -11006,6 +11006,11 @@ verror@1.10.0:
     core-util-is "1.0.2"
     extsprintf "^1.2.0"
 
+vm2@^3.9.3:
+  version "3.9.3"
+  resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.3.tgz#29917f6cc081cc43a3f580c26c5b553fd3c91f40"
+  integrity sha512-smLS+18RjXYMl9joyJxMNI9l4w7biW8ilSDaVRvFBDwOH8P0BK1ognFQTpg0wyQ6wIKLTblHJvROW692L/E53Q==
+
 vuvuzela@1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/vuvuzela/-/vuvuzela-1.0.3.tgz#3be145e58271c73ca55279dd851f12a682114b0b"

packages/string-templates/package.json

@@ -24,7 +24,8 @@
     "dayjs": "^1.10.4",
     "handlebars": "^4.7.6",
     "handlebars-utils": "^1.0.6",
-    "lodash": "^4.17.20"
+    "lodash": "^4.17.20",
+    "vm2": "^3.9.4"
   },
   "devDependencies": {
     "@rollup/plugin-commonjs": "^17.1.0",

packages/string-templates/rollup.config.js

@@ -7,7 +7,15 @@ import globals from "rollup-plugin-node-globals"
 
 const production = !process.env.ROLLUP_WATCH
 
-const plugins = [
+export default [
+  {
+    input: "src/index.mjs",
+    output: {
+      sourcemap: !production,
+      format: "esm",
+      file: "./dist/bundle.mjs",
+    },
+    plugins: [
       resolve({
         preferBuiltins: true,
         browser: true,
@@ -17,28 +25,6 @@ const plugins = [
       builtins(),
       json(),
       production && terser(),
-]
+    ],
-
-export default [
-  {
-    input: "src/index.mjs",
-    output: {
-      sourcemap: !production,
-      format: "esm",
-      file: "./dist/bundle.mjs",
   },
-    plugins,
-  },
-  // This is the valid configuration for a CommonJS bundle, but since we have
-  // no use for this, it's better to leave it out.
-  // {
-  //   input: "src/index.cjs",
-  //   output: {
-  //     sourcemap: !production,
-  //     format: "cjs",
-  //     file: "./dist/bundle.cjs",
-  //     exports: "named",
-  //   },
-  //   plugins,
-  // },
 ]

packages/string-templates/src/helpers/Helper.js

@@ -1,13 +1,23 @@
 class Helper {
-  constructor(name, fn) {
+  constructor(name, fn, useValueFallback = true) {
     this.name = name
     this.fn = fn
+    this.useValueFallback = useValueFallback
   }
 
   register(handlebars) {
     // wrap the function so that no helper can cause handlebars to break
-    handlebars.registerHelper(this.name, value => {
+    handlebars.registerHelper(this.name, (value, info) => {
-      return this.fn(value) || value
+      let context = {}
+      if (info && info.data && info.data.root) {
+        context = info.data.root
+      }
+      const result = this.fn(value, context)
+      if (result == null) {
+        return this.useValueFallback ? value : null
+      } else {
+        return result
+      }
     })
   }
 

packages/string-templates/src/helpers/constants.js

@@ -19,6 +19,7 @@ module.exports.HelperFunctionNames = {
   OBJECT: "object",
   ALL: "all",
   LITERAL: "literal",
+  JS: "js",
 }
 
 module.exports.LITERAL_MARKER = "%LITERAL%"

packages/string-templates/src/helpers/index.js

@@ -1,6 +1,7 @@
 const Helper = require("./Helper")
 const { SafeString } = require("handlebars")
 const externalHandlebars = require("./external")
+const { processJS } = require("./javascript")
 const {
   HelperFunctionNames,
   HelperFunctionBuiltin,
@@ -17,6 +18,8 @@ const HELPERS = [
   new Helper(HelperFunctionNames.OBJECT, value => {
     return new SafeString(JSON.stringify(value))
   }),
+  // javascript helper
+  new Helper(HelperFunctionNames.JS, processJS, false),
   // this help is applied to all statements
   new Helper(HelperFunctionNames.ALL, value => {
     if (

packages/string-templates/src/helpers/javascript.js

@@ -0,0 +1,49 @@
+const { atob } = require("../utilities")
+
+// The method of executing JS scripts depends on the bundle being built.
+// This setter is used in the entrypoint (either index.cjs or index.mjs).
+let runJS
+module.exports.setJSRunner = runner => (runJS = runner)
+
+// Helper utility to strip square brackets from a value
+const removeSquareBrackets = value => {
+  if (!value || typeof value !== "string") {
+    return value
+  }
+  const regex = /\[+(.+)]+/
+  const matches = value.match(regex)
+  if (matches && matches[1]) {
+    return matches[1]
+  }
+  return value
+}
+
+// Our context getter function provided to JS code as $.
+// Extracts a value from context.
+const getContextValue = (path, context) => {
+  let data = context
+  path.split(".").forEach(key => {
+    if (data == null || typeof data !== "object") {
+      return null
+    }
+    data = data[removeSquareBrackets(key)]
+  })
+  return data
+}
+
+// Evaluates JS code against a certain context
+module.exports.processJS = (handlebars, context) => {
+  try {
+    // Wrap JS in a function and immediately invoke it.
+    // This is required to allow the final `return` statement to be valid.
+    const js = `function run(){${atob(handlebars)}};run();`
+
+    // Our $ context function gets a value from context
+    const sandboxContext = { $: path => getContextValue(path, context) }
+
+    // Create a sandbox with out context and run the JS
+    return runJS(js, sandboxContext)
+  } catch (error) {
+    return "Error while executing JS"
+  }
+}

packages/string-templates/src/index.cjs

@@ -1,161 +1,28 @@
-const handlebars = require("handlebars")
+const { VM } = require("vm2")
-const { registerAll } = require("./helpers/index")
+const templates = require("./index.js")
-const processors = require("./processors")
+const { setJSRunner } = require("./helpers/javascript")
-const { removeHandlebarsStatements } = require("./utilities")
-const manifest = require("../manifest.json")
-
-const hbsInstance = handlebars.create()
-registerAll(hbsInstance)
 
 /**
- * utility function to check if the object is valid
+ * CJS entrypoint for rollup
  */
-function testObject(object) {
+module.exports.isValid = templates.isValid
-  // JSON stringify will fail if there are any cycles, stops infinite recursion
+module.exports.makePropSafe = templates.makePropSafe
-  try {
+module.exports.getManifest = templates.getManifest
-    JSON.stringify(object)
+module.exports.isJSBinding = templates.isJSBinding
-  } catch (err) {
+module.exports.encodeJSBinding = templates.encodeJSBinding
-    throw "Unable to process inputs to JSON, cannot recurse"
+module.exports.decodeJSBinding = templates.decodeJSBinding
-  }
+module.exports.processStringSync = templates.processStringSync
-}
+module.exports.processObjectSync = templates.processObjectSync
+module.exports.processString = templates.processString
+module.exports.processObject = templates.processObject
 
 /**
- * Given an input object this will recurse through all props to try and update any handlebars statements within.
+ * Use vm2 to run JS scripts in a node env
- * @param {object|array} object The input structure which is to be recursed, it is important to note that
- * if the structure contains any cycles then this will fail.
- * @param {object} context The context that handlebars should fill data from.
- * @returns {Promise<object|array>} The structure input, as fully updated as possible.
  */
-module.exports.processObject = async (object, context) => {
+setJSRunner((js, context) => {
-  testObject(object)
+  const vm = new VM({
-  for (let key of Object.keys(object || {})) {
+    sandbox: context,
-    if (object[key] != null) {
+    timeout: 1000
-      let val = object[key]
-      if (typeof val === "string") {
-        object[key] = await module.exports.processString(object[key], context)
-      } else if (typeof val === "object") {
-        object[key] = await module.exports.processObject(object[key], context)
-      }
-    }
-  }
-  return object
-}
-
-/**
- * This will process a single handlebars containing string. If the string passed in has no valid handlebars statements
- * then nothing will occur.
- * @param {string} string The template string which is the filled from the context object.
- * @param {object} context An object of information which will be used to enrich the string.
- * @returns {Promise<string>} The enriched string, all templates should have been replaced if they can be.
- */
-module.exports.processString = async (string, context) => {
-  // TODO: carry out any async calls before carrying out async call
-  return module.exports.processStringSync(string, context)
-}
-
-/**
- * Given an input object this will recurse through all props to try and update any handlebars statements within. This is
- * a pure sync call and therefore does not have the full functionality of the async call.
- * @param {object|array} object The input structure which is to be recursed, it is important to note that
- * if the structure contains any cycles then this will fail.
- * @param {object} context The context that handlebars should fill data from.
- * @returns {object|array} The structure input, as fully updated as possible.
- */
-module.exports.processObjectSync = (object, context) => {
-  testObject(object)
-  for (let key of Object.keys(object || {})) {
-    let val = object[key]
-    if (typeof val === "string") {
-      object[key] = module.exports.processStringSync(object[key], context)
-    } else if (typeof val === "object") {
-      object[key] = module.exports.processObjectSync(object[key], context)
-    }
-  }
-  return object
-}
-
-/**
- * This will process a single handlebars containing string. If the string passed in has no valid handlebars statements
- * then nothing will occur. This is a pure sync call and therefore does not have the full functionality of the async call.
- * @param {string} string The template string which is the filled from the context object.
- * @param {object} context An object of information which will be used to enrich the string.
- * @returns {string} The enriched string, all templates should have been replaced if they can be.
- */
-module.exports.processStringSync = (string, context) => {
-  if (!exports.isValid(string)) {
-    return string
-  }
-  // take a copy of input incase error
-  const input = string
-  if (typeof string !== "string") {
-    throw "Cannot process non-string types."
-  }
-  try {
-    string = processors.preprocess(string)
-    // this does not throw an error when template can't be fulfilled, have to try correct beforehand
-    const template = hbsInstance.compile(string, {
-      strict: false,
   })
-    return processors.postprocess(template({
+  return vm.run(js)
-      now: new Date().toISOString(),
+})
-      ...context,
-    }))
-  } catch (err) {
-    return removeHandlebarsStatements(input)
-  }
-}
-
-/**
- * Simple utility function which makes sure that a templating property has been wrapped in literal specifiers correctly.
- * @param {string} property The property which is to be wrapped.
- * @returns {string} The wrapped property ready to be added to a templating string.
- */
-module.exports.makePropSafe = property => {
-  return `[${property}]`.replace("[[", "[").replace("]]", "]")
-}
-
-/**
- * Checks whether or not a template string contains totally valid syntax (simply tries running it)
- * @param string The string to test for valid syntax - this may contain no templates and will be considered valid.
- * @returns {boolean} Whether or not the input string is valid.
- */
-module.exports.isValid = string => {
-  const validCases = [
-    "string",
-    "number",
-    "object",
-    "array",
-    "cannot read property",
-    "undefined",
-  ]
-  // this is a portion of a specific string always output by handlebars in the case of a syntax error
-  const invalidCases = [`expecting '`]
-  // don't really need a real context to check if its valid
-  const context = {}
-  try {
-    hbsInstance.compile(processors.preprocess(string, false))(context)
-    return true
-  } catch (err) {
-    const msg = err && err.message ? err.message : err
-    if (!msg) {
-      return false
-    }
-    const invalidCase = invalidCases.some(invalidCase =>
-      msg.toLowerCase().includes(invalidCase)
-    )
-    const validCase = validCases.some(validCase =>
-      msg.toLowerCase().includes(validCase)
-    )
-    // special case for maths functions - don't have inputs yet
-    return validCase && !invalidCase
-  }
-}
-
-/**
- * We have generated a static manifest file from the helpers that this string templating package makes use of.
- * This manifest provides information about each of the helpers and how it can be used.
- * @returns The manifest JSON which has been generated from the helpers.
- */
-module.exports.getManifest = () => {
-  return manifest
-}

packages/string-templates/src/index.js

@@ -0,0 +1,204 @@
+const handlebars = require("handlebars")
+const { registerAll } = require("./helpers/index")
+const processors = require("./processors")
+const { removeHandlebarsStatements, atob, btoa } = require("./utilities")
+const manifest = require("../manifest.json")
+
+const hbsInstance = handlebars.create()
+registerAll(hbsInstance)
+
+/**
+ * utility function to check if the object is valid
+ */
+function testObject(object) {
+  // JSON stringify will fail if there are any cycles, stops infinite recursion
+  try {
+    JSON.stringify(object)
+  } catch (err) {
+    throw "Unable to process inputs to JSON, cannot recurse"
+  }
+}
+
+/**
+ * Given an input object this will recurse through all props to try and update any handlebars statements within.
+ * @param {object|array} object The input structure which is to be recursed, it is important to note that
+ * if the structure contains any cycles then this will fail.
+ * @param {object} context The context that handlebars should fill data from.
+ * @returns {Promise<object|array>} The structure input, as fully updated as possible.
+ */
+module.exports.processObject = async (object, context) => {
+  testObject(object)
+  for (let key of Object.keys(object || {})) {
+    if (object[key] != null) {
+      let val = object[key]
+      if (typeof val === "string") {
+        object[key] = await module.exports.processString(object[key], context)
+      } else if (typeof val === "object") {
+        object[key] = await module.exports.processObject(object[key], context)
+      }
+    }
+  }
+  return object
+}
+
+/**
+ * This will process a single handlebars containing string. If the string passed in has no valid handlebars statements
+ * then nothing will occur.
+ * @param {string} string The template string which is the filled from the context object.
+ * @param {object} context An object of information which will be used to enrich the string.
+ * @returns {Promise<string>} The enriched string, all templates should have been replaced if they can be.
+ */
+module.exports.processString = async (string, context) => {
+  // TODO: carry out any async calls before carrying out async call
+  return module.exports.processStringSync(string, context)
+}
+
+/**
+ * Given an input object this will recurse through all props to try and update any handlebars statements within. This is
+ * a pure sync call and therefore does not have the full functionality of the async call.
+ * @param {object|array} object The input structure which is to be recursed, it is important to note that
+ * if the structure contains any cycles then this will fail.
+ * @param {object} context The context that handlebars should fill data from.
+ * @returns {object|array} The structure input, as fully updated as possible.
+ */
+module.exports.processObjectSync = (object, context) => {
+  testObject(object)
+  for (let key of Object.keys(object || {})) {
+    let val = object[key]
+    if (typeof val === "string") {
+      object[key] = module.exports.processStringSync(object[key], context)
+    } else if (typeof val === "object") {
+      object[key] = module.exports.processObjectSync(object[key], context)
+    }
+  }
+  return object
+}
+
+/**
+ * This will process a single handlebars containing string. If the string passed in has no valid handlebars statements
+ * then nothing will occur. This is a pure sync call and therefore does not have the full functionality of the async call.
+ * @param {string} string The template string which is the filled from the context object.
+ * @param {object} context An object of information which will be used to enrich the string.
+ * @returns {string} The enriched string, all templates should have been replaced if they can be.
+ */
+module.exports.processStringSync = (string, context) => {
+  if (!exports.isValid(string)) {
+    return string
+  }
+  // take a copy of input incase error
+  const input = string
+  if (typeof string !== "string") {
+    throw "Cannot process non-string types."
+  }
+  try {
+    string = processors.preprocess(string)
+    // this does not throw an error when template can't be fulfilled, have to try correct beforehand
+    const template = hbsInstance.compile(string, {
+      strict: false,
+    })
+    return processors.postprocess(
+      template({
+        now: new Date().toISOString(),
+        ...context,
+      })
+    )
+  } catch (err) {
+    return removeHandlebarsStatements(input)
+  }
+}
+
+/**
+ * Simple utility function which makes sure that a templating property has been wrapped in literal specifiers correctly.
+ * @param {string} property The property which is to be wrapped.
+ * @returns {string} The wrapped property ready to be added to a templating string.
+ */
+module.exports.makePropSafe = property => {
+  return `[${property}]`.replace("[[", "[").replace("]]", "]")
+}
+
+/**
+ * Checks whether or not a template string contains totally valid syntax (simply tries running it)
+ * @param string The string to test for valid syntax - this may contain no templates and will be considered valid.
+ * @returns {boolean} Whether or not the input string is valid.
+ */
+module.exports.isValid = string => {
+  const validCases = [
+    "string",
+    "number",
+    "object",
+    "array",
+    "cannot read property",
+    "undefined",
+  ]
+  // this is a portion of a specific string always output by handlebars in the case of a syntax error
+  const invalidCases = [`expecting '`]
+  // don't really need a real context to check if its valid
+  const context = {}
+  try {
+    hbsInstance.compile(processors.preprocess(string, false))(context)
+    return true
+  } catch (err) {
+    const msg = err && err.message ? err.message : err
+    if (!msg) {
+      return false
+    }
+    const invalidCase = invalidCases.some(invalidCase =>
+      msg.toLowerCase().includes(invalidCase)
+    )
+    const validCase = validCases.some(validCase =>
+      msg.toLowerCase().includes(validCase)
+    )
+    // special case for maths functions - don't have inputs yet
+    return validCase && !invalidCase
+  }
+}
+
+/**
+ * We have generated a static manifest file from the helpers that this string templating package makes use of.
+ * This manifest provides information about each of the helpers and how it can be used.
+ * @returns The manifest JSON which has been generated from the helpers.
+ */
+module.exports.getManifest = () => {
+  return manifest
+}
+
+/**
+ * Checks if a HBS expression is a valid JS HBS expression
+ * @param handlebars the HBS expression to check
+ * @returns {boolean} whether the expression is JS or not
+ */
+module.exports.isJSBinding = handlebars => {
+  return module.exports.decodeJSBinding(handlebars) != null
+}
+
+/**
+ * Encodes a raw JS string as a JS HBS expression
+ * @param javascript the JS code to encode
+ * @returns {string} the JS HBS expression
+ */
+module.exports.encodeJSBinding = javascript => {
+  return `{{ js "${btoa(javascript)}" }}`
+}
+
+/**
+ * Decodes a JS HBS expression to the raw JS code
+ * @param handlebars the JS HBS expression
+ * @returns {string|null} the raw JS code
+ */
+module.exports.decodeJSBinding = handlebars => {
+  if (!handlebars || typeof handlebars !== "string") {
+    return null
+  }
+
+  // JS is only valid if it is the only HBS expression
+  if (!handlebars.trim().startsWith("{{ js ")) {
+    return null
+  }
+
+  const captureJSRegex = new RegExp(/{{ js "(.*)" }}/)
+  const match = handlebars.match(captureJSRegex)
+  if (!match || match.length < 2) {
+    return null
+  }
+  return atob(match[1])
+}

packages/string-templates/src/index.mjs

@@ -1,12 +1,31 @@
-import templates from "./index.cjs"
+import vm from "vm"
+import templates from "./index.js"
+import { setJSRunner } from "./helpers/javascript"
 
 /**
- * This file is simply an entrypoint for rollup - makes a lot of cjs problems go away
+ * ES6 entrypoint for rollup
  */
 export const isValid = templates.isValid
 export const makePropSafe = templates.makePropSafe
 export const getManifest = templates.getManifest
+export const isJSBinding = templates.isJSBinding
+export const encodeJSBinding = templates.encodeJSBinding
+export const decodeJSBinding = templates.decodeJSBinding
 export const processStringSync = templates.processStringSync
 export const processObjectSync = templates.processObjectSync
 export const processString = templates.processString
 export const processObject = templates.processObject
+
+/**
+ * Use polyfilled vm to run JS scripts in a browser Env
+ */
+setJSRunner((js, context) => {
+  context = {
+    ...context,
+    alert: undefined,
+    setInterval: undefined,
+    setTimeout: undefined,
+  }
+  vm.createContext(context)
+  return vm.runInNewContext(js, context, { timeout: 1000 })
+})

packages/string-templates/src/utilities.js

@@ -22,3 +22,11 @@ module.exports.removeHandlebarsStatements = string => {
   }
   return string
 }
+
+module.exports.btoa = plainText => {
+  return Buffer.from(plainText, "utf-8").toString("base64")
+}
+
+module.exports.atob = base64 => {
+  return Buffer.from(base64, "base64").toString("utf-8")
+}

packages/string-templates/test/javascript.spec.js

@@ -0,0 +1,85 @@
+const { processStringSync, encodeJSBinding } = require("../src/index.cjs")
+
+const processJS = (js, context) => {
+  return processStringSync(encodeJSBinding(js), context)
+}
+
+describe("Test the JavaScript helper", () => {
+  it("should execute a simple expression", () => {
+    const output = processJS(`return 1 + 2`)
+    expect(output).toBe("3")
+  })
+
+  it("should be able to use primitive bindings", () => {
+    const output = processJS(`return $("foo")`, {
+      foo: "bar",
+    })
+    expect(output).toBe("bar")
+  })
+
+  it("should be able to use an object binding", () => {
+    const output = processJS(`return $("foo").bar`, {
+      foo: {
+        bar: "baz",
+      },
+    })
+    expect(output).toBe("baz")
+  })
+
+  it("should be able to use a complex object binding", () => {
+    const output = processJS(`return $("foo").bar[0].baz`, {
+      foo: {
+        bar: [
+          {
+            baz: "shazbat",
+          },
+        ],
+      },
+    })
+    expect(output).toBe("shazbat")
+  })
+
+  it("should be able to use a deep binding", () => {
+    const output = processJS(`return $("foo.bar.baz")`, {
+      foo: {
+        bar: {
+          baz: "shazbat",
+        },
+      },
+    })
+    expect(output).toBe("shazbat")
+  })
+
+  it("should be able to use a deep array binding", () => {
+    const output = processJS(`return $("foo.0.bar")`, {
+      foo: [
+        {
+          bar: "baz",
+        },
+      ],
+    })
+    expect(output).toBe("baz")
+  })
+
+  it("should handle errors", () => {
+    const output = processJS(`throw "Error"`)
+    expect(output).toBe("Error while executing JS")
+  })
+
+  it("should timeout after one second", () => {
+    const output = processJS(`while (true) {}`)
+    expect(output).toBe("Error while executing JS")
+  })
+
+  it("should prevent access to the process global", () => {
+    const output = processJS(`return process`)
+    expect(output).toBe("Error while executing JS")
+  })
+
+  it("should prevent sandbox escape", () => {
+    const output = processJS(
+      `return this.constructor.constructor("return process")()`
+    )
+    expect(output).toBe("Error while executing JS")
+  })
+})

packages/string-templates/yarn.lock

@@ -4572,6 +4572,11 @@ vlq@^0.2.2:
   resolved "https://registry.yarnpkg.com/vlq/-/vlq-0.2.3.tgz#8f3e4328cf63b1540c0d67e1b2778386f8975b26"
   integrity sha512-DRibZL6DsNhIgYQ+wNdWDL2SL3bKPlVrRiBqV5yuMm++op8W4kGFtaQfCs4KEJn0wBZcHVHJ3eoywX8983k1ow==
 
+vm2@^3.9.4:
+  version "3.9.4"
+  resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.4.tgz#2e118290fefe7bd8ea09ebe2f5faf53730dbddaa"
+  integrity sha512-sOdharrJ7KEePIpHekiWaY1DwgueuiBeX/ZBJUPgETsVlJsXuEx0K0/naATq2haFvJrvZnRiORQRubR0b7Ye6g==
+
 w3c-hr-time@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz#0a89cdf5cc15822df9c360543676963e0cc308cd"