Merge branch 'develop' into options-text-columns

.github/workflows/release-develop.yml

@@ -9,7 +9,7 @@ env:
   POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
   INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
   POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
-  SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+  SENTRY_DSN: ${{ secrets.SENTRY_DSN_SELF_HOST }}
       
 jobs:
   release:

.github/workflows/release-selfhost.yml

@@ -7,7 +7,7 @@ env:
   POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
   INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
   POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
-  SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+  SENTRY_DSN: ${{ secrets.SENTRY_DSN_SELF_HOST }}
 
 jobs:
   release:

hosting/docker-compose.yaml

@@ -21,7 +21,7 @@ services:
       PORT: 4002
       JWT_SECRET: ${JWT_SECRET}
       LOG_LEVEL: info
-      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
+      SENTRY_DSN: https://cc54bb0358fd4300ae97ef2273fbaf9f@o420233.ingest.sentry.io/6007553
       ENABLE_ANALYTICS: "true"
       REDIS_URL: redis-service:6379
       REDIS_PASSWORD: ${REDIS_PASSWORD}
@@ -51,7 +51,6 @@ services:
       INTERNAL_API_KEY: ${INTERNAL_API_KEY}
       REDIS_URL: redis-service:6379
       REDIS_PASSWORD: ${REDIS_PASSWORD}
-      ACCOUNT_PORTAL_URL: https://portal.budi.live
     volumes:
       - ./logs:/logs
     depends_on:

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.167-alpha.1",
+  "version": "0.9.167-alpha.12",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.167-alpha.1",
+  "version": "0.9.167-alpha.12",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.167-alpha.1",
+  "version": "0.9.167-alpha.12",
   "license": "AGPL-3.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/bbui/src/Table/CellRenderer.svelte

@@ -5,7 +5,6 @@
   import RelationshipRenderer from "./RelationshipRenderer.svelte"
   import AttachmentRenderer from "./AttachmentRenderer.svelte"
   import ArrayRenderer from "./ArrayRenderer.svelte"
-  import InternalRenderer from "./InternalRenderer.svelte"
 
   export let row
   export let schema
@@ -23,9 +22,7 @@
     number: StringRenderer,
     longform: StringRenderer,
     array: ArrayRenderer,
-    internal: InternalRenderer,
   }
-
   $: type = schema?.type ?? "string"
   $: customRenderer = customRenderers?.find(x => x.column === schema?.name)
   $: renderer = customRenderer?.component ?? typeMap[type] ?? StringRenderer

packages/bbui/src/Tabs/Tab.svelte

@@ -8,11 +8,19 @@
   const selected = getContext("tab")
   let tab
   let tabInfo
+
   const setTabInfo = () => {
+    // If the tabs are being rendered inside a component which uses
+    // a svelte transition to enter, then this initial getBoundingClientRect
+    // will return an incorrect position.
+    // We just need to get this off the main thread to fix this, by using
+    // a 0ms timeout.
+    setTimeout(() => {
       tabInfo = tab.getBoundingClientRect()
       if ($selected.title === title) {
         $selected.info = tabInfo
       }
+    }, 0)
   }
 
   onMount(() => {

packages/builder/cypress/integration/createTable.spec.js

@@ -31,7 +31,7 @@ context("Create a Table", () => {
     cy.contains("nameupdated ").should("contain", "nameupdated")
   })
 
-  /*
+  
   it("edits a row", () => {
     cy.contains("button", "Edit").click({ force: true })
     cy.wait(1000)
@@ -40,7 +40,7 @@ context("Create a Table", () => {
     cy.contains("Save").click()
     cy.contains("Updated").should("have.text", "Updated")
   })
-  */
+  
   it("deletes a row", () => {
     cy.get(".spectrum-Checkbox-input").check({ force: true })
     cy.contains("Delete 1 row(s)").click()

packages/builder/cypress/support/commands.js

@@ -36,18 +36,11 @@ Cypress.Commands.add("createApp", name => {
   cy.visit(`localhost:${Cypress.env("PORT")}/builder`)
   cy.wait(500)
   cy.contains(/Start from scratch/).click()
-  cy.get(".spectrum-Modal")
+  cy.get(".spectrum-Modal").within(() => {
-    .within(() => {
     cy.get("input").eq(0).type(name).should("have.value", name).blur()
     cy.get(".spectrum-ButtonGroup").contains("Create app").click()
     cy.wait(7000)
   })
-    .then(() => {
-      // Because we show the datasource modal on entry, we need to create a table to get rid of the modal in the future
-      cy.createInitialDatasource("initialTable")
-      cy.expandBudibaseConnection()
-      cy.get(".nav-item.selected > .content").should("be.visible")
-    })
 })
 
 Cypress.Commands.add("deleteApp", () => {
@@ -77,22 +70,6 @@ Cypress.Commands.add("createTestTableWithData", () => {
   cy.addColumn("dog", "age", "Number")
 })
 
-Cypress.Commands.add("createInitialDatasource", tableName => {
-  // Enter table name
-  cy.get(".spectrum-Modal").within(() => {
-    cy.contains("Budibase DB").trigger("mouseover").click().click()
-    cy.wait(1000)
-    cy.contains("Continue").click()
-  })
-
-  cy.get(".spectrum-Modal").within(() => {
-    cy.wait(1000)
-    cy.get("input").first().type(tableName).blur()
-    cy.get(".spectrum-ButtonGroup").contains("Create").click()
-  })
-  cy.contains(tableName).should("be.visible")
-})
-
 Cypress.Commands.add("createTable", tableName => {
   cy.contains("Budibase DB").click()
   cy.contains("Create new table").click()

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.167-alpha.1",
+  "version": "0.9.167-alpha.12",
   "license": "AGPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.167-alpha.1",
+    "@budibase/bbui": "^0.9.167-alpha.12",
-    "@budibase/client": "^0.9.167-alpha.1",
+    "@budibase/client": "^0.9.167-alpha.12",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.167-alpha.1",
+    "@budibase/string-templates": "^0.9.167-alpha.12",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/builderStore/api.js

@@ -15,7 +15,7 @@ const apiCall =
     if (resp.status === 403) {
       removeCookie(Cookies.Auth)
       // reload after removing cookie, go to login
-      if (!url.includes("self")) {
+      if (!url.includes("self") && !url.includes("login")) {
         location.reload()
       }
     }

packages/builder/src/builderStore/dataBinding.js

@@ -7,11 +7,17 @@ import {
 } from "./storeUtils"
 import { store } from "builderStore"
 import { queries as queriesStores, tables as tablesStore } from "stores/backend"
-import { makePropSafe } from "@budibase/string-templates"
+import {
+  makePropSafe,
+  isJSBinding,
+  decodeJSBinding,
+  encodeJSBinding,
+} from "@budibase/string-templates"
 import { TableNames } from "../constants"
 
 // Regex to match all instances of template strings
 const CAPTURE_VAR_INSIDE_TEMPLATE = /{{([^}]+)}}/g
+const CAPTURE_VAR_INSIDE_JS = /\$\("([^")]+)"\)/g
 const CAPTURE_HBS_TEMPLATE = /{{[\S\s]*?}}/g
 
 /**
@@ -430,6 +436,15 @@ function replaceBetween(string, start, end, replacement) {
  * utility function for the readableToRuntimeBinding and runtimeToReadableBinding.
  */
 function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
+  // Decide from base64 if using JS
+  const isJS = isJSBinding(textWithBindings)
+  if (isJS) {
+    textWithBindings = decodeJSBinding(textWithBindings)
+  }
+
+  // Determine correct regex to find bindings to replace
+  const regex = isJS ? CAPTURE_VAR_INSIDE_JS : CAPTURE_VAR_INSIDE_TEMPLATE
+
   const convertFrom =
     convertTo === "runtimeBinding" ? "readableBinding" : "runtimeBinding"
   if (typeof textWithBindings !== "string") {
@@ -441,7 +456,7 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
     .sort((a, b) => {
       return b.length - a.length
     })
-  const boundValues = textWithBindings.match(CAPTURE_VAR_INSIDE_TEMPLATE) || []
+  const boundValues = textWithBindings.match(regex) || []
   let result = textWithBindings
   for (let boundValue of boundValues) {
     let newBoundValue = boundValue
@@ -449,7 +464,7 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
     // in the search, working from longest to shortest so always use best match first
     let searchString = newBoundValue
     for (let from of convertFromProps) {
-      if (shouldReplaceBinding(newBoundValue, from, convertTo)) {
+      if (isJS || shouldReplaceBinding(newBoundValue, from, convertTo)) {
         const binding = bindableProperties.find(el => el[convertFrom] === from)
         let idx
         do {
@@ -472,6 +487,12 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
     }
     result = result.replace(boundValue, newBoundValue)
   }
+
+  // Re-encode to base64 if using JS
+  if (isJS) {
+    result = encodeJSBinding(result)
+  }
+
   return result
 }
 

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -103,7 +103,7 @@
           <Detail size="S">{block?.name?.toUpperCase() || ""}</Detail>
         </div>
       </div>
-      {#if testResult}
+      {#if testResult && testResult[0]}
         <span on:click={() => resultsModal.show()}>
           <StatusLight
             positive={isTrigger || testResult[0].outputs?.success}

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -194,6 +194,7 @@
             value={inputData[key]}
             on:change={e => onChange(e, key)}
             {bindings}
+            allowJS={false}
           />
         {/if}
       {:else if value.customType === "query"}
@@ -259,6 +260,7 @@
               value={inputData[key]}
               on:change={e => onChange(e, key)}
               {bindings}
+              allowJS={false}
             />
           </div>
         {/if}

packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte

@@ -39,6 +39,7 @@
         type="string"
         {bindings}
         fillWidth={true}
+        allowJS={false}
       />
     {/each}
   </div>

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -1,6 +1,12 @@
 <script>
   import { tables } from "stores/backend"
-  import { Select, Toggle, DatePicker, Multiselect } from "@budibase/bbui"
+  import {
+    Select,
+    Toggle,
+    DatePicker,
+    Multiselect,
+    TextArea,
+  } from "@budibase/bbui"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
@@ -52,7 +58,6 @@
   getOptionLabel={table => table.name}
   getOptionValue={table => table._id}
 />
-
 {#if schemaFields.length}
   <div class="schema-fields">
     {#each schemaFields as [field, schema]}
@@ -82,6 +87,8 @@
             label={field}
             options={schema.constraints.inclusion}
           />
+        {:else if schema.type === "longform"}
+          <TextArea label={field} bind:value={value[field]} />
         {:else if schema.type === "link"}
           <LinkedRowSelector bind:linkedRows={value[field]} {schema} />
         {:else if schema.type === "string" || schema.type === "number"}
@@ -103,6 +110,7 @@
               type="string"
               {bindings}
               fillWidth={true}
+              allowJS={false}
             />
           {/if}
         {/if}

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -16,29 +16,11 @@
   import { Pagination } from "@budibase/bbui"
 
   let hideAutocolumns = true
-  let schema
   $: isUsersTable = $tables.selected?._id === TableNames.USERS
   $: type = $tables.selected?.type
   $: isInternal = type !== "external"
-  $: {
+  $: schema = $tables.selected?.schema
-    schema = $tables.selected?.schema
 
-    // Manually add these as we don't want them to be 'real' auto-columns
-    schema._id = {
-      type: "internal",
-      editable: false,
-      displayName: "ID",
-      autocolumn: true,
-    }
-    if (isInternal) {
-      schema._rev = {
-        type: "internal",
-        editable: false,
-        displayName: "Revision",
-        autocolumn: true,
-      }
-    }
-  }
   $: id = $tables.selected?._id
   $: search = searchTable(id)
   $: columnOptions = Object.keys($search.schema || {})

packages/builder/src/components/common/CodeMirrorEditor.svelte

@@ -0,0 +1,159 @@
+<script context="module">
+  import { Label } from "@budibase/bbui"
+
+  export const EditorModes = {
+    JS: {
+      name: "javascript",
+      json: false,
+    },
+    JSON: {
+      name: "javascript",
+      json: true,
+    },
+    SQL: {
+      name: "sql",
+    },
+    Handlebars: {
+      name: "handlebars",
+      base: "text/html",
+    },
+  }
+</script>
+
+<script>
+  import CodeMirror from "components/integration/codemirror"
+  import { themeStore } from "builderStore"
+  import { createEventDispatcher, onMount } from "svelte"
+
+  export let mode = EditorModes.JS
+  export let value = ""
+  export let height = 300
+  export let resize = "none"
+  export let readonly = false
+  export let hints = []
+  export let label
+
+  const dispatch = createEventDispatcher()
+  let textarea
+  let editor
+
+  // Keep editor up to date with value
+  $: editor?.setValue(value || "")
+
+  // Creates an instance of a code mirror editor
+  async function createEditor(mode, value) {
+    if (!CodeMirror || !textarea || editor) {
+      return
+    }
+
+    // Configure CM options
+    const lightTheme = $themeStore.theme.includes("light")
+    const options = {
+      mode,
+      value: value || "",
+      readOnly: readonly,
+      theme: lightTheme ? "default" : "tomorrow-night-eighties",
+
+      // Style
+      lineNumbers: true,
+      lineWrapping: true,
+      indentWithTabs: true,
+      indentUnit: 2,
+      tabSize: 2,
+
+      // QOL addons
+      extraKeys: { "Ctrl-Space": "autocomplete" },
+      styleActiveLine: { nonEmpty: true },
+      autoCloseBrackets: true,
+      matchBrackets: true,
+    }
+
+    // Register hints plugin if desired
+    if (hints?.length) {
+      CodeMirror.registerHelper("hint", "dictionaryHint", function (editor) {
+        const cursor = editor.getCursor()
+        return {
+          list: hints,
+          from: CodeMirror.Pos(cursor.line, cursor.ch),
+          to: CodeMirror.Pos(cursor.line, cursor.ch),
+        }
+      })
+      CodeMirror.commands.autocomplete = function (cm) {
+        CodeMirror.showHint(cm, CodeMirror.hint.dictionaryHint)
+      }
+    }
+
+    // Construct CM instance
+    editor = CodeMirror.fromTextArea(textarea, options)
+
+    // Use a blur handler to update the value
+    editor.on("blur", instance => {
+      dispatch("change", instance.getValue())
+    })
+  }
+
+  // Export a function to expose caret position
+  export const getCaretPosition = () => {
+    const cursor = editor.getCursor()
+    return {
+      start: cursor.ch,
+      end: cursor.ch,
+    }
+  }
+
+  onMount(() => {
+    // Create the editor with initial value
+    createEditor(mode, value)
+
+    // Clean up editor on unmount
+    return () => {
+      if (editor) {
+        editor.toTextArea()
+      }
+    }
+  })
+</script>
+
+{#if label}
+  <div style="margin-bottom: var(--spacing-s)">
+    <Label small>{label}</Label>
+  </div>
+{/if}
+<div
+  style={`--code-mirror-height: ${height}px; --code-mirror-resize: ${resize}`}
+>
+  <textarea tabindex="0" bind:this={textarea} readonly {value} />
+</div>
+
+<style>
+  div :global(.CodeMirror) {
+    height: var(--code-mirror-height);
+    min-height: var(--code-mirror-height);
+    font-family: monospace;
+    line-height: 1.3;
+    border: var(--spectrum-alias-border-size-thin) solid;
+    border-color: var(--spectrum-alias-border-color);
+    border-radius: var(--border-radius-s);
+    resize: var(--code-mirror-resize);
+    overflow: hidden;
+  }
+
+  /* Override default active line highlight colour in dark theme */
+  div
+    :global(.CodeMirror-focused.cm-s-tomorrow-night-eighties
+      .CodeMirror-activeline-background) {
+    background: rgba(255, 255, 255, 0.075);
+  }
+
+  /* Remove active line styling when not focused */
+  div
+    :global(.CodeMirror:not(.CodeMirror-focused)
+      .CodeMirror-activeline-background) {
+    background: unset;
+  }
+
+  /* Add a spectrum themed border when focused */
+  div :global(.CodeMirror-focused) {
+    border-color: var(--spectrum-alias-border-color-mouse-focus);
+  }
+</style>

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -1,32 +1,98 @@
 <script>
   import groupBy from "lodash/fp/groupBy"
-  import { Search, TextArea, DrawerContent } from "@budibase/bbui"
+  import {
-  import { createEventDispatcher } from "svelte"
+    Search,
-  import { isValid } from "@budibase/string-templates"
+    TextArea,
+    DrawerContent,
+    Tabs,
+    Tab,
+    Body,
+    Layout,
+  } from "@budibase/bbui"
+  import { createEventDispatcher, onMount } from "svelte"
+  import {
+    isValid,
+    decodeJSBinding,
+    encodeJSBinding,
+  } from "@budibase/string-templates"
   import { readableToRuntimeBinding } from "builderStore/dataBinding"
   import { handlebarsCompletions } from "constants/completions"
-  import { addToText } from "./utils"
+  import { addHBSBinding, addJSBinding } from "./utils"
+  import CodeMirrorEditor from "components/common/CodeMirrorEditor.svelte"
 
   const dispatch = createEventDispatcher()
 
   export let bindableProperties
   export let value = ""
   export let valid
+  export let allowJS = false
 
   let helpers = handlebarsCompletions()
   let getCaretPosition
   let search = ""
+  let initialValueJS = value?.startsWith("{{ js ")
+  let mode = initialValueJS ? "JavaScript" : "Handlebars"
+  let jsValue = initialValueJS ? value : null
+  let hbsValue = initialValueJS ? null : value
 
-  $: valid = isValid(readableToRuntimeBinding(bindableProperties, value))
+  $: usingJS = mode === "JavaScript"
-  $: dispatch("change", value)
   $: ({ context } = groupBy("type", bindableProperties))
   $: searchRgx = new RegExp(search, "ig")
-  $: filteredColumns = context?.filter(context => {
+  $: filteredBindings = context?.filter(context => {
     return context.readableBinding.match(searchRgx)
   })
   $: filteredHelpers = helpers?.filter(helper => {
     return helper.label.match(searchRgx) || helper.description.match(searchRgx)
   })
+
+  const updateValue = value => {
+    valid = isValid(readableToRuntimeBinding(bindableProperties, value))
+    if (valid) {
+      dispatch("change", value)
+    }
+  }
+
+  // Adds a HBS helper to the expression
+  const addHelper = helper => {
+    hbsValue = addHBSBinding(value, getCaretPosition(), helper.text)
+    updateValue(hbsValue)
+  }
+
+  // Adds a data binding to the expression
+  const addBinding = binding => {
+    if (usingJS) {
+      let js = decodeJSBinding(jsValue)
+      js = addJSBinding(js, getCaretPosition(), binding.readableBinding)
+      jsValue = encodeJSBinding(js)
+      updateValue(jsValue)
+    } else {
+      hbsValue = addHBSBinding(
+        hbsValue,
+        getCaretPosition(),
+        binding.readableBinding
+      )
+      updateValue(hbsValue)
+    }
+  }
+
+  const onChangeMode = e => {
+    mode = e.detail
+    updateValue(mode === "JavaScript" ? jsValue : hbsValue)
+  }
+
+  const onChangeHBSValue = e => {
+    hbsValue = e.detail
+    updateValue(hbsValue)
+  }
+
+  const onChangeJSValue = e => {
+    jsValue = encodeJSBinding(e.detail)
+    updateValue(jsValue)
+  }
+
+  onMount(() => {
+    valid = isValid(readableToRuntimeBinding(bindableProperties, value))
+  })
 </script>
 
 <DrawerContent>
@@ -36,32 +102,24 @@
         <div class="heading">Search</div>
         <Search placeholder="Search" bind:value={search} />
       </section>
-      {#if filteredColumns?.length}
+      {#if filteredBindings?.length}
         <section>
           <div class="heading">Bindable Values</div>
           <ul>
-            {#each filteredColumns as { readableBinding }}
+            {#each filteredBindings as binding}
-              <li
+              <li on:click={() => addBinding(binding)}>
-                on:click={() => {
+                {binding.readableBinding}
-                  value = addToText(value, getCaretPosition(), readableBinding)
-                }}
-              >
-                {readableBinding}
               </li>
             {/each}
           </ul>
         </section>
       {/if}
-      {#if filteredHelpers?.length}
+      {#if filteredHelpers?.length && !usingJS}
         <section>
           <div class="heading">Helpers</div>
           <ul>
             {#each filteredHelpers as helper}
-              <li
+              <li on:click={() => addHelper(helper)}>
-                on:click={() => {
-                  value = addToText(value, getCaretPosition(), helper.text)
-                }}
-              >
                 <div class="helper">
                   <div class="helper__name">{helper.displayText}</div>
                   <div class="helper__description">
@@ -77,9 +135,13 @@
     </div>
   </svelte:fragment>
   <div class="main">
+    <Tabs selected={mode} on:select={onChangeMode}>
+      <Tab title="Handlebars">
+        <div class="main-content">
           <TextArea
             bind:getCaretPosition
-      bind:value
+            value={hbsValue}
+            on:change={onChangeHBSValue}
             placeholder="Add text, or click the objects on the left to add them to the textbox."
           />
           {#if !valid}
@@ -90,11 +152,39 @@
             </p>
           {/if}
         </div>
+      </Tab>
+      {#if allowJS}
+        <Tab title="JavaScript">
+          <div class="main-content">
+            <Layout noPadding gap="XS">
+              <CodeMirrorEditor
+                bind:getCaretPosition
+                height={200}
+                value={decodeJSBinding(jsValue)}
+                on:change={onChangeJSValue}
+                hints={context?.map(x => `$("${x.readableBinding}")`)}
+              />
+              <Body size="S">
+                JavaScript expressions are executed as functions, so ensure that
+                your expression returns a value.
+              </Body>
+            </Layout>
+          </div>
+        </Tab>
+      {/if}
+    </Tabs>
+  </div>
 </DrawerContent>
 
 <style>
   .main :global(textarea) {
-    min-height: 150px !important;
+    min-height: 202px !important;
+  }
+  .main {
+    margin: calc(-1 * var(--spacing-xl));
+  }
+  .main-content {
+    padding: var(--spacing-s) var(--spacing-xl);
   }
 
   .container {

packages/builder/src/components/common/bindings/DrawerBindableCombobox.svelte

@@ -6,6 +6,7 @@
   } from "builderStore/dataBinding"
   import BindingPanel from "components/common/bindings/BindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
+  import { isJSBinding } from "@budibase/string-templates"
 
   export let panel = BindingPanel
   export let value = ""
@@ -15,11 +16,14 @@
   export let label
   export let disabled = false
   export let options
+  export let allowJS = true
 
   const dispatch = createEventDispatcher()
   let bindingDrawer
+
   $: readableValue = runtimeToReadableBinding(bindings, value)
   $: tempValue = readableValue
+  $: isJS = isJSBinding(value)
 
   const handleClose = () => {
     onChange(tempValue)
@@ -35,7 +39,7 @@
   <Combobox
     {label}
     {disabled}
-    value={readableValue}
+    value={isJS ? "(JavaScript function)" : readableValue}
     on:change={event => onChange(event.detail)}
     {placeholder}
     {options}
@@ -58,6 +62,7 @@
     close={handleClose}
     on:change={event => (tempValue = event.detail)}
     bindableProperties={bindings}
+    {allowJS}
   />
 </Drawer>
 

packages/builder/src/components/common/bindings/DrawerBindableInput.svelte

@@ -6,6 +6,7 @@
   } from "builderStore/dataBinding"
   import BindingPanel from "components/common/bindings/BindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
+  import { isJSBinding } from "@budibase/string-templates"
 
   export let panel = BindingPanel
   export let value = ""
@@ -15,12 +16,15 @@
   export let label
   export let disabled = false
   export let fillWidth
+  export let allowJS = true
 
   const dispatch = createEventDispatcher()
   let bindingDrawer
   let valid = true
+
   $: readableValue = runtimeToReadableBinding(bindings, value)
   $: tempValue = readableValue
+  $: isJS = isJSBinding(value)
 
   const saveBinding = () => {
     onChange(tempValue)
@@ -36,7 +40,7 @@
   <Input
     {label}
     {disabled}
-    value={readableValue}
+    value={isJS ? "(JavaScript function)" : readableValue}
     on:change={event => onChange(event.detail)}
     {placeholder}
   />
@@ -60,6 +64,7 @@
     value={readableValue}
     on:change={event => (tempValue = event.detail)}
     bindableProperties={bindings}
+    {allowJS}
   />
 </Drawer>
 

packages/builder/src/components/common/bindings/ServerBindingPanel.svelte

@@ -5,7 +5,7 @@
   import { isValid } from "@budibase/string-templates"
   import { handlebarsCompletions } from "constants/completions"
   import { readableToRuntimeBinding } from "builderStore/dataBinding"
-  import { addToText } from "./utils"
+  import { addHBSBinding } from "./utils"
 
   const dispatch = createEventDispatcher()
 
@@ -47,7 +47,7 @@
               {#each bindings as binding}
                 <li
                   on:click={() => {
-                    value = addToText(value, getCaretPosition(), binding)
+                    value = addHBSBinding(value, getCaretPosition(), binding)
                   }}
                 >
                   <span class="binding__label">{binding.label}</span>
@@ -71,7 +71,7 @@
             {#each filteredHelpers as helper}
               <li
                 on:click={() => {
-                  value = addToText(value, getCaretPosition(), helper.text)
+                  value = addHBSBinding(value, getCaretPosition(), helper.text)
                 }}
               >
                 <div class="helper">

packages/builder/src/components/common/bindings/utils.js

@@ -1,4 +1,4 @@
-export function addToText(value, caretPos, binding) {
+export function addHBSBinding(value, caretPos, binding) {
   binding = typeof binding === "string" ? binding : binding.path
   value = value == null ? "" : value
   if (!value.includes("{{") && !value.includes("}}")) {
@@ -14,3 +14,18 @@ export function addToText(value, caretPos, binding) {
   }
   return value
 }
+
+export function addJSBinding(value, caretPos, binding) {
+  binding = typeof binding === "string" ? binding : binding.path
+  value = value == null ? "" : value
+  binding = `$("${binding}")`
+  if (caretPos.start) {
+    value =
+      value.substring(0, caretPos.start) +
+      binding +
+      value.substring(caretPos.end, value.length)
+  } else {
+    value += binding
+  }
+  return value
+}

packages/builder/src/components/design/PropertiesPanel/PropertyControls/Input.svelte

@@ -0,0 +1,15 @@
+<script>
+  import { Input } from "@budibase/bbui"
+  import { isJSBinding } from "@budibase/string-templates"
+
+  export let value
+
+  $: isJS = isJSBinding(value)
+</script>
+
+<Input
+  {...$$props}
+  value={isJS ? "(JavaScript function)" : value}
+  readonly={isJS}
+  on:change
+/>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/PropertyControl.svelte

@@ -107,6 +107,7 @@
           value={safeValue}
           on:change={e => (tempValue = e.detail)}
           bindableProperties={bindings}
+          allowJS
         />
       </Drawer>
     {/if}

packages/builder/src/components/design/PropertiesPanel/PropertyControls/componentSettings.js

@@ -1,4 +1,4 @@
-import { Checkbox, Input, Select, Stepper } from "@budibase/bbui"
+import { Checkbox, Select, Stepper } from "@budibase/bbui"
 import DataSourceSelect from "./DataSourceSelect.svelte"
 import DataProviderSelect from "./DataProviderSelect.svelte"
 import EventsEditor from "./EventsEditor"
@@ -15,6 +15,7 @@ import URLSelect from "./URLSelect.svelte"
 import OptionsEditor from "./OptionsEditor/OptionsEditor.svelte"
 import FormFieldSelect from "./FormFieldSelect.svelte"
 import ValidationEditor from "./ValidationEditor/ValidationEditor.svelte"
+import Input from "./Input.svelte"
 
 const componentMap = {
   text: Input,

packages/builder/src/components/integration/QueryViewer.svelte

@@ -21,12 +21,15 @@
   import ParameterBuilder from "components/integration/QueryParameterBuilder.svelte"
   import { datasources, integrations, queries } from "stores/backend"
   import { capitalise } from "../../helpers"
+  import CodeMirrorEditor from "components/common/CodeMirrorEditor.svelte"
 
   export let query
   export let fields = []
 
   let parameters
   let data = []
+  const transformerDocs =
+    "https://docs.budibase.com/building-apps/data/transformers"
   const typeOptions = [
     { label: "Text", value: "STRING" },
     { label: "Number", value: "NUMBER" },
@@ -52,6 +55,11 @@
   $: readQuery = query.queryVerb === "read" || query.readable
   $: queryInvalid = !query.name || (readQuery && data.length === 0)
 
+  // seed the transformer
+  if (query && !query.transformer) {
+    query.transformer = "return data"
+  }
+
   function newField() {
     fields = [...fields, {}]
   }
@@ -74,6 +82,7 @@
       const response = await api.post(`/api/queries/preview`, {
         fields: query.fields,
         queryVerb: query.queryVerb,
+        transformer: query.transformer,
         parameters: query.parameters.reduce(
           (acc, next) => ({
             ...acc,
@@ -160,12 +169,34 @@
       <IntegrationQueryEditor
         {datasource}
         {query}
-        height={300}
+        height={200}
         schema={queryConfig[query.queryVerb]}
         bind:parameters
       />
       <Divider />
     </div>
+    <div class="config">
+      <div class="help-heading">
+        <Heading size="S">Transformer</Heading>
+        <Icon
+          on:click={() => window.open(transformerDocs)}
+          hoverable
+          name="Help"
+          size="L"
+        />
+      </div>
+      <Body size="S"
+        >Add a JavaScript function to transform the query result.</Body
+      >
+      <CodeMirrorEditor
+        height={200}
+        label="Transformer"
+        value={query.transformer}
+        resize="vertical"
+        on:change={e => (query.transformer = e.detail)}
+      />
+      <Divider />
+    </div>
     <div class="viewer-controls">
       <Heading size="S">Results</Heading>
       <ButtonGroup>
@@ -220,6 +251,7 @@
     display: grid;
     grid-gap: var(--spacing-s);
   }
+
   .config-field {
     display: grid;
     grid-template-columns: 20% 1fr;
@@ -227,6 +259,11 @@
     align-items: center;
   }
 
+  .help-heading {
+    display: flex;
+    justify-content: space-between;
+  }
+
   .field {
     display: grid;
     grid-template-columns: 1fr 1fr 5%;

packages/builder/src/components/integration/codemirror.js

@@ -1,12 +1,22 @@
 import CodeMirror from "codemirror"
 import "codemirror/lib/codemirror.css"
-import "codemirror/theme/tomorrow-night-eighties.css"
+
-import "codemirror/addon/hint/show-hint.css"
+// Modes
-import "codemirror/theme/neo.css"
+import "codemirror/mode/javascript/javascript"
 import "codemirror/mode/sql/sql"
 import "codemirror/mode/css/css"
 import "codemirror/mode/handlebars/handlebars"
-import "codemirror/mode/javascript/javascript"
+
+// Hints
 import "codemirror/addon/hint/show-hint"
+import "codemirror/addon/hint/show-hint.css"
+
+// Theming
+import "codemirror/theme/tomorrow-night-eighties.css"
+
+// Functional addons
+import "codemirror/addon/selection/active-line"
+import "codemirror/addon/edit/closebrackets"
+import "codemirror/addon/edit/matchbrackets"
 
 export default CodeMirror

packages/builder/src/components/start/CreateAppModal.svelte

@@ -150,7 +150,6 @@
     showCancelButton={false}
     showCloseIcon={false}
   >
-    <Body size="M">Select a template below, or start from scratch.</Body>
     <TemplateList
       onSelect={selected => {
         if (!selected) {

packages/builder/src/components/start/TemplateList.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { Heading, Layout, Icon } from "@budibase/bbui"
+  import { Heading, Layout, Icon, Body } from "@budibase/bbui"
   import Spinner from "components/common/Spinner.svelte"
   import api from "builderStore/api"
 
@@ -7,6 +7,7 @@
 
   async function fetchTemplates() {
     const response = await api.get("/api/templates?type=app")
+    console.log("Responded")
     return await response.json()
   }
 
@@ -19,6 +20,11 @@
       <Spinner size="30" />
     </div>
   {:then templates}
+    {#if templates?.length > 0}
+      <Body size="M">Select a template below, or start from scratch.</Body>
+    {:else}
+      <Body size="M">Start your app from scratch below.</Body>
+    {/if}
     <div class="templates">
       {#each templates as template}
         <div class="template" on:click={() => onSelect(template)}>

packages/builder/src/helpers/fetchTableData.js

@@ -48,7 +48,6 @@ export const fetchTableData = opts => {
   const fetchPage = async bookmark => {
     lastBookmark = bookmark
     const { tableId, limit, sortColumn, sortOrder, paginate } = options
-    store.update($store => ({ ...$store, loading: true }))
     const res = await API.post(`/api/${options.tableId}/search`, {
       tableId,
       query,
@@ -59,7 +58,6 @@ export const fetchTableData = opts => {
       paginate,
       bookmark,
     })
-    store.update($store => ({ ...$store, loading: false, loaded: true }))
     return await res.json()
   }
 
@@ -103,7 +101,7 @@ export const fetchTableData = opts => {
     if (!schema) {
       return
     }
-    store.update($store => ({ ...$store, schema }))
+    store.update($store => ({ ...$store, schema, loading: true }))
 
     // Work out what sort type to use
     if (!sortColumn || !schema[sortColumn]) {
@@ -135,6 +133,7 @@ export const fetchTableData = opts => {
     }
 
     // Fetch next page
+    store.update($store => ({ ...$store, loading: true }))
     const page = await fetchPage(state.bookmarks[state.pageNumber + 1])
 
     // Update state
@@ -148,6 +147,7 @@ export const fetchTableData = opts => {
         pageNumber: pageNumber + 1,
         rows: page.rows,
         bookmarks,
+        loading: false,
       }
     })
   }
@@ -160,6 +160,7 @@ export const fetchTableData = opts => {
     }
 
     // Fetch previous page
+    store.update($store => ({ ...$store, loading: true }))
     const page = await fetchPage(state.bookmarks[state.pageNumber - 1])
 
     // Update state
@@ -168,6 +169,7 @@ export const fetchTableData = opts => {
         ...$store,
         pageNumber: $store.pageNumber - 1,
         rows: page.rows,
+        loading: false,
       }
     })
   }

packages/builder/src/pages/builder/app/[application]/data/index.svelte

@@ -1,6 +1,7 @@
 <script>
   import { goto } from "@roxi/routify"
   import { onMount } from "svelte"
+  import { admin } from "stores/portal"
   import CreateDatasourceModal from "components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte"
   import { datasources } from "stores/backend"
 
@@ -10,7 +11,7 @@
     $datasources.list.length > 1
 
   onMount(() => {
-    if (!setupComplete) {
+    if (!setupComplete && !$admin.isDev) {
       modal.show()
     } else {
       $goto("./table")

packages/builder/src/pages/builder/apps/index.svelte

@@ -34,6 +34,12 @@
   const publishedAppsOnly = app => app.status === AppStatus.DEPLOYED
 
   $: publishedApps = $apps.filter(publishedAppsOnly)
+
+  $: userApps = $auth.user?.builder?.global
+    ? publishedApps
+    : publishedApps.filter(app =>
+        Object.keys($auth.user?.roles).includes(app.prodId)
+      )
 </script>
 
 {#if $auth.user && loaded}
@@ -82,11 +88,11 @@
             </Body>
           </Layout>
           <Divider />
-          {#if publishedApps.length}
+          {#if userApps.length}
             <Heading>Apps</Heading>
             <div class="group">
               <Layout gap="S" noPadding>
-                {#each publishedApps as app, idx (app.appId)}
+                {#each userApps as app, idx (app.appId)}
                   <a class="app" target="_blank" href={`/${app.prodId}`}>
                     <div class="preview" use:gradient={{ seed: app.name }} />
                     <div class="app-info">

packages/builder/src/pages/builder/portal/manage/users/[userId].svelte

@@ -34,9 +34,13 @@
     role: {},
   }
 
-  $: defaultRoleId = $userFetch?.data?.builder?.global ? "ADMIN" : "BASIC"
+  const noRoleSchema = {
+    name: { displayName: "App" },
+  }
+
+  $: defaultRoleId = $userFetch?.data?.builder?.global ? "ADMIN" : ""
   // Merge the Apps list and the roles response to get something that makes sense for the table
-  $: appList = Object.keys($apps?.data).map(id => {
+  $: allAppList = Object.keys($apps?.data).map(id => {
     const roleId = $userFetch?.data?.roles?.[id] || defaultRoleId
     const role = $apps?.data?.[id].roles.find(role => role._id === roleId)
     return {
@@ -45,6 +49,15 @@
       role: [role],
     }
   })
+
+  $: appList = allAppList.filter(app => !!app.role[0])
+  $: noRoleAppList = allAppList
+    .filter(app => !app.role[0])
+    .map(app => {
+      delete app.role
+      return app
+    })
+
   let selectedApp
 
   const userFetch = fetchData(`/api/global/users/${userId}`)
@@ -173,6 +186,7 @@
   <Divider size="S" />
   <Layout gap="S" noPadding>
     <Heading size="S">Configure roles</Heading>
+    <Body>Specify a role to grant access to an app.</Body>
     <Table
       on:click={openUpdateRolesModal}
       schema={roleSchema}
@@ -183,6 +197,21 @@
       customRenderers={[{ column: "role", component: TagsRenderer }]}
     />
   </Layout>
+  <Layout gap="S" noPadding>
+    <Heading size="XS">No Access</Heading>
+    <Body
+      >Apps do not appear in the users portal. Public pages may still be viewed
+      if visited directly.</Body
+    >
+    <Table
+      on:click={openUpdateRolesModal}
+      schema={noRoleSchema}
+      data={noRoleAppList}
+      allowEditColumns={false}
+      allowEditRows={false}
+      allowSelectRows={false}
+    />
+  </Layout>
   <Divider size="S" />
   <Layout gap="XS" noPadding>
     <Heading size="S">Delete user</Heading>

packages/builder/src/pages/builder/portal/manage/users/_components/UpdateRolesModal.svelte

@@ -6,22 +6,38 @@
   export let app
   export let user
 
+  const NO_ACCESS = "NO_ACCESS"
+
   const dispatch = createEventDispatcher()
 
   const roles = app.roles
-  let options = roles
+  let options = roles.map(role => ({ value: role._id, label: role.name }))
-    .filter(role => role._id !== "PUBLIC")
+  options.push({ value: NO_ACCESS, label: "No Access" })
-    .map(role => ({ value: role._id, label: role.name }))
   let selectedRole = user?.roles?.[app?._id]
 
   async function updateUserRoles() {
-    const res = await users.save({
+    let res
+    if (selectedRole === NO_ACCESS) {
+      // remove the user role
+      const filteredRoles = { ...user.roles }
+      delete filteredRoles[app?._id]
+      res = await users.save({
+        ...user,
+        roles: {
+          ...filteredRoles,
+        },
+      })
+    } else {
+      // add the user role
+      res = await users.save({
         ...user,
         roles: {
           ...user.roles,
           [app._id]: selectedRole,
         },
       })
+    }
+
     if (res.status === 400) {
       notifications.error("Failed to update role")
     } else {

packages/builder/src/stores/portal/admin.js

@@ -7,6 +7,7 @@ export function createAdminStore() {
     loaded: false,
     multiTenancy: false,
     cloud: false,
+    isDev: false,
     disableAccountPortal: false,
     accountPortalUrl: "",
     importComplete: false,
@@ -62,6 +63,7 @@ export function createAdminStore() {
     let cloud = false
     let disableAccountPortal = false
     let accountPortalUrl = ""
+    let isDev = false
     try {
       const response = await api.get(`/api/system/environment`)
       const json = await response.json()
@@ -69,6 +71,7 @@ export function createAdminStore() {
       cloud = json.cloud
       disableAccountPortal = json.disableAccountPortal
       accountPortalUrl = json.accountPortalUrl
+      isDev = json.isDev
     } catch (err) {
       // just let it stay disabled
     }
@@ -77,6 +80,7 @@ export function createAdminStore() {
       store.cloud = cloud
       store.disableAccountPortal = disableAccountPortal
       store.accountPortalUrl = accountPortalUrl
+      store.isDev = isDev
       return store
     })
   }

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "0.9.167-alpha.1",
+  "version": "0.9.167-alpha.12",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/manifest.json

@@ -2386,7 +2386,7 @@
   },
   "dataprovider": {
     "name": "Data Provider",
-    "info": "Pagination is only available for data stored in internal tables.",
+    "info": "Pagination is only available for data stored in tables.",
     "icon": "Data",
     "illegalChildren": ["section"],
     "hasChildren": true,

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "0.9.167-alpha.1",
+  "version": "0.9.167-alpha.12",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.167-alpha.1",
+    "@budibase/bbui": "^0.9.167-alpha.12",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^0.9.167-alpha.1",
+    "@budibase/string-templates": "^0.9.167-alpha.12",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/client/src/components/CustomThemeWrapper.svelte

@@ -16,7 +16,10 @@
     /*  Buttons */
     --spectrum-semantic-cta-color-background-default: var(--primaryColor);
     --spectrum-semantic-cta-color-background-hover: var(--primaryColorHover);
+    --spectrum-button-primary-s-border-radius: var(--buttonBorderRadius);
     --spectrum-button-primary-m-border-radius: var(--buttonBorderRadius);
+    --spectrum-button-primary-l-border-radius: var(--buttonBorderRadius);
+    --spectrum-button-primary-xl-border-radius: var(--buttonBorderRadius);
 
     /* Loading spinners */
     --spectrum-progresscircle-medium-track-fill-color: var(--primaryColor);

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.167-alpha.1",
+  "version": "0.9.167-alpha.12",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -68,9 +68,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.167-alpha.1",
+    "@budibase/auth": "^0.9.167-alpha.12",
-    "@budibase/client": "^0.9.167-alpha.1",
+    "@budibase/client": "^0.9.167-alpha.12",
-    "@budibase/string-templates": "^0.9.167-alpha.1",
+    "@budibase/string-templates": "^0.9.167-alpha.12",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
@@ -119,6 +119,7 @@
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
+    "vm2": "^3.9.3",
     "yargs": "13.2.4",
     "zlib": "1.0.5"
   },

packages/server/src/api/controllers/dev.js

@@ -7,11 +7,13 @@ const { clearLock } = require("../../utilities/redis")
 const { Replication } = require("@budibase/auth").db
 const { DocumentTypes } = require("../../db/utils")
 
-async function redirect(ctx, method) {
+async function redirect(ctx, method, path = "global") {
   const { devPath } = ctx.params
   const queryString = ctx.originalUrl.split("?")[1] || ""
   const response = await fetch(
-    checkSlashesInUrl(`${env.WORKER_URL}/api/global/${devPath}?${queryString}`),
+    checkSlashesInUrl(
+      `${env.WORKER_URL}/api/${path}/${devPath}?${queryString}`
+    ),
     request(
       ctx,
       {
@@ -41,16 +43,22 @@ async function redirect(ctx, method) {
   ctx.cookies
 }
 
-exports.redirectGet = async ctx => {
+exports.buildRedirectGet = path => {
-  await redirect(ctx, "GET")
+  return async ctx => {
+    await redirect(ctx, "GET", path)
+  }
 }
 
-exports.redirectPost = async ctx => {
+exports.buildRedirectPost = path => {
-  await redirect(ctx, "POST")
+  return async ctx => {
+    await redirect(ctx, "POST", path)
+  }
 }
 
-exports.redirectDelete = async ctx => {
+exports.buildRedirectDelete = path => {
-  await redirect(ctx, "DELETE")
+  return async ctx => {
+    await redirect(ctx, "DELETE", path)
+  }
 }
 
 exports.clearLock = async ctx => {

packages/server/src/api/controllers/query.js

@@ -4,6 +4,7 @@ const { generateQueryID, getQueryParams } = require("../../db/utils")
 const { integrations } = require("../../integrations")
 const { BaseQueryVerbs } = require("../../constants")
 const env = require("../../environment")
+const ScriptRunner = require("../../utilities/scriptRunner")
 
 // simple function to append "readable" to all read queries
 function enrichQueries(input) {
@@ -28,12 +29,39 @@ function formatResponse(resp) {
       resp = { response: resp }
     }
   }
-  if (!Array.isArray(resp)) {
-    resp = [resp]
-  }
   return resp
 }
 
+async function runAndTransform(
+  integration,
+  queryVerb,
+  enrichedQuery,
+  transformer
+) {
+  let rows = formatResponse(await integration[queryVerb](enrichedQuery))
+
+  // transform as required
+  if (transformer) {
+    const runner = new ScriptRunner(transformer, { data: rows })
+    rows = runner.execute()
+  }
+
+  // needs to an array for next step
+  if (!Array.isArray(rows)) {
+    rows = [rows]
+  }
+
+  // map into JSON if just raw primitive here
+  if (rows.find(row => typeof row !== "object")) {
+    rows = rows.map(value => ({ value }))
+  }
+
+  // get all the potential fields in the schema
+  let keys = rows.flatMap(Object.keys)
+
+  return { rows, keys }
+}
+
 exports.fetch = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
@@ -122,15 +150,16 @@ exports.preview = async function (ctx) {
     ctx.throw(400, "Integration type does not exist.")
   }
 
-  const { fields, parameters, queryVerb } = ctx.request.body
+  const { fields, parameters, queryVerb, transformer } = ctx.request.body
-
   const enrichedQuery = await enrichQueryFields(fields, parameters)
-
   const integration = new Integration(datasource.config)
-  const rows = formatResponse(await integration[queryVerb](enrichedQuery))
 
-  // get all the potential fields in the schema
+  const { rows, keys } = await runAndTransform(
-  const keys = rows.flatMap(Object.keys)
+    integration,
+    queryVerb,
+    enrichedQuery,
+    transformer
+  )
 
   ctx.body = {
     rows,
@@ -158,10 +187,16 @@ exports.execute = async function (ctx) {
     query.fields,
     ctx.request.body.parameters
   )
-
   const integration = new Integration(datasource.config)
+
   // call the relevant CRUD method on the integration class
-  ctx.body = formatResponse(await integration[query.queryVerb](enrichedQuery))
+  const { rows } = await runAndTransform(
+    integration,
+    query.queryVerb,
+    enrichedQuery,
+    query.transformer
+  )
+  ctx.body = rows
   // cleanup
   if (integration.end) {
     integration.end()

packages/server/src/api/controllers/script.js

@@ -1,24 +1,9 @@
-const fetch = require("node-fetch")
+const ScriptRunner = require("../../utilities/scriptRunner")
-const vm = require("vm")
-
-class ScriptExecutor {
-  constructor(body) {
-    const code = `let fn = () => {\n${body.script}\n}; out = fn();`
-    this.script = new vm.Script(code)
-    this.context = vm.createContext(body.context)
-    this.context.fetch = fetch
-  }
-
-  execute() {
-    this.script.runInContext(this.context)
-    return this.context.out
-  }
-}
 
 exports.execute = async function (ctx) {
-  const executor = new ScriptExecutor(ctx.request.body)
+  const { script, context } = ctx.request.body
-
+  const runner = new ScriptRunner(script, context)
-  ctx.body = executor.execute()
+  ctx.body = runner.execute()
 }
 
 exports.save = async function (ctx) {

packages/server/src/api/controllers/table/index.js

@@ -91,6 +91,9 @@ exports.save = async function (ctx) {
     for (let propKey of Object.keys(tableToSave.schema)) {
       let column = tableToSave.schema[propKey]
       let oldColumn = oldTable.schema[propKey]
+      if (oldColumn && oldColumn.type === "internal") {
+        oldColumn.type = "auto"
+      }
       if (oldColumn && oldColumn.type !== column.type) {
         ctx.throw(400, "Cannot change the type of a column")
       }

packages/server/src/api/controllers/table/utils.js

@@ -1,5 +1,4 @@
 const CouchDB = require("../../../db")
-const linkRows = require("../../../db/linkedRows")
 const csvParser = require("../../../utilities/csvParser")
 const {
   getRowParams,
@@ -93,19 +92,10 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
         }
       }
 
-      // make sure link rows are up to date
+      finalData.push(row)
-      finalData.push(
-        linkRows.updateLinks({
-          appId,
-          eventType: linkRows.EventType.ROW_SAVE,
-          row,
-          tableId: row.tableId,
-          table,
-        })
-      )
     }
 
-    await db.bulkDocs(await Promise.all(finalData))
+    await db.bulkDocs(finalData)
     let response = await db.put(table)
     table._rev = response._rev
   }

packages/server/src/api/controllers/templates.js

@@ -7,11 +7,23 @@ const DEFAULT_TEMPLATES_BUCKET =
 
 exports.fetch = async function (ctx) {
   const { type = "app" } = ctx.query
-  const response = await fetch(
+  let response,
-    `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`
+    error = false
-  )
+  try {
+    response = await fetch(`https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`)
+    if (response.status !== 200) {
+      error = true
+    }
+  } catch (err) {
+    error = true
+  }
+  // if there is an error, simply return no templates
+  if (!error && response) {
     const json = await response.json()
     ctx.body = Object.values(json.templates[type])
+  } else {
+    ctx.body = []
+  }
 }
 
 // can't currently test this, have to ignore from coverage

packages/server/src/api/routes/dev.js

@@ -6,11 +6,16 @@ const { BUILDER } = require("@budibase/auth/permissions")
 
 const router = Router()
 
-if (env.isDev() || env.isTest()) {
+function redirectPath(path) {
   router
-    .get("/api/global/:devPath(.*)", controller.redirectGet)
+    .get(`/api/${path}/:devPath(.*)`, controller.buildRedirectGet(path))
-    .post("/api/global/:devPath(.*)", controller.redirectPost)
+    .post(`/api/${path}/:devPath(.*)`, controller.buildRedirectPost(path))
-    .delete("/api/global/:devPath(.*)", controller.redirectDelete)
+    .delete(`/api/${path}/:devPath(.*)`, controller.buildRedirectDelete(path))
+}
+
+if (env.isDev() || env.isTest()) {
+  redirectPath("global")
+  redirectPath("system")
 }
 
 router

packages/server/src/api/routes/query.js

@@ -31,7 +31,8 @@ function generateQueryValidation() {
     })),
     queryVerb: Joi.string().allow().required(),
     extra: Joi.object().optional(),
-    schema: Joi.object({}).required().unknown(true)
+    schema: Joi.object({}).required().unknown(true),
+    transformer: Joi.string().optional(),
   }))
 }
 
@@ -42,6 +43,7 @@ function generateQueryPreviewValidation() {
     queryVerb: Joi.string().allow().required(),
     extra: Joi.object().optional(),
     datasourceId: Joi.string().required(),
+    transformer: Joi.string().optional(),
     parameters: Joi.object({}).required().unknown(true)
   }))
 }

packages/server/src/automations/actions.js

@@ -3,7 +3,6 @@ const createRow = require("./steps/createRow")
 const updateRow = require("./steps/updateRow")
 const deleteRow = require("./steps/deleteRow")
 const executeScript = require("./steps/executeScript")
-const bash = require("./steps/bash")
 const executeQuery = require("./steps/executeQuery")
 const outgoingWebhook = require("./steps/outgoingWebhook")
 const serverLog = require("./steps/serverLog")
@@ -14,6 +13,7 @@ const integromat = require("./steps/integromat")
 let filter = require("./steps/filter")
 let delay = require("./steps/delay")
 let queryRow = require("./steps/queryRows")
+const env = require("../environment")
 
 const ACTION_IMPLS = {
   SEND_EMAIL_SMTP: sendSmtpEmail.run,
@@ -22,7 +22,6 @@ const ACTION_IMPLS = {
   DELETE_ROW: deleteRow.run,
   OUTGOING_WEBHOOK: outgoingWebhook.run,
   EXECUTE_SCRIPT: executeScript.run,
-  EXECUTE_BASH: bash.run,
   EXECUTE_QUERY: executeQuery.run,
   SERVER_LOG: serverLog.run,
   DELAY: delay.run,
@@ -42,7 +41,6 @@ const ACTION_DEFINITIONS = {
   OUTGOING_WEBHOOK: outgoingWebhook.definition,
   EXECUTE_SCRIPT: executeScript.definition,
   EXECUTE_QUERY: executeQuery.definition,
-  EXECUTE_BASH: bash.definition,
   SERVER_LOG: serverLog.definition,
   DELAY: delay.definition,
   FILTER: filter.definition,
@@ -54,6 +52,15 @@ const ACTION_DEFINITIONS = {
   integromat: integromat.definition,
 }
 
+// don't add the bash script/definitions unless in self host
+// the fact this isn't included in any definitions means it cannot be
+// ran at all
+if (env.SELF_HOSTED) {
+  const bash = require("./steps/bash")
+  ACTION_IMPLS["EXECUTE_BASH"] = bash.run
+  ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition
+}
+
 /* istanbul ignore next */
 exports.getAction = async function (actionName) {
   if (ACTION_IMPLS[actionName] != null) {

packages/server/src/automations/thread.js

@@ -10,6 +10,7 @@ const env = require("../environment")
 const usage = require("../utilities/usageQuota")
 
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
+const STOPPED_STATUS = { success: false, status: "STOPPED" }
 
 /**
  * The automation orchestrator is a class responsible for executing automations.
@@ -68,7 +69,13 @@ class Orchestrator {
   async execute() {
     let automation = this._automation
     const app = await this.getApp()
+    let stopped = false
     for (let step of automation.definition.steps) {
+      // execution stopped, record state for that
+      if (stopped) {
+        this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
+        continue
+      }
       let stepFn = await this.getStepFunctionality(step.stepId)
       step.inputs = await processObject(step.inputs, this._context)
       step.inputs = automationUtils.cleanInputValues(
@@ -86,10 +93,17 @@ class Orchestrator {
             context: this._context,
           })
         })
-        if (step.stepId === FILTER_STEP_ID && !outputs.success) {
-          break
-        }
         this._context.steps.push(outputs)
+        // if filter causes us to stop execution don't break the loop, set a var
+        // so that we can finish iterating through the steps and record that it stopped
+        if (step.stepId === FILTER_STEP_ID && !outputs.success) {
+          stopped = true
+          this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
+            ...outputs,
+            ...STOPPED_STATUS,
+          })
+          continue
+        }
         this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs)
       } catch (err) {
         console.error(`Automation error - ${step.stepId} - ${err}`)
@@ -99,7 +113,7 @@ class Orchestrator {
 
     // Increment quota for automation runs
     if (!env.SELF_HOSTED && !isDevAppID(this._appId)) {
-      usage.update(usage.Properties.AUTOMATION, 1)
+      await usage.update(usage.Properties.AUTOMATION, 1)
     }
     return this.executionOutput
   }

packages/server/src/constants/index.js

@@ -42,6 +42,7 @@ exports.FieldTypes = {
   FORMULA: "formula",
   AUTO: "auto",
   JSON: "json",
+  INTERNAL: "internal",
 }
 
 exports.RelationshipTypes = {

packages/server/src/integrations/base/sql.ts

@@ -98,7 +98,9 @@ function addFilters(
 }
 
 function addRelationships(
+  knex: Knex,
   query: KnexQuery,
+  fields: string | string[],
   fromTable: string,
   relationships: RelationshipsJson[] | undefined
 ): KnexQuery {
@@ -114,7 +116,7 @@ function addRelationships(
       query = query.leftJoin(
         toTable,
         `${fromTable}.${from}`,
-        `${relationship.tableName}.${to}`
+        `${toTable}.${to}`
       )
     } else {
       const throughTable = relationship.through
@@ -130,7 +132,7 @@ function addRelationships(
         .leftJoin(toTable, `${toTable}.${toPrimary}`, `${throughTable}.${to}`)
     }
   }
-  return query
+  return query.limit(BASE_LIMIT)
 }
 
 function buildCreate(
@@ -199,7 +201,7 @@ function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery {
     [tableName]: query,
   }).select(selectStatement)
   // handle joins
-  return addRelationships(preQuery, tableName, relationships)
+  return addRelationships(knex, preQuery, selectStatement, tableName, relationships)
 }
 
 function buildUpdate(

packages/server/src/integrations/tests/mongo.spec.js

@@ -27,7 +27,7 @@ describe("MongoDB Integration", () => {
     const body = {
       name: "Hello"
     }
-    const response = await config.integration.create({ 
+    await config.integration.create({
       index: indexName,
       json: body,
       extra: { collection: 'testCollection', actionTypes: 'insertOne'}
@@ -54,7 +54,7 @@ describe("MongoDB Integration", () => {
       },
       extra: { collection: 'testCollection', actionTypes: 'deleteOne'}
     }
-    const response = await config.integration.delete(query)
+    await config.integration.delete(query)
     expect(config.integration.client.deleteOne).toHaveBeenCalledWith(query.json)
   })
 
@@ -65,7 +65,7 @@ describe("MongoDB Integration", () => {
       },
       extra: { collection: 'testCollection', actionTypes: 'updateOne'}
     }
-    const response = await config.integration.update(query)
+    await config.integration.update(query)
     expect(config.integration.client.updateOne).toHaveBeenCalledWith(query.json)
   })
 
@@ -75,10 +75,14 @@ describe("MongoDB Integration", () => {
     const query = {
       extra: { collection: 'testCollection', actionTypes: 'deleteOne'}
     }
-    // Weird, need to do an IIFE for jest to recognize that it throws
-    expect(() => config.integration.read(query)()).toThrow(expect.any(Object))
 
+    let error = null
+    try {
+      await config.integration.read(query)
+    } catch (err) {
+      error = err
+    }
+    expect(error).toBeDefined()
     restore()
   })
-
 })

packages/server/src/middleware/currentapp.js

@@ -45,7 +45,7 @@ module.exports = async (ctx, next) => {
     const globalUser = await getCachedSelf(ctx, requestAppId)
     appId = requestAppId
     // retrieving global user gets the right role
-    roleId = globalUser.roleId || BUILTIN_ROLE_IDS.BASIC
+    roleId = globalUser.roleId || roleId
   }
 
   // nothing more to do

packages/server/src/middleware/tests/currentapp.spec.js

@@ -127,8 +127,8 @@ describe("Current app middleware", () => {
       } else {
         expect(cookieFn).not.toHaveBeenCalled()
       }
-      expect(config.ctx.roleId).toEqual("BASIC")
+      expect(config.ctx.roleId).toEqual("PUBLIC")
-      expect(config.ctx.user.role._id).toEqual("BASIC")
+      expect(config.ctx.user.role._id).toEqual("PUBLIC")
       expect(config.ctx.appId).toEqual("app_test")
       expect(config.next).toHaveBeenCalled()
     }
@@ -163,7 +163,7 @@ describe("Current app middleware", () => {
             return "app_test"
           },
           setCookie: jest.fn(),
-          getCookie: () => ({appId: "app_test", roleId: "BASIC"}),
+          getCookie: () => ({appId: "app_test", roleId: "PUBLIC"}),
         },
         constants: { Cookies: {} },
       }))

packages/server/src/utilities/global.js

@@ -26,7 +26,7 @@ exports.updateAppRole = (appId, user) => {
   if (!user.roleId && user.builder && user.builder.global) {
     user.roleId = BUILTIN_ROLE_IDS.ADMIN
   } else if (!user.roleId) {
-    user.roleId = BUILTIN_ROLE_IDS.BASIC
+    user.roleId = BUILTIN_ROLE_IDS.PUBLIC
   }
   delete user.roles
   return user

packages/server/src/utilities/rowProcessor/index.js

@@ -150,6 +150,10 @@ exports.processAutoColumn = processAutoColumn
  * @returns {object} The coerced value
  */
 exports.coerce = (row, type) => {
+  // no coercion specified for type, skip it
+  if (!TYPE_TRANSFORM_MAP[type]) {
+    return row
+  }
   // eslint-disable-next-line no-prototype-builtins
   if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(row)) {
     return TYPE_TRANSFORM_MAP[type][row]
@@ -196,6 +200,12 @@ exports.inputProcessing = (
       clonedRow[key] = exports.coerce(value, field.type)
     }
   }
+
+  if (!clonedRow._id || !clonedRow._rev) {
+    clonedRow._id = row._id
+    clonedRow._rev = row._rev
+  }
+
   // handle auto columns - this returns an object like {table, row}
   return processAutoColumn(user, copiedTable, clonedRow, opts)
 }

packages/server/src/utilities/scriptRunner.js

@@ -0,0 +1,21 @@
+const fetch = require("node-fetch")
+const { VM, VMScript } = require("vm2")
+
+class ScriptRunner {
+  constructor(script, context) {
+    const code = `let fn = () => {\n${script}\n}; results.out = fn();`
+    this.vm = new VM()
+    this.results = { out: "" }
+    this.vm.setGlobals(context)
+    this.vm.setGlobal("fetch", fetch)
+    this.vm.setGlobal("results", this.results)
+    this.script = new VMScript(code)
+  }
+
+  execute() {
+    this.vm.run(this.script)
+    return this.results.out
+  }
+}
+
+module.exports = ScriptRunner

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "0.9.167-alpha.1",
+  "version": "0.9.167-alpha.12",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",
@@ -24,7 +24,8 @@
     "dayjs": "^1.10.4",
     "handlebars": "^4.7.6",
     "handlebars-utils": "^1.0.6",
-    "lodash": "^4.17.20"
+    "lodash": "^4.17.20",
+    "vm2": "^3.9.4"
   },
   "devDependencies": {
     "@rollup/plugin-commonjs": "^17.1.0",

packages/string-templates/rollup.config.js

@@ -7,7 +7,15 @@ import globals from "rollup-plugin-node-globals"
 
 const production = !process.env.ROLLUP_WATCH
 
-const plugins = [
+export default [
+  {
+    input: "src/index.mjs",
+    output: {
+      sourcemap: !production,
+      format: "esm",
+      file: "./dist/bundle.mjs",
+    },
+    plugins: [
       resolve({
         preferBuiltins: true,
         browser: true,
@@ -17,28 +25,6 @@ const plugins = [
       builtins(),
       json(),
       production && terser(),
-]
+    ],
-
-export default [
-  {
-    input: "src/index.mjs",
-    output: {
-      sourcemap: !production,
-      format: "esm",
-      file: "./dist/bundle.mjs",
   },
-    plugins,
-  },
-  // This is the valid configuration for a CommonJS bundle, but since we have
-  // no use for this, it's better to leave it out.
-  // {
-  //   input: "src/index.cjs",
-  //   output: {
-  //     sourcemap: !production,
-  //     format: "cjs",
-  //     file: "./dist/bundle.cjs",
-  //     exports: "named",
-  //   },
-  //   plugins,
-  // },
 ]

packages/string-templates/src/helpers/Helper.js

@@ -1,13 +1,23 @@
 class Helper {
-  constructor(name, fn) {
+  constructor(name, fn, useValueFallback = true) {
     this.name = name
     this.fn = fn
+    this.useValueFallback = useValueFallback
   }
 
   register(handlebars) {
     // wrap the function so that no helper can cause handlebars to break
-    handlebars.registerHelper(this.name, value => {
+    handlebars.registerHelper(this.name, (value, info) => {
-      return this.fn(value) || value
+      let context = {}
+      if (info && info.data && info.data.root) {
+        context = info.data.root
+      }
+      const result = this.fn(value, context)
+      if (result == null) {
+        return this.useValueFallback ? value : null
+      } else {
+        return result
+      }
     })
   }
 

packages/string-templates/src/helpers/constants.js

@@ -19,6 +19,7 @@ module.exports.HelperFunctionNames = {
   OBJECT: "object",
   ALL: "all",
   LITERAL: "literal",
+  JS: "js",
 }
 
 module.exports.LITERAL_MARKER = "%LITERAL%"

packages/string-templates/src/helpers/index.js

@@ -1,6 +1,7 @@
 const Helper = require("./Helper")
 const { SafeString } = require("handlebars")
 const externalHandlebars = require("./external")
+const { processJS } = require("./javascript")
 const {
   HelperFunctionNames,
   HelperFunctionBuiltin,
@@ -17,6 +18,8 @@ const HELPERS = [
   new Helper(HelperFunctionNames.OBJECT, value => {
     return new SafeString(JSON.stringify(value))
   }),
+  // javascript helper
+  new Helper(HelperFunctionNames.JS, processJS, false),
   // this help is applied to all statements
   new Helper(HelperFunctionNames.ALL, value => {
     if (

packages/string-templates/src/helpers/javascript.js

@@ -0,0 +1,49 @@
+const { atob } = require("../utilities")
+
+// The method of executing JS scripts depends on the bundle being built.
+// This setter is used in the entrypoint (either index.cjs or index.mjs).
+let runJS
+module.exports.setJSRunner = runner => (runJS = runner)
+
+// Helper utility to strip square brackets from a value
+const removeSquareBrackets = value => {
+  if (!value || typeof value !== "string") {
+    return value
+  }
+  const regex = /\[+(.+)]+/
+  const matches = value.match(regex)
+  if (matches && matches[1]) {
+    return matches[1]
+  }
+  return value
+}
+
+// Our context getter function provided to JS code as $.
+// Extracts a value from context.
+const getContextValue = (path, context) => {
+  let data = context
+  path.split(".").forEach(key => {
+    if (data == null || typeof data !== "object") {
+      return null
+    }
+    data = data[removeSquareBrackets(key)]
+  })
+  return data
+}
+
+// Evaluates JS code against a certain context
+module.exports.processJS = (handlebars, context) => {
+  try {
+    // Wrap JS in a function and immediately invoke it.
+    // This is required to allow the final `return` statement to be valid.
+    const js = `function run(){${atob(handlebars)}};run();`
+
+    // Our $ context function gets a value from context
+    const sandboxContext = { $: path => getContextValue(path, context) }
+
+    // Create a sandbox with out context and run the JS
+    return runJS(js, sandboxContext)
+  } catch (error) {
+    return "Error while executing JS"
+  }
+}

packages/string-templates/src/index.cjs

@@ -1,161 +1,28 @@
-const handlebars = require("handlebars")
+const { VM } = require("vm2")
-const { registerAll } = require("./helpers/index")
+const templates = require("./index.js")
-const processors = require("./processors")
+const { setJSRunner } = require("./helpers/javascript")
-const { removeHandlebarsStatements } = require("./utilities")
-const manifest = require("../manifest.json")
-
-const hbsInstance = handlebars.create()
-registerAll(hbsInstance)
 
 /**
- * utility function to check if the object is valid
+ * CJS entrypoint for rollup
  */
-function testObject(object) {
+module.exports.isValid = templates.isValid
-  // JSON stringify will fail if there are any cycles, stops infinite recursion
+module.exports.makePropSafe = templates.makePropSafe
-  try {
+module.exports.getManifest = templates.getManifest
-    JSON.stringify(object)
+module.exports.isJSBinding = templates.isJSBinding
-  } catch (err) {
+module.exports.encodeJSBinding = templates.encodeJSBinding
-    throw "Unable to process inputs to JSON, cannot recurse"
+module.exports.decodeJSBinding = templates.decodeJSBinding
-  }
+module.exports.processStringSync = templates.processStringSync
-}
+module.exports.processObjectSync = templates.processObjectSync
+module.exports.processString = templates.processString
+module.exports.processObject = templates.processObject
 
 /**
- * Given an input object this will recurse through all props to try and update any handlebars statements within.
+ * Use vm2 to run JS scripts in a node env
- * @param {object|array} object The input structure which is to be recursed, it is important to note that
- * if the structure contains any cycles then this will fail.
- * @param {object} context The context that handlebars should fill data from.
- * @returns {Promise<object|array>} The structure input, as fully updated as possible.
  */
-module.exports.processObject = async (object, context) => {
+setJSRunner((js, context) => {
-  testObject(object)
+  const vm = new VM({
-  for (let key of Object.keys(object || {})) {
+    sandbox: context,
-    if (object[key] != null) {
+    timeout: 1000
-      let val = object[key]
-      if (typeof val === "string") {
-        object[key] = await module.exports.processString(object[key], context)
-      } else if (typeof val === "object") {
-        object[key] = await module.exports.processObject(object[key], context)
-      }
-    }
-  }
-  return object
-}
-
-/**
- * This will process a single handlebars containing string. If the string passed in has no valid handlebars statements
- * then nothing will occur.
- * @param {string} string The template string which is the filled from the context object.
- * @param {object} context An object of information which will be used to enrich the string.
- * @returns {Promise<string>} The enriched string, all templates should have been replaced if they can be.
- */
-module.exports.processString = async (string, context) => {
-  // TODO: carry out any async calls before carrying out async call
-  return module.exports.processStringSync(string, context)
-}
-
-/**
- * Given an input object this will recurse through all props to try and update any handlebars statements within. This is
- * a pure sync call and therefore does not have the full functionality of the async call.
- * @param {object|array} object The input structure which is to be recursed, it is important to note that
- * if the structure contains any cycles then this will fail.
- * @param {object} context The context that handlebars should fill data from.
- * @returns {object|array} The structure input, as fully updated as possible.
- */
-module.exports.processObjectSync = (object, context) => {
-  testObject(object)
-  for (let key of Object.keys(object || {})) {
-    let val = object[key]
-    if (typeof val === "string") {
-      object[key] = module.exports.processStringSync(object[key], context)
-    } else if (typeof val === "object") {
-      object[key] = module.exports.processObjectSync(object[key], context)
-    }
-  }
-  return object
-}
-
-/**
- * This will process a single handlebars containing string. If the string passed in has no valid handlebars statements
- * then nothing will occur. This is a pure sync call and therefore does not have the full functionality of the async call.
- * @param {string} string The template string which is the filled from the context object.
- * @param {object} context An object of information which will be used to enrich the string.
- * @returns {string} The enriched string, all templates should have been replaced if they can be.
- */
-module.exports.processStringSync = (string, context) => {
-  if (!exports.isValid(string)) {
-    return string
-  }
-  // take a copy of input incase error
-  const input = string
-  if (typeof string !== "string") {
-    throw "Cannot process non-string types."
-  }
-  try {
-    string = processors.preprocess(string)
-    // this does not throw an error when template can't be fulfilled, have to try correct beforehand
-    const template = hbsInstance.compile(string, {
-      strict: false,
   })
-    return processors.postprocess(template({
+  return vm.run(js)
-      now: new Date().toISOString(),
+})
-      ...context,
-    }))
-  } catch (err) {
-    return removeHandlebarsStatements(input)
-  }
-}
-
-/**
- * Simple utility function which makes sure that a templating property has been wrapped in literal specifiers correctly.
- * @param {string} property The property which is to be wrapped.
- * @returns {string} The wrapped property ready to be added to a templating string.
- */
-module.exports.makePropSafe = property => {
-  return `[${property}]`.replace("[[", "[").replace("]]", "]")
-}
-
-/**
- * Checks whether or not a template string contains totally valid syntax (simply tries running it)
- * @param string The string to test for valid syntax - this may contain no templates and will be considered valid.
- * @returns {boolean} Whether or not the input string is valid.
- */
-module.exports.isValid = string => {
-  const validCases = [
-    "string",
-    "number",
-    "object",
-    "array",
-    "cannot read property",
-    "undefined",
-  ]
-  // this is a portion of a specific string always output by handlebars in the case of a syntax error
-  const invalidCases = [`expecting '`]
-  // don't really need a real context to check if its valid
-  const context = {}
-  try {
-    hbsInstance.compile(processors.preprocess(string, false))(context)
-    return true
-  } catch (err) {
-    const msg = err && err.message ? err.message : err
-    if (!msg) {
-      return false
-    }
-    const invalidCase = invalidCases.some(invalidCase =>
-      msg.toLowerCase().includes(invalidCase)
-    )
-    const validCase = validCases.some(validCase =>
-      msg.toLowerCase().includes(validCase)
-    )
-    // special case for maths functions - don't have inputs yet
-    return validCase && !invalidCase
-  }
-}
-
-/**
- * We have generated a static manifest file from the helpers that this string templating package makes use of.
- * This manifest provides information about each of the helpers and how it can be used.
- * @returns The manifest JSON which has been generated from the helpers.
- */
-module.exports.getManifest = () => {
-  return manifest
-}

packages/string-templates/src/index.js

@@ -0,0 +1,204 @@
+const handlebars = require("handlebars")
+const { registerAll } = require("./helpers/index")
+const processors = require("./processors")
+const { removeHandlebarsStatements, atob, btoa } = require("./utilities")
+const manifest = require("../manifest.json")
+
+const hbsInstance = handlebars.create()
+registerAll(hbsInstance)
+
+/**
+ * utility function to check if the object is valid
+ */
+function testObject(object) {
+  // JSON stringify will fail if there are any cycles, stops infinite recursion
+  try {
+    JSON.stringify(object)
+  } catch (err) {
+    throw "Unable to process inputs to JSON, cannot recurse"
+  }
+}
+
+/**
+ * Given an input object this will recurse through all props to try and update any handlebars statements within.
+ * @param {object|array} object The input structure which is to be recursed, it is important to note that
+ * if the structure contains any cycles then this will fail.
+ * @param {object} context The context that handlebars should fill data from.
+ * @returns {Promise<object|array>} The structure input, as fully updated as possible.
+ */
+module.exports.processObject = async (object, context) => {
+  testObject(object)
+  for (let key of Object.keys(object || {})) {
+    if (object[key] != null) {
+      let val = object[key]
+      if (typeof val === "string") {
+        object[key] = await module.exports.processString(object[key], context)
+      } else if (typeof val === "object") {
+        object[key] = await module.exports.processObject(object[key], context)
+      }
+    }
+  }
+  return object
+}
+
+/**
+ * This will process a single handlebars containing string. If the string passed in has no valid handlebars statements
+ * then nothing will occur.
+ * @param {string} string The template string which is the filled from the context object.
+ * @param {object} context An object of information which will be used to enrich the string.
+ * @returns {Promise<string>} The enriched string, all templates should have been replaced if they can be.
+ */
+module.exports.processString = async (string, context) => {
+  // TODO: carry out any async calls before carrying out async call
+  return module.exports.processStringSync(string, context)
+}
+
+/**
+ * Given an input object this will recurse through all props to try and update any handlebars statements within. This is
+ * a pure sync call and therefore does not have the full functionality of the async call.
+ * @param {object|array} object The input structure which is to be recursed, it is important to note that
+ * if the structure contains any cycles then this will fail.
+ * @param {object} context The context that handlebars should fill data from.
+ * @returns {object|array} The structure input, as fully updated as possible.
+ */
+module.exports.processObjectSync = (object, context) => {
+  testObject(object)
+  for (let key of Object.keys(object || {})) {
+    let val = object[key]
+    if (typeof val === "string") {
+      object[key] = module.exports.processStringSync(object[key], context)
+    } else if (typeof val === "object") {
+      object[key] = module.exports.processObjectSync(object[key], context)
+    }
+  }
+  return object
+}
+
+/**
+ * This will process a single handlebars containing string. If the string passed in has no valid handlebars statements
+ * then nothing will occur. This is a pure sync call and therefore does not have the full functionality of the async call.
+ * @param {string} string The template string which is the filled from the context object.
+ * @param {object} context An object of information which will be used to enrich the string.
+ * @returns {string} The enriched string, all templates should have been replaced if they can be.
+ */
+module.exports.processStringSync = (string, context) => {
+  if (!exports.isValid(string)) {
+    return string
+  }
+  // take a copy of input incase error
+  const input = string
+  if (typeof string !== "string") {
+    throw "Cannot process non-string types."
+  }
+  try {
+    string = processors.preprocess(string)
+    // this does not throw an error when template can't be fulfilled, have to try correct beforehand
+    const template = hbsInstance.compile(string, {
+      strict: false,
+    })
+    return processors.postprocess(
+      template({
+        now: new Date().toISOString(),
+        ...context,
+      })
+    )
+  } catch (err) {
+    return removeHandlebarsStatements(input)
+  }
+}
+
+/**
+ * Simple utility function which makes sure that a templating property has been wrapped in literal specifiers correctly.
+ * @param {string} property The property which is to be wrapped.
+ * @returns {string} The wrapped property ready to be added to a templating string.
+ */
+module.exports.makePropSafe = property => {
+  return `[${property}]`.replace("[[", "[").replace("]]", "]")
+}
+
+/**
+ * Checks whether or not a template string contains totally valid syntax (simply tries running it)
+ * @param string The string to test for valid syntax - this may contain no templates and will be considered valid.
+ * @returns {boolean} Whether or not the input string is valid.
+ */
+module.exports.isValid = string => {
+  const validCases = [
+    "string",
+    "number",
+    "object",
+    "array",
+    "cannot read property",
+    "undefined",
+  ]
+  // this is a portion of a specific string always output by handlebars in the case of a syntax error
+  const invalidCases = [`expecting '`]
+  // don't really need a real context to check if its valid
+  const context = {}
+  try {
+    hbsInstance.compile(processors.preprocess(string, false))(context)
+    return true
+  } catch (err) {
+    const msg = err && err.message ? err.message : err
+    if (!msg) {
+      return false
+    }
+    const invalidCase = invalidCases.some(invalidCase =>
+      msg.toLowerCase().includes(invalidCase)
+    )
+    const validCase = validCases.some(validCase =>
+      msg.toLowerCase().includes(validCase)
+    )
+    // special case for maths functions - don't have inputs yet
+    return validCase && !invalidCase
+  }
+}
+
+/**
+ * We have generated a static manifest file from the helpers that this string templating package makes use of.
+ * This manifest provides information about each of the helpers and how it can be used.
+ * @returns The manifest JSON which has been generated from the helpers.
+ */
+module.exports.getManifest = () => {
+  return manifest
+}
+
+/**
+ * Checks if a HBS expression is a valid JS HBS expression
+ * @param handlebars the HBS expression to check
+ * @returns {boolean} whether the expression is JS or not
+ */
+module.exports.isJSBinding = handlebars => {
+  return module.exports.decodeJSBinding(handlebars) != null
+}
+
+/**
+ * Encodes a raw JS string as a JS HBS expression
+ * @param javascript the JS code to encode
+ * @returns {string} the JS HBS expression
+ */
+module.exports.encodeJSBinding = javascript => {
+  return `{{ js "${btoa(javascript)}" }}`
+}
+
+/**
+ * Decodes a JS HBS expression to the raw JS code
+ * @param handlebars the JS HBS expression
+ * @returns {string|null} the raw JS code
+ */
+module.exports.decodeJSBinding = handlebars => {
+  if (!handlebars || typeof handlebars !== "string") {
+    return null
+  }
+
+  // JS is only valid if it is the only HBS expression
+  if (!handlebars.trim().startsWith("{{ js ")) {
+    return null
+  }
+
+  const captureJSRegex = new RegExp(/{{ js "(.*)" }}/)
+  const match = handlebars.match(captureJSRegex)
+  if (!match || match.length < 2) {
+    return null
+  }
+  return atob(match[1])
+}

packages/string-templates/src/index.mjs

@@ -1,12 +1,31 @@
-import templates from "./index.cjs"
+import vm from "vm"
+import templates from "./index.js"
+import { setJSRunner } from "./helpers/javascript"
 
 /**
- * This file is simply an entrypoint for rollup - makes a lot of cjs problems go away
+ * ES6 entrypoint for rollup
  */
 export const isValid = templates.isValid
 export const makePropSafe = templates.makePropSafe
 export const getManifest = templates.getManifest
+export const isJSBinding = templates.isJSBinding
+export const encodeJSBinding = templates.encodeJSBinding
+export const decodeJSBinding = templates.decodeJSBinding
 export const processStringSync = templates.processStringSync
 export const processObjectSync = templates.processObjectSync
 export const processString = templates.processString
 export const processObject = templates.processObject
+
+/**
+ * Use polyfilled vm to run JS scripts in a browser Env
+ */
+setJSRunner((js, context) => {
+  context = {
+    ...context,
+    alert: undefined,
+    setInterval: undefined,
+    setTimeout: undefined,
+  }
+  vm.createContext(context)
+  return vm.runInNewContext(js, context, { timeout: 1000 })
+})

packages/string-templates/src/utilities.js

@@ -22,3 +22,11 @@ module.exports.removeHandlebarsStatements = string => {
   }
   return string
 }
+
+module.exports.btoa = plainText => {
+  return Buffer.from(plainText, "utf-8").toString("base64")
+}
+
+module.exports.atob = base64 => {
+  return Buffer.from(base64, "base64").toString("utf-8")
+}

packages/string-templates/test/javascript.spec.js

@@ -0,0 +1,85 @@
+const { processStringSync, encodeJSBinding } = require("../src/index.cjs")
+
+const processJS = (js, context) => {
+  return processStringSync(encodeJSBinding(js), context)
+}
+
+describe("Test the JavaScript helper", () => {
+  it("should execute a simple expression", () => {
+    const output = processJS(`return 1 + 2`)
+    expect(output).toBe("3")
+  })
+
+  it("should be able to use primitive bindings", () => {
+    const output = processJS(`return $("foo")`, {
+      foo: "bar",
+    })
+    expect(output).toBe("bar")
+  })
+
+  it("should be able to use an object binding", () => {
+    const output = processJS(`return $("foo").bar`, {
+      foo: {
+        bar: "baz",
+      },
+    })
+    expect(output).toBe("baz")
+  })
+
+  it("should be able to use a complex object binding", () => {
+    const output = processJS(`return $("foo").bar[0].baz`, {
+      foo: {
+        bar: [
+          {
+            baz: "shazbat",
+          },
+        ],
+      },
+    })
+    expect(output).toBe("shazbat")
+  })
+
+  it("should be able to use a deep binding", () => {
+    const output = processJS(`return $("foo.bar.baz")`, {
+      foo: {
+        bar: {
+          baz: "shazbat",
+        },
+      },
+    })
+    expect(output).toBe("shazbat")
+  })
+
+  it("should be able to use a deep array binding", () => {
+    const output = processJS(`return $("foo.0.bar")`, {
+      foo: [
+        {
+          bar: "baz",
+        },
+      ],
+    })
+    expect(output).toBe("baz")
+  })
+
+  it("should handle errors", () => {
+    const output = processJS(`throw "Error"`)
+    expect(output).toBe("Error while executing JS")
+  })
+
+  it("should timeout after one second", () => {
+    const output = processJS(`while (true) {}`)
+    expect(output).toBe("Error while executing JS")
+  })
+
+  it("should prevent access to the process global", () => {
+    const output = processJS(`return process`)
+    expect(output).toBe("Error while executing JS")
+  })
+
+  it("should prevent sandbox escape", () => {
+    const output = processJS(
+      `return this.constructor.constructor("return process")()`
+    )
+    expect(output).toBe("Error while executing JS")
+  })
+})

packages/string-templates/yarn.lock

@@ -4572,6 +4572,11 @@ vlq@^0.2.2:
   resolved "https://registry.yarnpkg.com/vlq/-/vlq-0.2.3.tgz#8f3e4328cf63b1540c0d67e1b2778386f8975b26"
   integrity sha512-DRibZL6DsNhIgYQ+wNdWDL2SL3bKPlVrRiBqV5yuMm++op8W4kGFtaQfCs4KEJn0wBZcHVHJ3eoywX8983k1ow==
 
+vm2@^3.9.4:
+  version "3.9.4"
+  resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.4.tgz#2e118290fefe7bd8ea09ebe2f5faf53730dbddaa"
+  integrity sha512-sOdharrJ7KEePIpHekiWaY1DwgueuiBeX/ZBJUPgETsVlJsXuEx0K0/naATq2haFvJrvZnRiORQRubR0b7Ye6g==
+
 w3c-hr-time@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz#0a89cdf5cc15822df9c360543676963e0cc308cd"

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "0.9.167-alpha.1",
+  "version": "0.9.167-alpha.12",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -29,8 +29,8 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.167-alpha.1",
+    "@budibase/auth": "^0.9.167-alpha.12",
-    "@budibase/string-templates": "^0.9.167-alpha.1",
+    "@budibase/string-templates": "^0.9.167-alpha.12",
     "@koa/router": "^8.0.0",
     "@techpass/passport-openidconnect": "^0.3.0",
     "aws-sdk": "^2.811.0",

packages/worker/src/api/controllers/system/environment.js

@@ -6,5 +6,6 @@ exports.fetch = async ctx => {
     cloud: !env.SELF_HOSTED,
     accountPortalUrl: env.ACCOUNT_PORTAL_URL,
     disableAccountPortal: env.DISABLE_ACCOUNT_PORTAL,
+    isDev: env.isDev(),
   }
 }