Merge pull request #1430 from Budibase/labday/mike-formulas

Handlebars formulas

packages/bbui/src/Drawer/DrawerContent.svelte

@@ -28,11 +28,9 @@
     border-right: var(--border-light);
     overflow: auto;
   }
-
   .sidebar::-webkit-scrollbar {
     display: none;
   }
-
   .main {
     font-family: var(--font-sans);
   }

packages/bbui/src/Form/Core/TextArea.svelte

@@ -37,6 +37,7 @@
       <use xlink:href="#spectrum-icon-18-Alert" />
     </svg>
   {/if}
+  <!-- prettier-ignore -->
   <textarea
     bind:this={textarea}
     placeholder={placeholder || ""}
@@ -45,9 +46,7 @@
     {id}
     on:focus={() => (focus = true)}
     on:blur={onChange}
-  >
-    {value || ""}
-  </textarea>
+  >{value || ""}</textarea>
 </div>
 
 <style>

packages/bbui/src/Modal/ModalContent.svelte

@@ -83,6 +83,7 @@
   .spectrum-Dialog--extraLarge {
     width: 1000px;
   }
+
   .content-grid {
     display: grid;
     position: relative;

packages/builder/src/builderStore/dataBinding.js

@@ -16,7 +16,7 @@ const CAPTURE_HBS_TEMPLATE = /{{[\S\s]*?}}/g
 export const getBindableProperties = (asset, componentId) => {
   const contextBindings = getContextBindings(asset, componentId)
   const userBindings = getUserBindings()
-  const urlBindings = getUrlBindings(asset, componentId)
+  const urlBindings = getUrlBindings(asset)
   return [...contextBindings, ...userBindings, ...urlBindings]
 }
 
@@ -338,6 +338,29 @@ export function removeBindings(obj) {
   return obj
 }
 
+/**
+ * When converting from readable to runtime it can sometimes add too many square brackets,
+ * this makes sure that doesn't happen.
+ */
+function shouldReplaceBinding(currentValue, from, convertTo) {
+  if (!currentValue?.includes(from)) {
+    return false
+  }
+  if (convertTo === "readableBinding") {
+    return true
+  }
+  // remove all the spaces, if the input is surrounded by spaces e.g. [ Auto ID ] then
+  // this makes sure it is detected
+  const noSpaces = currentValue.replace(/\s+/g, "")
+  const fromNoSpaces = from.replace(/\s+/g, "")
+  const invalids = [
+    `[${fromNoSpaces}]`,
+    `"${fromNoSpaces}"`,
+    `'${fromNoSpaces}'`,
+  ]
+  return !invalids.find(invalid => noSpaces?.includes(invalid))
+}
+
 /**
  * utility function for the readableToRuntimeBinding and runtimeToReadableBinding.
  */
@@ -357,7 +380,7 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
   for (let boundValue of boundValues) {
     let newBoundValue = boundValue
     for (let from of convertFromProps) {
-      if (newBoundValue.includes(from)) {
+      if (shouldReplaceBinding(newBoundValue, from, convertTo)) {
         const binding = bindableProperties.find(el => el[convertFrom] === from)
         newBoundValue = newBoundValue.replace(from, binding[convertTo])
       }

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -1,16 +1,12 @@
 <script>
   import TableSelector from "./TableSelector.svelte"
   import RowSelector from "./RowSelector.svelte"
-  import QuerySelector from "./QuerySelector.svelte"
   import SchemaSetup from "./SchemaSetup.svelte"
-  import QueryParamSelector from "./QueryParamSelector.svelte"
   import { Button, Input, Select, Label } from "@budibase/bbui"
   import { automationStore } from "builderStore"
   import WebhookDisplay from "../Shared/WebhookDisplay.svelte"
-  import DrawerBindableInput from "../../common/DrawerBindableInput.svelte"
-  import AutomationBindingPanel from "./AutomationBindingPanel.svelte"
-  import Editor from "components/integration/QueryEditor.svelte"
-  import CodeEditorModal from "./CodeEditorModal.svelte"
+  import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
+  import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
 
   export let block
   export let webhookModal

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -1,8 +1,8 @@
 <script>
   import { tables } from "stores/backend"
   import { Select } from "@budibase/bbui"
-  import DrawerBindableInput from "../../common/DrawerBindableInput.svelte"
-  import AutomationBindingPanel from "./AutomationBindingPanel.svelte"
+  import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
+  import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
 
   export let value
   export let bindings

packages/builder/src/components/backend/DataTable/formula.js

@@ -0,0 +1,75 @@
+import { FIELDS } from "constants/backend"
+import { tables } from "stores/backend"
+import { get as svelteGet } from "svelte/store"
+
+// currently supported level of relationship depth (server side)
+const MAX_DEPTH = 1
+const TYPES_TO_SKIP = [
+  FIELDS.FORMULA.type,
+  FIELDS.LONGFORM.type,
+  FIELDS.ATTACHMENT.type,
+]
+
+export function getBindings({
+  table,
+  path = null,
+  category = null,
+  depth = 0,
+}) {
+  let bindings = []
+  if (!table) {
+    return bindings
+  }
+  for (let [column, schema] of Object.entries(table.schema)) {
+    const isRelationship = schema.type === FIELDS.LINK.type
+    // skip relationships after a certain depth and types which
+    // can't bind to
+    if (
+      TYPES_TO_SKIP.includes(schema.type) ||
+      (isRelationship && depth >= MAX_DEPTH)
+    ) {
+      continue
+    }
+    category = category == null ? `${table.name} Fields` : category
+    if (isRelationship && depth < MAX_DEPTH) {
+      const relatedTable = svelteGet(tables).list.find(
+        table => table._id === schema.tableId
+      )
+      const relatedBindings = bindings.concat(
+        getBindings({
+          table: relatedTable,
+          path: column,
+          category: `${column} Relationships`,
+          depth: depth + 1,
+        })
+      )
+      // remove the ones that have already been found
+      bindings = bindings.concat(
+        relatedBindings.filter(
+          related => !bindings.find(binding => binding.path === related.path)
+        )
+      )
+    }
+    const field = Object.values(FIELDS).find(
+      field => field.type === schema.type
+    )
+    const label = path == null ? column : `${path}.0.${column}`
+    // only supply a description for relationship paths
+    const description =
+      path == null
+        ? undefined
+        : `Update the "0" with the index of relationships or use array helpers`
+    bindings.push({
+      label: label,
+      type: field.name === FIELDS.LINK.name ? "Array" : field.name,
+      category,
+      path: label,
+      description,
+      // don't include path, it messes things up, relationship path
+      // will be replaced by the main array binding
+      readableBinding: column,
+      runtimeBinding: `[${column}]`,
+    })
+  }
+  return bindings
+}

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -23,9 +23,12 @@
   import ValuesList from "components/common/ValuesList.svelte"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
   import { truncate } from "lodash"
+  import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
+  import { getBindings } from "components/backend/DataTable/formula"
   import { getContext } from "svelte"
 
-  const AUTO_COL = "auto"
+  const AUTO_TYPE = "auto"
+  const FORMULA_TYPE = FIELDS.FORMULA.type
   const LINK_TYPE = FIELDS.LINK.type
   let fieldDefinitions = cloneDeep(FIELDS)
   const { hide } = getContext(Context.Modal)
@@ -67,14 +70,18 @@
   $: canBeSearched =
     field.type !== LINK_TYPE &&
     field.subtype !== AUTO_COLUMN_SUB_TYPES.CREATED_BY &&
-    field.subtype !== AUTO_COLUMN_SUB_TYPES.UPDATED_BY
-  $: canBeDisplay = field.type !== LINK_TYPE && field.type !== AUTO_COL
+    field.subtype !== AUTO_COLUMN_SUB_TYPES.UPDATED_BY &&
+    field.type !== FORMULA_TYPE
+  $: canBeDisplay =
+    field.type !== LINK_TYPE &&
+    field.type !== AUTO_TYPE &&
+    field.type !== FORMULA_TYPE
   $: canBeRequired =
-    field.type !== LINK_TYPE && !uneditable && field.type !== AUTO_COL
+    field.type !== LINK_TYPE && !uneditable && field.type !== AUTO_TYPE
   $: relationshipOptions = getRelationshipOptions(field)
 
   async function saveColumn() {
-    if (field.type === AUTO_COL) {
+    if (field.type === AUTO_TYPE) {
       field = buildAutoColumn($tables.draft.name, field.name, field.subtype)
     }
     tables.saveField({
@@ -195,7 +202,7 @@
     on:change={handleTypeChange}
     options={[
       ...Object.values(fieldDefinitions),
-      { name: "Auto Column", type: AUTO_COL },
+      { name: "Auto Column", type: AUTO_TYPE },
     ]}
     getOptionLabel={field => field.name}
     getOptionValue={field => field.type}
@@ -288,7 +295,16 @@
       />
     {/if}
     <Input label={`Column name in other table`} bind:value={field.fieldName} />
-  {:else if field.type === AUTO_COL}
+  {:else if field.type === FORMULA_TYPE}
+    <ModalBindableInput
+      title="Handlebars Formula"
+      label="Formula"
+      value={field.formula}
+      on:change={e => (field.formula = e.detail)}
+      bindings={getBindings({ table })}
+      serverSide="true"
+    />
+  {:else if field.type === AUTO_TYPE}
     <Select
       label="Auto Column Type"
       value={field.subtype}

packages/builder/src/components/backend/DataTable/modals/CreateEditRow.svelte

@@ -5,6 +5,9 @@
   import * as api from "../api"
   import { ModalContent } from "@budibase/bbui"
   import ErrorsBox from "components/common/ErrorsBox.svelte"
+  import { FIELDS } from "constants/backend"
+
+  const FORMULA_TYPE = FIELDS.FORMULA.type
 
   export let row = {}
 
@@ -45,7 +48,7 @@
 >
   <ErrorsBox {errors} />
   {#each tableSchema as [key, meta]}
-    {#if !meta.autocolumn}
+    {#if !meta.autocolumn && meta.type !== FORMULA_TYPE}
       <div>
         <RowFieldControl {meta} bind:value={row[key]} />
       </div>

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -16,6 +16,7 @@
   } from "builderStore/dataBinding"
   import { currentAsset, store } from "../../../builderStore"
   import { handlebarsCompletions } from "constants/completions"
+  import { addToText } from "./utils"
 
   const dispatch = createEventDispatcher()
 
@@ -44,20 +45,6 @@
     valid = isValid(runtimeBinding)
   }
 
-  function addToText(readableBinding) {
-    const position = getCaretPosition()
-    const toAdd = `{{ ${readableBinding} }}`
-
-    if (position.start) {
-      value =
-        value.substring(0, position.start) +
-        toAdd +
-        value.substring(position.end, value.length)
-    } else {
-      value = toAdd
-    }
-  }
-
   export function cancel() {
     dispatch("update", originalValue)
     bindingDrawer.close()
@@ -75,7 +62,11 @@
             {#each context.filter(context =>
               context.readableBinding.match(searchRgx)
             ) as { readableBinding }}
-              <li on:click={() => addToText(readableBinding)}>
+              <li
+                on:click={() => {
+                  value = addToText(value, getCaretPosition(), readableBinding)
+                }}
+              >
                 {readableBinding}
               </li>
             {/each}
@@ -100,7 +91,11 @@
         <Heading size="XS">Helpers</Heading>
         <ul>
           {#each helpers.filter(helper => helper.label.match(searchRgx) || helper.description.match(searchRgx)) as helper}
-            <li on:click={() => addToText(helper.text)}>
+            <li
+              on:click={() => {
+                value = addToText(value, getCaretPosition(), helper.text)
+              }}
+            >
               <div>
                 <Label extraSmall>{helper.displayText}</Label>
                 <div class="description">
@@ -134,13 +129,17 @@
   .main {
     padding: var(--spacing-m);
   }
+
+  .main :global(textarea) {
+    min-height: 150px !important;
+  }
+
   section {
     display: grid;
     grid-gap: var(--spacing-s);
   }
   ul {
     list-style: none;
-    padding-left: 0;
     margin: 0;
     padding: 0;
   }

packages/builder/src/components/common/bindings/DrawerBindableInput.svelte

@@ -4,7 +4,7 @@
     readableToRuntimeBinding,
     runtimeToReadableBinding,
   } from "builderStore/dataBinding"
-  import BindingPanel from "components/design/PropertiesPanel/BindingPanel.svelte"
+  import BindingPanel from "components/common/bindings/BindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
 
   export let panel = BindingPanel

packages/builder/src/components/common/bindings/ModalBindableInput.svelte

@@ -0,0 +1,103 @@
+<script>
+  import { Icon, Input, Modal, Body, ModalContent } from "@budibase/bbui"
+  import {
+    readableToRuntimeBinding,
+    runtimeToReadableBinding,
+  } from "builderStore/dataBinding"
+  import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
+  import { createEventDispatcher } from "svelte"
+  const dispatch = createEventDispatcher()
+
+  export let panel = ServerBindingPanel
+  export let value = ""
+  export let bindings = []
+  export let thin = true
+  export let title = "Bindings"
+  export let placeholder
+  export let label
+
+  let bindingModal
+  let validity = true
+
+  $: readableValue = runtimeToReadableBinding(bindings, value)
+  $: tempValue = readableValue
+  $: invalid = !validity
+
+  const saveBinding = () => {
+    onChange(tempValue)
+    bindingModal.hide()
+  }
+
+  const onChange = input => {
+    dispatch("change", readableToRuntimeBinding(bindings, input))
+  }
+</script>
+
+<div class="control">
+  <Input
+    {label}
+    {thin}
+    value={readableValue}
+    on:change={event => onChange(event.target.value)}
+    {placeholder}
+  />
+  <div class="icon" on:click={bindingModal.show}>
+    <Icon size="S" name="FlashOn" />
+  </div>
+</div>
+<Modal bind:this={bindingModal}>
+  <ModalContent
+    {title}
+    onConfirm={saveBinding}
+    bind:disabled={invalid}
+    size="XL"
+  >
+    <Body extraSmall grey>
+      Add the objects on the left to enrich your text.
+    </Body>
+    <svelte:component
+      this={panel}
+      serverSide
+      value={readableValue}
+      bind:validity
+      on:update={event => (tempValue = event.detail)}
+      bindableProperties={bindings}
+    />
+  </ModalContent>
+</Modal>
+
+<style>
+  .control {
+    flex: 1;
+    position: relative;
+  }
+
+  .icon {
+    right: 1px;
+    bottom: 1px;
+    position: absolute;
+    justify-content: center;
+    align-items: center;
+    display: flex;
+    flex-direction: row;
+    box-sizing: border-box;
+    border-left: 1px solid var(--spectrum-alias-border-color);
+    border-top-right-radius: var(--spectrum-alias-border-radius-regular);
+    border-bottom-right-radius: var(--spectrum-alias-border-radius-regular);
+    width: 31px;
+    color: var(--spectrum-alias-text-color);
+    background-color: var(--spectrum-global-color-gray-75);
+    transition: background-color
+        var(--spectrum-global-animation-duration-100, 130ms),
+      box-shadow var(--spectrum-global-animation-duration-100, 130ms),
+      border-color var(--spectrum-global-animation-duration-100, 130ms);
+    height: calc(var(--spectrum-alias-item-height-m) - 2px);
+  }
+
+  .icon:hover {
+    cursor: pointer;
+    color: var(--spectrum-alias-text-color-hover);
+    background-color: var(--spectrum-global-color-gray-50);
+    border-color: var(--spectrum-alias-border-color-hover);
+  }
+</style>

packages/builder/src/components/common/bindings/ServerBindingPanel.svelte

@@ -10,18 +10,20 @@
   import { createEventDispatcher } from "svelte"
   import { isValid } from "@budibase/string-templates"
   import { handlebarsCompletions } from "constants/completions"
+  import { readableToRuntimeBinding } from "../../../builderStore/dataBinding"
+  import { addToText } from "./utils"
 
   const dispatch = createEventDispatcher()
 
-  export let value = ""
-  export let bindingDrawer
+  export let bindingContainer
   export let bindableProperties = []
+  export let validity = true
+  export let value = ""
 
-  let originalValue = value
+  let hasReadable = bindableProperties[0].readableBinding != null
   let helpers = handlebarsCompletions()
   let getCaretPosition
   let search = ""
-  let validity = true
 
   $: categories = Object.entries(groupBy("category", bindableProperties))
   $: value && checkValid()
@@ -29,25 +31,13 @@
   $: searchRgx = new RegExp(search, "ig")
 
   function checkValid() {
-    validity = isValid(value)
-  }
-
-  function addToText(binding) {
-    const position = getCaretPosition()
-    const toAdd = `{{ ${binding.path} }}`
-    if (position.start) {
-      value =
-        value.substring(0, position.start) +
-        toAdd +
-        value.substring(position.end, value.length)
+    if (hasReadable) {
+      const runtime = readableToRuntimeBinding(bindableProperties, value)
+      validity = isValid(runtime)
     } else {
-      value += toAdd
+      validity = isValid(value)
     }
   }
-  export function cancel() {
-    dispatch("update", originalValue)
-    bindingDrawer.close()
-  }
 </script>
 
 <DrawerContent>
@@ -63,7 +53,12 @@
           {#each bindableProperties.filter(binding =>
             binding.label.match(searchRgx)
           ) as binding}
-            <div class="binding" on:click={() => addToText(binding)}>
+            <div
+              class="binding"
+              on:click={() => {
+                value = addToText(value, getCaretPosition(), binding)
+              }}
+            >
               <span class="binding__label">{binding.label}</span>
               <span class="binding__type">{binding.type}</span>
               <br />
@@ -77,13 +72,18 @@
       <div class="section">
         <Heading size="XS">Helpers</Heading>
         {#each helpers.filter(helper => helper.label.match(searchRgx) || helper.description.match(searchRgx)) as helper}
-          <div class="binding" on:click={() => addToText(helper)}>
+          <div
+            class="binding"
+            on:click={() => {
+              value = addToText(value, getCaretPosition(), helper.text)
+            }}
+          >
             <span class="binding__label">{helper.label}</span>
             <br />
             <div class="binding__description">
               {@html helper.description || ""}
             </div>
-            <pre>{helper.example || ''}</pre>
+            <pre>{helper.example || ""}</pre>
           </div>
         {/each}
       </div>
@@ -92,7 +92,6 @@
   <div class="text">
     <TextArea
       bind:getCaretPosition
-      thin
       bind:value
       placeholder="Add text, or click the objects on the left to add them to the textbox."
     />
@@ -122,7 +121,7 @@
     font-family: var(--font-sans);
   }
   .text :global(textarea) {
-    min-height: 100px;
+    min-height: 150px !important;
   }
   .text :global(p) {
     margin: 0;
@@ -130,8 +129,12 @@
 
   .binding {
     font-size: 12px;
-    padding: var(--spacing-s);
-    border-radius: var(--border-radius-m);
+    border: var(--border-light);
+    border-width: 1px 0 0 0;
+    padding: var(--spacing-m) 0;
+    margin: auto 0;
+    align-items: center;
+    cursor: pointer;
   }
   .binding:hover {
     background-color: var(--grey-2);

packages/builder/src/components/common/bindings/utils.js

@@ -0,0 +1,16 @@
+export function addToText(value, caretPos, binding) {
+  binding = typeof binding === "string" ? binding : binding.path
+  value = value == null ? "" : value
+  if (!value.includes("{{") && !value.includes("}}")) {
+    binding = `{{ ${binding} }}`
+  }
+  if (caretPos.start) {
+    value =
+      value.substring(0, caretPos.start) +
+      binding +
+      value.substring(caretPos.end, value.length)
+  } else {
+    value += binding
+  }
+  return value
+}

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/DeleteRow.svelte

@@ -3,7 +3,7 @@
   import { store, currentAsset } from "builderStore"
   import { tables } from "stores/backend"
   import { getBindableProperties } from "builderStore/dataBinding"
-  import DrawerBindableInput from "components/common/DrawerBindableInput.svelte"
+  import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
 
   export let parameters
 

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/LogIn.svelte

@@ -2,7 +2,7 @@
   import { Label } from "@budibase/bbui"
   import { getBindableProperties } from "builderStore/dataBinding"
   import { currentAsset, store } from "builderStore"
-  import DrawerBindableInput from "components/common/DrawerBindableInput.svelte"
+  import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
 
   export let parameters
 

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/NavigateTo.svelte

@@ -2,7 +2,7 @@
   import { Label } from "@budibase/bbui"
   import { getBindableProperties } from "builderStore/dataBinding"
   import { currentAsset, store } from "builderStore"
-  import DrawerBindableInput from "components/common/DrawerBindableInput.svelte"
+  import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
 
   export let parameters
 

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/SaveFields.svelte

@@ -3,7 +3,7 @@
   import { store, currentAsset } from "builderStore"
   import { getBindableProperties } from "builderStore/dataBinding"
   import { createEventDispatcher } from "svelte"
-  import DrawerBindableInput from "components/common/DrawerBindableInput.svelte"
+  import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
 
   const dispatch = createEventDispatcher()
 

packages/builder/src/components/design/PropertiesPanel/PropertyControls/PropertyControl.svelte

@@ -6,7 +6,7 @@
     readableToRuntimeBinding,
     runtimeToReadableBinding,
   } from "builderStore/dataBinding"
-  import BindingPanel from "components/design/PropertiesPanel/BindingPanel.svelte"
+  import BindingPanel from "components/common/bindings/BindingPanel.svelte"
   import { capitalise } from "../../../../helpers"
 
   export let label = ""

packages/builder/src/components/integration/QueryParameterBuilder.svelte

@@ -4,7 +4,7 @@
     readableToRuntimeBinding,
     runtimeToReadableBinding,
   } from "builderStore/dataBinding"
-  import DrawerBindableInput from "components/common/DrawerBindableInput.svelte"
+  import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
 
   export let bindable = true
   export let parameters = []

packages/builder/src/constants/backend/index.js

@@ -80,6 +80,15 @@ export const FIELDS = {
       presence: false,
     },
   },
+  FORMULA: {
+    name: "Formula",
+    icon: "ri-braces-line",
+    type: "formula",
+    constraints: {
+      type: "string",
+      presence: false,
+    },
+  },
 }
 
 export const AUTO_COLUMN_SUB_TYPES = {

packages/server/src/api/controllers/user.js

@@ -88,6 +88,7 @@ exports.updateMetadata = async function (ctx) {
   })
   const metadata = {
     ...globalUser,
+    tableId: InternalTables.USER_METADATA,
     _id: user._id || generateUserMetadataID(globalUser._id),
     _rev: user._rev,
   }

packages/server/src/constants/index.js

@@ -13,6 +13,7 @@ exports.FieldTypes = {
   DATETIME: "datetime",
   ATTACHMENT: "attachment",
   LINK: "link",
+  FORMULA: "formula",
   AUTO: "auto",
 }
 

packages/server/src/db/linkedRows/index.js

@@ -10,6 +10,7 @@ const {
 } = require("./linkUtils")
 const { flatten } = require("lodash")
 const CouchDB = require("../../db")
+const { FieldTypes } = require("../../constants")
 const { getMultiIDParams } = require("../../db/utils")
 
 /**
@@ -141,14 +142,14 @@ exports.attachLinkIDs = async (appId, rows) => {
 }
 
 /**
- * Given information about the table we can extract the display name from the linked rows, this
- * is what we do for showing the display name of each linked row when in a table format.
+ * Given a table and a list of rows this will retrieve all of the attached docs and enrich them into the row.
+ * This is required for formula fields, this may only be utilised internally (for now).
  * @param {string} appId The app in which the tables/rows/links exist.
  * @param {object} table The table from which the rows originated.
- * @param {array<object>} rows The rows which are to be enriched with the linked display names/IDs.
- * @returns {Promise<Array>} The enriched rows after having display names/IDs attached to the linked fields.
+ * @param {array<object>} rows The rows which are to be enriched.
+ * @return {Promise<*>} returns the rows with all of the enriched relationships on it.
  */
-exports.attachLinkedPrimaryDisplay = async (appId, table, rows) => {
+exports.attachFullLinkedDocs = async (appId, table, rows) => {
   const linkedTableIds = getLinkedTableIDs(table)
   if (linkedTableIds.length === 0) {
     return rows
@@ -161,7 +162,6 @@ exports.attachLinkedPrimaryDisplay = async (appId, table, rows) => {
   const linked = (await db.allDocs(getMultiIDParams(linkedRowIds))).rows.map(
     row => row.doc
   )
-  // will populate this as we find them
   const linkedTables = []
   for (let row of rows) {
     for (let link of links.filter(link => link.thisId === row._id)) {
@@ -175,13 +175,44 @@ exports.attachLinkedPrimaryDisplay = async (appId, table, rows) => {
       if (!linkedRow || !linkedTable) {
         continue
       }
-      const obj = { _id: linkedRow._id }
-      // if we know the display column, add it
-      if (linkedRow[linkedTable.primaryDisplay] != null) {
-        obj.primaryDisplay = linkedRow[linkedTable.primaryDisplay]
-      }
-      row[link.fieldName].push(obj)
+      row[link.fieldName].push(linkedRow)
     }
   }
   return rows
 }
+
+/**
+ * This function will take the given enriched rows and squash the links to only contain the primary display field.
+ * @param {string} appId The app in which the tables/rows/links exist.
+ * @param {object} table The table from which the rows originated.
+ * @param {array<object>} enriched The pre-enriched rows (full docs) which are to be squashed.
+ * @returns {Promise<Array>} The rows after having their links squashed to only contain the ID and primary display.
+ */
+exports.squashLinksToPrimaryDisplay = async (appId, table, enriched) => {
+  const db = new CouchDB(appId)
+  // will populate this as we find them
+  const linkedTables = []
+  for (let [column, schema] of Object.entries(table.schema)) {
+    if (schema.type !== FieldTypes.LINK) {
+      continue
+    }
+    for (let row of enriched) {
+      if (!row[column] || !row[column].length) {
+        continue
+      }
+      const newLinks = []
+      for (let link of row[column]) {
+        const linkTblId = link.tableId || getRelatedTableForField(table, column)
+        // this only fetches the table if its not already in array
+        const linkedTable = await getLinkedTable(db, linkTblId, linkedTables)
+        const obj = { _id: link._id }
+        if (link[linkedTable.primaryDisplay]) {
+          obj.primaryDisplay = link[linkedTable.primaryDisplay]
+        }
+        newLinks.push(obj)
+      }
+      row[column] = newLinks
+    }
+  }
+  return enriched
+}

packages/server/src/utilities/rowProcessor.js

@@ -1,7 +1,7 @@
-// const { ObjectStoreBuckets } = require("../constants")
 const linkRows = require("../db/linkedRows")
 const { cloneDeep } = require("lodash/fp")
 const { FieldTypes, AutoFieldSubTypes } = require("../constants")
+const { processStringSync } = require("@budibase/string-templates")
 const { attachmentsRelativeURL } = require("./index")
 
 const BASE_AUTO_ID = 1
@@ -34,6 +34,11 @@ const TYPE_TRANSFORM_MAP = {
     [null]: "",
     [undefined]: undefined,
   },
+  [FieldTypes.FORMULA]: {
+    "": "",
+    [null]: "",
+    [undefined]: undefined,
+  },
   [FieldTypes.LONGFORM]: {
     "": "",
     [null]: "",
@@ -118,6 +123,41 @@ function processAutoColumn(user, table, row) {
   return { table, row }
 }
 
+/**
+ * Given a set of rows and the table they came from this function will sort by auto ID or a custom
+ * method if provided (not implemented yet).
+ */
+function sortRows(table, rows) {
+  // sort based on auto ID (if found)
+  let autoIDColumn = Object.entries(table.schema).find(
+    schema => schema[1].subtype === AutoFieldSubTypes.AUTO_ID
+  )
+  // get the column name, this is the first element in the array (Object.entries)
+  autoIDColumn = autoIDColumn && autoIDColumn.length ? autoIDColumn[0] : null
+  if (autoIDColumn) {
+    // sort in ascending order
+    rows.sort((a, b) => a[autoIDColumn] - b[autoIDColumn])
+  }
+  return rows
+}
+
+/**
+ * Looks through the rows provided and finds formulas - which it then processes.
+ */
+function processFormulas(table, rows) {
+  for (let [column, schema] of Object.entries(table.schema)) {
+    if (schema.type !== FieldTypes.FORMULA) {
+      continue
+    }
+    // iterate through rows and process formula
+    rows = rows.map(row => ({
+      ...row,
+      [column]: processStringSync(schema.formula, row),
+    }))
+  }
+  return rows
+}
+
 /**
  * This will coerce a value to the correct types based on the type transform map
  * @param {object} row The value to coerce
@@ -173,16 +213,18 @@ exports.outputProcessing = async (appId, table, rows) => {
     rows = [rows]
     wasArray = false
   }
+  // sort by auto ID
+  rows = sortRows(table, rows)
   // attach any linked row information
-  const outputRows = await linkRows.attachLinkedPrimaryDisplay(
-    appId,
-    table,
-    rows
-  )
+  let enriched = await linkRows.attachFullLinkedDocs(appId, table, rows)
+
+  // process formulas
+  enriched = processFormulas(table, enriched)
+
   // update the attachments URL depending on hosting
   for (let [property, column] of Object.entries(table.schema)) {
     if (column.type === FieldTypes.ATTACHMENT) {
-      for (let row of outputRows) {
+      for (let row of enriched) {
         if (row[property] == null || row[property].length === 0) {
           continue
         }
@@ -192,5 +234,6 @@ exports.outputProcessing = async (appId, table, rows) => {
       }
     }
   }
-  return wasArray ? outputRows : outputRows[0]
+  enriched = await linkRows.squashLinksToPrimaryDisplay(appId, table, enriched)
+  return wasArray ? enriched : enriched[0]
 }

packages/server/src/utilities/workerRequests.js

@@ -122,12 +122,15 @@ exports.saveGlobalUser = async (ctx, appId, body) => {
   if (json.status !== 200 && response.status !== 200) {
     ctx.throw(400, "Unable to save global user.")
   }
-  delete body.email
   delete body.password
-  delete body.roleId
-  delete body.status
   delete body.roles
   delete body.builder
+  // TODO: for now these have been left in as they are
+  // TODO: pretty important to keeping relationships working
+  // TODO: however if user metadata is changed this should be removed
+  // delete body.email
+  // delete body.roleId
+  // delete body.status
   return {
     ...body,
     _id: json._id,

packages/string-templates/test/helpers.spec.js

@@ -273,6 +273,19 @@ describe("test the comparison helpers", () => {
   })
 })
 
+describe("Test the object/array helper", () => {
+  it("should allow plucking from an array of objects", async () => {
+    const context = {
+      items: [
+        { price: 20 },
+        { price: 30 },
+      ]
+    }
+    const output = await processString("{{ literal ( sum ( pluck items 'price' ) ) }}", context)
+    expect(output).toBe(50)
+  })
+})
+
 describe("Test the literal helper", () => {
   it("should allow use of the literal specifier for a number", async () => {
     const output = await processString(`{{literal a}}`, {