Merge master.

.gitignore

@@ -107,3 +107,4 @@ budibase-component
 budibase-datasource
 
 *.iml
+.nx

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.22.8",
+  "version": "2.22.12",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/bbui/src/Actions/click_outside.js

@@ -79,7 +79,8 @@ const removeHandler = id => {
 export default (element, opts) => {
   const id = Math.random()
   const update = newOpts => {
-    const callback = newOpts?.callback || newOpts
+    const callback =
+      newOpts?.callback || (typeof newOpts === "function" ? newOpts : null)
     const anchor = newOpts?.anchor || element
     const allowedType = newOpts?.allowedType || "click"
     updateHandler(id, element, anchor, callback, allowedType)

packages/bbui/src/Drawer/DrawerContent.svelte

@@ -42,7 +42,6 @@
   .main {
     height: 100%;
     overflow: auto;
-    overflow-x: hidden;
   }
   .padding .main {
     padding: var(--spacing-xl);

packages/bbui/src/Table/CellRenderer.svelte

@@ -12,6 +12,7 @@
   export let schema
   export let value
   export let customRenderers = []
+  export let snippets
 
   let renderer
   const typeMap = {
@@ -44,7 +45,7 @@
     if (!template) {
       return value
     }
-    return processStringSync(template, { value })
+    return processStringSync(template, { value, snippets })
   }
 </script>
 

packages/bbui/src/Table/Table.svelte

@@ -42,6 +42,7 @@
   export let customPlaceholder = false
   export let showHeaderBorder = true
   export let placeholderText = "No rows found"
+  export let snippets = []
 
   const dispatch = createEventDispatcher()
 
@@ -425,6 +426,7 @@
                   <CellRenderer
                     {customRenderers}
                     {row}
+                    {snippets}
                     schema={schema[field]}
                     value={deepGet(row, field)}
                     on:clickrelationship

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -191,8 +191,10 @@
     // don't make field IDs for auto types
     if (type === AUTO_TYPE || autocolumn) {
       return type.toUpperCase()
-    } else {
+    } else if (type === FieldType.BB_REFERENCE) {
       return `${type}${subtype || ""}`.toUpperCase()
+    } else {
+      return type.toUpperCase()
     }
   }
 
@@ -703,24 +705,6 @@
       thin
       text="Allow multiple users"
     />
-  {:else if editableColumn.type === FieldType.ATTACHMENT}
-    <Toggle
-      value={editableColumn.constraints?.length?.maximum !== 1}
-      on:change={e => {
-        if (!e.detail) {
-          editableColumn.constraints ??= { length: {} }
-          editableColumn.constraints.length ??= {}
-          editableColumn.constraints.length.maximum = 1
-          editableColumn.constraints.length.message =
-            "cannot contain multiple files"
-        } else {
-          delete editableColumn.constraints?.length?.maximum
-          delete editableColumn.constraints?.length?.message
-        }
-      }}
-      thin
-      text="Allow multiple"
-    />
   {/if}
   {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
     <Select

packages/builder/src/components/common/CodeEditor/index.js

@@ -313,7 +313,7 @@ export const bindingsToCompletions = (bindings, mode) => {
       ...bindingByCategory[catKey].reduce((acc, binding) => {
         let displayType = binding.fieldSchema?.type || binding.display?.type
         acc.push({
-          label: binding.display?.name || "NO NAME",
+          label: binding.display?.name || binding.readableBinding || "NO NAME",
           info: completion => {
             return buildBindingInfoNode(completion, binding)
           },

packages/builder/src/components/common/bindings/ClientBindingPanel.svelte

@@ -8,6 +8,7 @@
   export let allowJS = false
   export let allowHelpers = true
   export let autofocusEditor = false
+  export let context = null
 
   $: enrichedBindings = enrichBindings(bindings)
 
@@ -27,7 +28,7 @@
 
 <BindingPanel
   bindings={enrichedBindings}
-  context={$previewStore.selectedComponentContext}
+  context={{ ...$previewStore.selectedComponentContext, ...context }}
   snippets={$snippets}
   {value}
   {allowJS}

packages/builder/src/components/deploy/AppActions.svelte

@@ -32,10 +32,14 @@
   import TourWrap from "components/portal/onboarding/TourWrap.svelte"
   import { TOUR_STEP_KEYS } from "components/portal/onboarding/tours.js"
   import { goto } from "@roxi/routify"
+  import { onMount } from "svelte"
+  import PosthogClient from "../../analytics/PosthogClient"
 
   export let application
   export let loaded
 
+  const posthog = new PosthogClient(process.env.POSTHOG_TOKEN)
+
   let unpublishModal
   let updateAppModal
   let revertModal
@@ -44,6 +48,8 @@
   let appActionPopoverOpen = false
   let appActionPopoverAnchor
   let publishing = false
+  let showNpsSurvey = false
+  let lastOpened
 
   $: filteredApps = $appsStore.apps.filter(app => app.devId === application)
   $: selectedApp = filteredApps?.length ? filteredApps[0] : null
@@ -57,7 +63,7 @@
     $appStore.version &&
     $appStore.upgradableVersion !== $appStore.version
   $: canPublish = !publishing && loaded && $sortedScreens.length > 0
-  $: lastDeployed = getLastDeployedString($deploymentStore)
+  $: lastDeployed = getLastDeployedString($deploymentStore, lastOpened)
 
   const initialiseApp = async () => {
     const applicationPkg = await API.fetchAppPackage($appStore.devId)
@@ -97,6 +103,7 @@
         type: "success",
         icon: "GlobeCheck",
       })
+      showNpsSurvey = true
       await completePublish()
     } catch (error) {
       console.error(error)
@@ -147,6 +154,10 @@
       notifications.error("Error refreshing app")
     }
   }
+
+  onMount(() => {
+    posthog.init()
+  })
 </script>
 
 <!-- svelte-ignore a11y-click-events-have-key-events -->
@@ -201,6 +212,7 @@
       class="app-action-button publish app-action-popover"
       on:click={() => {
         if (!appActionPopoverOpen) {
+          lastOpened = new Date()
           appActionPopover.show()
         } else {
           appActionPopover.hide()
@@ -343,6 +355,10 @@
 <RevertModal bind:this={revertModal} />
 <VersionModal hideIcon bind:this={versionModal} />
 
+{#if showNpsSurvey}
+  <div class="nps-survey" />
+{/if}
+
 <style>
   .app-action-popover-content {
     padding: var(--spacing-xl);

packages/builder/src/components/design/settings/controls/ColumnEditor/CellDrawer.svelte

@@ -7,10 +7,13 @@
     Layout,
     Label,
   } from "@budibase/bbui"
-  import { themeStore } from "stores/builder"
+  import { themeStore, previewStore } from "stores/builder"
   import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
 
   export let column
+
+  $: columnValue =
+    $previewStore.selectedComponentContext?.eventContext?.row?.[column.name]
 </script>
 
 <DrawerContent>
@@ -41,6 +44,9 @@
             icon: "TableColumnMerge",
           },
         ]}
+        context={{
+          value: columnValue,
+        }}
       />
       <Layout noPadding gap="XS">
         <Label>Background color</Label>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/new/_components/componentStructure.json

@@ -3,8 +3,6 @@
     "name": "Blocks",
     "icon": "Article",
     "children": [
-      "gridblock",
-      "tableblock",
       "cardsblock",
       "repeaterblock",
       "formblock",
@@ -16,7 +14,7 @@
   {
     "name": "Layout",
     "icon": "ClassicGridView",
-    "children": ["container", "section", "grid", "sidepanel"]
+    "children": ["container", "section", "sidepanel"]
   },
   {
     "name": "Data",
@@ -24,7 +22,7 @@
     "children": [
       "dataprovider",
       "repeater",
-      "table",
+      "gridblock",
       "spreadsheet",
       "dynamicfilter",
       "daterangepicker"

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte

@@ -19,7 +19,8 @@
   import { goto } from "@roxi/routify"
   import { TOUR_KEYS } from "components/portal/onboarding/tours.js"
   import formScreen from "templates/formScreen"
-  import rowListScreen from "templates/rowListScreen"
+  import gridListScreen from "templates/gridListScreen"
+  import gridDetailsScreen from "templates/gridDetailsScreen"
 
   let mode
   let pendingScreen
@@ -127,7 +128,7 @@
     screenAccessRole = Roles.BASIC
     formType = null
 
-    if (mode === "table" || mode === "grid" || mode === "form") {
+    if (mode === "grid" || mode === "gridDetails" || mode === "form") {
       datasourceModal.show()
     } else if (mode === "blank") {
       let templates = getTemplates($tables.list)
@@ -153,7 +154,10 @@
 
   // Handler for Datasource Screen Creation
   const completeDatasourceScreenCreation = async () => {
-    templates = rowListScreen(selectedDatasources, mode)
+    templates =
+      mode === "grid"
+        ? gridListScreen(selectedDatasources)
+        : gridDetailsScreen(selectedDatasources)
 
     const screens = templates.map(template => {
       let screenTemplate = template.create()

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/index.svelte

@@ -2,8 +2,8 @@
   import { Body } from "@budibase/bbui"
   import CreationPage from "components/common/CreationPage.svelte"
   import blankImage from "./images/blank.png"
-  import tableImage from "./images/table.png"
+  import tableInline from "./images/tableInline.png"
-  import gridImage from "./images/grid.png"
+  import tableDetails from "./images/tableDetails.png"
   import formImage from "./images/form.png"
   import CreateScreenModal from "./CreateScreenModal.svelte"
   import { screenStore } from "stores/builder"
@@ -38,23 +38,23 @@
         </div>
       </div>
 
-      <div class="card" on:click={() => createScreenModal.show("table")}>
+      <div class="card" on:click={() => createScreenModal.show("grid")}>
         <div class="image">
-          <img alt="" src={tableImage} />
+          <img alt="" src={tableInline} />
         </div>
         <div class="text">
-          <Body size="S">Table</Body>
+          <Body size="S">Table with inline editing</Body>
-          <Body size="XS">View, edit and delete rows on a table</Body>
+          <Body size="XS">View, edit and delete rows inline</Body>
         </div>
       </div>
 
-      <div class="card" on:click={() => createScreenModal.show("grid")}>
+      <div class="card" on:click={() => createScreenModal.show("gridDetails")}>
         <div class="image">
-          <img alt="" src={gridImage} />
+          <img alt="" src={tableDetails} />
         </div>
         <div class="text">
-          <Body size="S">Grid</Body>
+          <Body size="S">Table with details panel</Body>
-          <Body size="XS">View and manipulate rows on a grid</Body>
+          <Body size="XS">Manage your row details in a side panel</Body>
         </div>
       </div>
 
@@ -113,6 +113,11 @@
     width: 100%;
   }
 
+  .card .image {
+    min-height: 130px;
+    min-width: 235px;
+  }
+
   .text {
     border: 1px solid var(--grey-4);
     border-radius: 0 0 4px 4px;

packages/builder/src/stores/builder/components.js

@@ -279,12 +279,10 @@ export class ComponentStore extends BudiStore {
       else {
         if (setting.type === "dataProvider") {
           // Validate data provider exists, or else clear it
-          const treeId = parent?._id || component._id
+          const providers = findAllMatchingComponents(
-          const path = findComponentPath(screen?.props, treeId)
+            screen?.props,
-          const providers = path.filter(component =>
+            x => x._component === "@budibase/standard-components/dataprovider"
-            component._component?.endsWith("/dataprovider")
           )
-          // Validate non-empty values
           const valid = providers?.some(dp => value.includes?.(dp._id))
           if (!valid) {
             if (providers.length) {

packages/builder/src/stores/builder/hover.js

@@ -7,12 +7,25 @@ export const INITIAL_HOVER_STATE = {
 }
 
 export class HoverStore extends BudiStore {
+  hoverTimeout
+
   constructor() {
     super({ ...INITIAL_HOVER_STATE })
     this.hover = this.hover.bind(this)
   }
 
   hover(componentId, notifyClient = true) {
+    clearTimeout(this.hoverTimeout)
+    if (componentId) {
+      this.processHover(componentId, notifyClient)
+    } else {
+      this.hoverTimeout = setTimeout(() => {
+        this.processHover(componentId, notifyClient)
+      }, 10)
+    }
+  }
+
+  processHover(componentId, notifyClient) {
     if (componentId === get(this.store).componentId) {
       return
     }

packages/builder/src/templates/gridDetailsScreen.js

@@ -0,0 +1,158 @@
+import sanitizeUrl from "helpers/sanitizeUrl"
+import { Screen } from "./Screen"
+import { Component } from "./Component"
+import { generate } from "shortid"
+import { makePropSafe as safe } from "@budibase/string-templates"
+import { Utils } from "@budibase/frontend-core"
+
+export default function (datasources) {
+  if (!Array.isArray(datasources)) {
+    return []
+  }
+  return datasources.map(datasource => {
+    return {
+      name: `${datasource.label} - List with panel`,
+      create: () => createScreen(datasource),
+      id: GRID_DETAILS_TEMPLATE,
+      resourceId: datasource.resourceId,
+    }
+  })
+}
+
+export const GRID_DETAILS_TEMPLATE = "GRID_DETAILS_TEMPLATE"
+export const gridDetailsUrl = datasource => sanitizeUrl(`/${datasource.label}`)
+
+const createScreen = datasource => {
+  /*
+    Create Row
+   */
+  const createRowSidePanel = new Component(
+    "@budibase/standard-components/sidepanel"
+  ).instanceName("New row side panel")
+
+  const buttonGroup = new Component("@budibase/standard-components/buttongroup")
+  const createButton = new Component("@budibase/standard-components/button")
+
+  createButton.customProps({
+    onClick: [
+      {
+        id: 0,
+        "##eventHandlerType": "Open Side Panel",
+        parameters: {
+          id: createRowSidePanel._json._id,
+        },
+      },
+    ],
+    text: "Create row",
+    type: "cta",
+  })
+
+  buttonGroup.instanceName(`${datasource.label} - Create`).customProps({
+    hAlign: "right",
+    buttons: [createButton.json()],
+  })
+
+  const gridHeader = new Component("@budibase/standard-components/container")
+    .instanceName("Heading container")
+    .customProps({
+      direction: "row",
+      hAlign: "stretch",
+    })
+
+  const heading = new Component("@budibase/standard-components/heading")
+    .instanceName("Table heading")
+    .customProps({
+      text: datasource?.label,
+    })
+
+  gridHeader.addChild(heading)
+  gridHeader.addChild(buttonGroup)
+
+  const createFormBlock = new Component(
+    "@budibase/standard-components/formblock"
+  )
+  createFormBlock.instanceName("Create row form block").customProps({
+    dataSource: datasource,
+    labelPosition: "left",
+    buttonPosition: "top",
+    actionType: "Create",
+    title: "Create row",
+    buttons: Utils.buildFormBlockButtonConfig({
+      _id: createFormBlock._json._id,
+      showDeleteButton: false,
+      showSaveButton: true,
+      saveButtonLabel: "Save",
+      actionType: "Create",
+      dataSource: datasource,
+    }),
+  })
+
+  createRowSidePanel.addChild(createFormBlock)
+
+  /*
+    Edit Row
+   */
+  const stateKey = `ID_${generate()}`
+  const detailsSidePanel = new Component(
+    "@budibase/standard-components/sidepanel"
+  ).instanceName("Edit row side panel")
+
+  const editFormBlock = new Component("@budibase/standard-components/formblock")
+  editFormBlock.instanceName("Edit row form block").customProps({
+    dataSource: datasource,
+    labelPosition: "left",
+    buttonPosition: "top",
+    actionType: "Update",
+    title: "Edit",
+    rowId: `{{ ${safe("state")}.${safe(stateKey)} }}`,
+    buttons: Utils.buildFormBlockButtonConfig({
+      _id: editFormBlock._json._id,
+      showDeleteButton: true,
+      showSaveButton: true,
+      saveButtonLabel: "Save",
+      deleteButtonLabel: "Delete",
+      actionType: "Update",
+      dataSource: datasource,
+    }),
+  })
+
+  detailsSidePanel.addChild(editFormBlock)
+
+  const gridBlock = new Component("@budibase/standard-components/gridblock")
+  gridBlock
+    .customProps({
+      table: datasource,
+      allowAddRows: false,
+      allowEditRows: false,
+      allowDeleteRows: false,
+      onRowClick: [
+        {
+          id: 0,
+          "##eventHandlerType": "Update State",
+          parameters: {
+            key: stateKey,
+            type: "set",
+            persist: false,
+            value: `{{ ${safe("eventContext")}.${safe("row")}._id }}`,
+          },
+        },
+        {
+          id: 1,
+          "##eventHandlerType": "Open Side Panel",
+          parameters: {
+            id: detailsSidePanel._json._id,
+          },
+        },
+      ],
+    })
+    .instanceName(`${datasource.label} - Table`)
+
+  return new Screen()
+    .route(gridDetailsUrl(datasource))
+    .instanceName(`${datasource.label} - List and details`)
+    .addChild(gridHeader)
+    .addChild(gridBlock)
+    .addChild(createRowSidePanel)
+    .addChild(detailsSidePanel)
+    .json()
+}

packages/builder/src/templates/gridListScreen.js

@@ -0,0 +1,41 @@
+import sanitizeUrl from "helpers/sanitizeUrl"
+import { Screen } from "./Screen"
+import { Component } from "./Component"
+
+export default function (datasources) {
+  if (!Array.isArray(datasources)) {
+    return []
+  }
+  return datasources.map(datasource => {
+    return {
+      name: `${datasource.label} - List`,
+      create: () => createScreen(datasource),
+      id: GRID_LIST_TEMPLATE,
+      resourceId: datasource.resourceId,
+    }
+  })
+}
+
+export const GRID_LIST_TEMPLATE = "GRID_LIST_TEMPLATE"
+export const gridListUrl = datasource => sanitizeUrl(`/${datasource.label}`)
+
+const createScreen = datasource => {
+  const heading = new Component("@budibase/standard-components/heading")
+    .instanceName("Table heading")
+    .customProps({
+      text: datasource?.label,
+    })
+
+  const gridBlock = new Component("@budibase/standard-components/gridblock")
+    .instanceName(`${datasource.label} - Table`)
+    .customProps({
+      table: datasource,
+    })
+
+  return new Screen()
+    .route(gridListUrl(datasource))
+    .instanceName(`${datasource.label} - List`)
+    .addChild(heading)
+    .addChild(gridBlock)
+    .json()
+}

packages/builder/src/templates/index.js

@@ -1,9 +1,11 @@
-import rowListScreen from "./rowListScreen"
+import gridListScreen from "./gridListScreen"
+import gridDetailsScreen from "./gridDetailsScreen"
 import createFromScratchScreen from "./createFromScratchScreen"
 import formScreen from "./formScreen"
 
 const allTemplates = datasources => [
-  ...rowListScreen(datasources),
+  ...gridListScreen(datasources),
+  ...gridDetailsScreen(datasources),
   ...formScreen(datasources),
 ]
 

packages/builder/src/templates/rowListScreen.js

@@ -1,63 +0,0 @@
-import sanitizeUrl from "helpers/sanitizeUrl"
-import { Screen } from "./Screen"
-import { Component } from "./Component"
-
-export default function (datasources, mode = "table") {
-  if (!Array.isArray(datasources)) {
-    return []
-  }
-  return datasources.map(datasource => {
-    return {
-      name: `${datasource.label} - List`,
-      create: () => createScreen(datasource, mode),
-      id: ROW_LIST_TEMPLATE,
-      resourceId: datasource.resourceId,
-    }
-  })
-}
-
-export const ROW_LIST_TEMPLATE = "ROW_LIST_TEMPLATE"
-export const rowListUrl = datasource => sanitizeUrl(`/${datasource.label}`)
-
-const generateTableBlock = datasource => {
-  const tableBlock = new Component("@budibase/standard-components/tableblock")
-  tableBlock
-    .customProps({
-      title: datasource.label,
-      dataSource: datasource,
-      sortOrder: "Ascending",
-      size: "spectrum--medium",
-      paginate: true,
-      rowCount: 8,
-      clickBehaviour: "details",
-      showTitleButton: true,
-      titleButtonText: "Create row",
-      titleButtonClickBehaviour: "new",
-      sidePanelSaveLabel: "Save",
-      sidePanelDeleteLabel: "Delete",
-    })
-    .instanceName(`${datasource.label} - Table block`)
-  return tableBlock
-}
-
-const generateGridBlock = datasource => {
-  const gridBlock = new Component("@budibase/standard-components/gridblock")
-  gridBlock
-    .customProps({
-      table: datasource,
-    })
-    .instanceName(`${datasource.label} - Grid block`)
-  return gridBlock
-}
-
-const createScreen = (datasource, mode) => {
-  return new Screen()
-    .route(rowListUrl(datasource))
-    .instanceName(`${datasource.label} - List`)
-    .addChild(
-      mode === "table"
-        ? generateTableBlock(datasource)
-        : generateGridBlock(datasource)
-    )
-    .json()
-}

packages/client/manifest.json

@@ -4673,6 +4673,7 @@
     }
   },
   "table": {
+    "deprecated": true,
     "name": "Table",
     "icon": "Table",
     "illegalChildren": ["section"],
@@ -5418,6 +5419,7 @@
     ]
   },
   "tableblock": {
+    "deprecated": true,
     "block": true,
     "name": "Table Block",
     "icon": "Table",
@@ -6595,7 +6597,7 @@
     ]
   },
   "gridblock": {
-    "name": "Grid Block",
+    "name": "Table",
     "icon": "Table",
     "styles": ["size"],
     "size": {

packages/client/src/components/Component.svelte

@@ -246,15 +246,18 @@
       return
     }
 
+    const cacheId = `${definition.name}${
+      definition?.deprecated === true ? "_deprecated" : ""
+    }`
     // Get the settings definition for this component, and cache it
-    if (SettingsDefinitionCache[definition.name]) {
+    if (SettingsDefinitionCache[cacheId]) {
-      settingsDefinition = SettingsDefinitionCache[definition.name]
+      settingsDefinition = SettingsDefinitionCache[cacheId]
-      settingsDefinitionMap = SettingsDefinitionMapCache[definition.name]
+      settingsDefinitionMap = SettingsDefinitionMapCache[cacheId]
     } else {
       settingsDefinition = getSettingsDefinition(definition)
       settingsDefinitionMap = getSettingsDefinitionMap(settingsDefinition)
-      SettingsDefinitionCache[definition.name] = settingsDefinition
+      SettingsDefinitionCache[cacheId] = settingsDefinition
-      SettingsDefinitionMapCache[definition.name] = settingsDefinitionMap
+      SettingsDefinitionMapCache[cacheId] = settingsDefinitionMap
     }
 
     // Parse the instance settings, and cache them

packages/client/src/components/app/blocks/MultiStepFormblock.svelte

@@ -26,9 +26,12 @@
 
   let schema
 
+  $: id = $component.id
+  $: selected = $component.selected
+  $: builderStep = $builderStore.metadata?.step
   $: fetchSchema(dataSource)
-  $: enrichedSteps = enrichSteps(steps, schema, $component.id, $currentStep)
+  $: enrichedSteps = enrichSteps(steps, schema, id)
-  $: updateCurrentStep(enrichedSteps, $builderStore, $component)
+  $: updateCurrentStep(enrichedSteps, selected, builderStep)
 
   // Provide additional data context for live binding eval
   export const getAdditionalDataContext = () => {
@@ -40,30 +43,22 @@
     }
   }
 
-  const updateCurrentStep = (steps, builderStore, component) => {
+  const updateCurrentStep = (steps, selected, builderStep) => {
-    const { componentId, step } = builderStore.metadata || {}
+    // If we aren't selected in the builder then just allowing the normal form
-
+    // to take control.
-    // If we aren't in the builder or aren't selected then don't update the step
+    if (!selected) {
-    // context at all, allowing the normal form to take control.
-    if (
-      !component.selected ||
-      !builderStore.inBuilder ||
-      componentId !== component.id
-    ) {
       return
     }
 
     // Ensure we have a valid step selected
-    let newStep = Math.min(step || 0, steps.length - 1)
+    let newStep = Math.min(builderStep || 0, steps.length - 1)
-
-    // Sanity check
     newStep = Math.max(newStep, 0)
 
     // Add 1 because the form component expects 1 indexed rather than 0 indexed
     currentStep.set(newStep + 1)
   }
 
-  const fetchSchema = async () => {
+  const fetchSchema = async dataSource => {
     schema = (await fetchDatasourceSchema(dataSource)) || {}
   }
 

packages/client/src/components/app/blocks/form/FormBlock.svelte

@@ -34,6 +34,7 @@
   $: formattedFields = convertOldFieldFormat(fields)
   $: fieldsOrDefault = getDefaultFields(formattedFields, schema)
   $: fetchSchema(dataSource)
+  $: id = $component.id
   // We could simply spread $$props into the inner form and append our
   // additions, but that would create svelte warnings about unused props and
   // make maintenance in future more confusing as we typically always have a
@@ -53,7 +54,7 @@
     buttons:
       buttons ||
       Utils.buildFormBlockButtonConfig({
-        _id: $component.id,
+        _id: id,
         showDeleteButton,
         showSaveButton,
         saveButtonLabel,

packages/client/src/components/app/blocks/index.js

@@ -1,4 +1,3 @@
-export { default as tableblock } from "./TableBlock.svelte"
 export { default as cardsblock } from "./CardsBlock.svelte"
 export { default as repeaterblock } from "./RepeaterBlock.svelte"
 export { default as formblock } from "./form/FormBlock.svelte"

packages/client/src/components/app/deprecated/TableBlock.svelte

@@ -50,6 +50,7 @@
   let schemaLoaded = false
 
   $: deleteLabel = setDeleteLabel(sidePanelDeleteLabel, sidePanelShowDelete)
+  $: id = $component.id
   $: isDSPlus = dataSource?.type === "table" || dataSource?.type === "viewV2"
   $: fetchSchema(dataSource)
   $: enrichSearchColumns(searchColumns, schema).then(
@@ -279,7 +280,7 @@
               dataSource,
               buttonPosition: "top",
               buttons: Utils.buildFormBlockButtonConfig({
-                _id: $component.id + "-form-edit",
+                _id: id + "-form-edit",
                 showDeleteButton: deleteLabel !== "",
                 showSaveButton: true,
                 saveButtonLabel: sidePanelSaveLabel || "Save",
@@ -313,7 +314,7 @@
               dataSource,
               buttonPosition: "top",
               buttons: Utils.buildFormBlockButtonConfig({
-                _id: $component.id + "-form-new",
+                _id: id + "-form-new",
                 showDeleteButton: false,
                 showSaveButton: true,
                 saveButtonLabel: "Save",

packages/client/src/components/app/deprecated/table/Table.svelte

@@ -3,7 +3,7 @@
   import { Table } from "@budibase/bbui"
   import SlotRenderer from "./SlotRenderer.svelte"
   import { canBeSortColumn } from "@budibase/shared-core"
-  import Provider from "../../context/Provider.svelte"
+  import Provider from "components/context/Provider.svelte"
 
   export let dataProvider
   export let columns
@@ -16,8 +16,15 @@
   export let noRowsMessage
 
   const component = getContext("component")
-  const { styleable, getAction, ActionTypes, rowSelectionStore } =
+  const context = getContext("context")
-    getContext("sdk")
+  const {
+    styleable,
+    getAction,
+    ActionTypes,
+    rowSelectionStore,
+    generateGoldenSample,
+  } = getContext("sdk")
+
   const customColumnKey = `custom-${Math.random()}`
   const customRenderers = [
     {
@@ -28,6 +35,7 @@
 
   let selectedRows = []
 
+  $: snippets = $context.snippets
   $: hasChildren = $component.children
   $: loading = dataProvider?.loading ?? false
   $: data = dataProvider?.rows || []
@@ -61,6 +69,16 @@
     selectedRows,
   }
 
+  // Provide additional data context for live binding eval
+  export const getAdditionalDataContext = () => {
+    const goldenRow = generateGoldenSample(data)
+    return {
+      eventContext: {
+        row: goldenRow,
+      },
+    }
+  }
+
   const getFields = (
     schema,
     customColumns,
@@ -178,6 +196,7 @@
       {quiet}
       {compact}
       {customRenderers}
+      {snippets}
       allowSelectRows={allowSelectRows && table}
       bind:selectedRows
       allowEditRows={false}

packages/client/src/components/app/forms/optionsParser.js

@@ -6,36 +6,24 @@ export const getOptions = (
   valueColumn,
   customOptions
 ) => {
-  const isArray = fieldSchema?.type === "array"
   // Take options from schema
   if (optionsSource == null || optionsSource === "schema") {
     return fieldSchema?.constraints?.inclusion ?? []
   }
 
-  if (optionsSource === "provider" && isArray) {
-    let optionsSet = {}
-
-    dataProvider?.rows?.forEach(row => {
-      const value = row?.[valueColumn]
-      if (value != null) {
-        const label = row[labelColumn] || value
-        optionsSet[value] = { value, label }
-      }
-    })
-    return Object.values(optionsSet)
-  }
-
   // Extract options from data provider
   if (optionsSource === "provider" && valueColumn) {
-    let optionsSet = {}
+    let valueCache = {}
+    let options = []
     dataProvider?.rows?.forEach(row => {
       const value = row?.[valueColumn]
-      if (value != null) {
+      if (value != null && !valueCache[value]) {
+        valueCache[value] = true
         const label = row[labelColumn] || value
-        optionsSet[value] = { value, label }
+        options.push({ value, label })
       }
     })
-    return Object.values(optionsSet)
+    return options
   }
 
   // Extract custom options

packages/client/src/components/app/index.js

@@ -40,11 +40,12 @@ export { default as sidepanel } from "./SidePanel.svelte"
 export { default as gridblock } from "./GridBlock.svelte"
 export * from "./charts"
 export * from "./forms"
-export * from "./table"
 export * from "./blocks"
 export * from "./dynamic-filter"
 
 // Deprecated component left for compatibility in old apps
+export * from "./deprecated/table"
+export { default as tableblock } from "./deprecated/TableBlock.svelte"
 export { default as navigation } from "./deprecated/Navigation.svelte"
 export { default as cardhorizontal } from "./deprecated/CardHorizontal.svelte"
 export { default as stackedlist } from "./deprecated/StackedList.svelte"

packages/client/src/components/preview/DNDHandler.svelte

@@ -345,8 +345,7 @@
 <IndicatorSet
   componentId={$dndParent}
   color="var(--spectrum-global-color-static-green-500)"
-  zIndex="930"
+  zIndex={920}
-  transition
   prefix="Inside"
 />
 

packages/client/src/components/preview/HoverIndicator.svelte

@@ -1,10 +1,11 @@
 <script>
   import { onMount, onDestroy } from "svelte"
   import IndicatorSet from "./IndicatorSet.svelte"
-  import { builderStore, dndIsDragging, hoverStore } from "stores"
+  import { dndIsDragging, hoverStore, builderStore } from "stores"
 
   $: componentId = $hoverStore.hoveredComponentId
-  $: zIndex = componentId === $builderStore.selectedComponentId ? 900 : 920
+  $: selectedComponentId = $builderStore.selectedComponentId
+  $: selected = componentId === selectedComponentId
 
   const onMouseOver = e => {
     // Ignore if dragging
@@ -45,7 +46,6 @@
 <IndicatorSet
   componentId={$dndIsDragging ? null : componentId}
   color="var(--spectrum-global-color-static-blue-200)"
-  transition
+  zIndex={selected ? 890 : 910}
-  {zIndex}
   allowResizeAnchors
 />

packages/client/src/components/preview/Indicator.svelte

@@ -1,5 +1,4 @@
 <script>
-  import { fade } from "svelte/transition"
   import { Icon } from "@budibase/bbui"
 
   export let top
@@ -11,7 +10,6 @@
   export let color
   export let zIndex
   export let componentId
-  export let transition = false
   export let line = false
   export let alignRight = false
   export let showResizeAnchors = false
@@ -31,10 +29,6 @@
 </script>
 
 <div
-  in:fade={{
-    delay: transition ? 100 : 0,
-    duration: transition ? 100 : 0,
-  }}
   class="indicator"
   class:flipped
   class:line
@@ -127,10 +121,6 @@
     font-weight: 600;
   }
 
-  /* Icon styles */
-  .label :global(.spectrum-Icon + .text) {
-  }
-
   /* Anchor */
   .anchor {
     --size: 24px;

packages/client/src/components/preview/IndicatorSet.svelte

@@ -4,27 +4,39 @@
   import { domDebounce } from "utils/domDebounce"
   import { builderStore } from "stores"
 
-  export let componentId
+  export let componentId = null
-  export let color
+  export let color = null
-  export let transition
+  export let zIndex = 900
-  export let zIndex
   export let prefix = null
   export let allowResizeAnchors = false
 
-  let indicators = []
+  const errorColor = "var(--spectrum-global-color-static-red-600)"
+  const defaultState = () => ({
+    // Cached props
+    componentId,
+    color,
+    zIndex,
+    prefix,
+    allowResizeAnchors,
+
+    // Computed state
+    indicators: [],
+    text: null,
+    icon: null,
+    insideGrid: false,
+    error: false,
+  })
+
   let interval
-  let text
+  let state = defaultState()
-  let icon
+  let nextState = null
-  let insideGrid = false
-  let errorState = false
-
-  $: visibleIndicators = indicators.filter(x => x.visible)
-  $: offset = $builderStore.inBuilder ? 0 : 2
-
   let updating = false
   let observers = []
   let callbackCount = 0
-  let nextIndicators = []
+
+  $: visibleIndicators = state.indicators.filter(x => x.visible)
+  $: offset = $builderStore.inBuilder ? 0 : 2
+  $: $$props, debouncedUpdate()
 
   const checkInsideGrid = id => {
     const component = document.getElementsByClassName(id)[0]
@@ -44,10 +56,10 @@
     if (callbackCount >= observers.length) {
       return
     }
-    nextIndicators[idx].visible =
+    nextState.indicators[idx].visible =
-      nextIndicators[idx].insideSidePanel || entries[0].isIntersecting
+      nextState.indicators[idx].insideSidePanel || entries[0].isIntersecting
     if (++callbackCount === observers.length) {
-      indicators = nextIndicators
+      state = nextState
       updating = false
     }
   }
@@ -59,7 +71,7 @@
 
     // Sanity check
     if (!componentId) {
-      indicators = []
+      state = defaultState()
       return
     }
 
@@ -68,25 +80,25 @@
     callbackCount = 0
     observers.forEach(o => o.disconnect())
     observers = []
-    nextIndicators = []
+    nextState = defaultState()
 
     // Check if we're inside a grid
     if (allowResizeAnchors) {
-      insideGrid = checkInsideGrid(componentId)
+      nextState.insideGrid = checkInsideGrid(componentId)
     }
 
     // Determine next set of indicators
     const parents = document.getElementsByClassName(componentId)
     if (parents.length) {
-      text = parents[0].dataset.name
+      nextState.text = parents[0].dataset.name
-      if (prefix) {
+      if (nextState.prefix) {
-        text = `${prefix} ${text}`
+        nextState.text = `${nextState.prefix} ${nextState.text}`
       }
       if (parents[0].dataset.icon) {
-        icon = parents[0].dataset.icon
+        nextState.icon = parents[0].dataset.icon
       }
     }
-    errorState = parents?.[0]?.classList.contains("error")
+    nextState.error = parents?.[0]?.classList.contains("error")
 
     // Batch reads to minimize reflow
     const scrollX = window.scrollX
@@ -102,8 +114,9 @@
 
     // If there aren't any nodes then reset
     if (!children.length) {
-      indicators = []
+      state = defaultState()
       updating = false
+      return
     }
 
     const device = document.getElementById("app-root")
@@ -119,7 +132,7 @@
       observers.push(observer)
 
       const elBounds = child.getBoundingClientRect()
-      nextIndicators.push({
+      nextState.indicators.push({
         top: elBounds.top + scrollY - deviceBounds.top - offset,
         left: elBounds.left + scrollX - deviceBounds.left - offset,
         width: elBounds.width + 4,
@@ -144,20 +157,17 @@
   })
 </script>
 
-{#key componentId}
+{#each visibleIndicators as indicator, idx}
-  {#each visibleIndicators as indicator, idx}
+  <Indicator
-    <Indicator
+    top={indicator.top}
-      top={indicator.top}
+    left={indicator.left}
-      left={indicator.left}
+    width={indicator.width}
-      width={indicator.width}
+    height={indicator.height}
-      height={indicator.height}
+    text={idx === 0 ? state.text : null}
-      text={idx === 0 ? text : null}
+    icon={idx === 0 ? state.icon : null}
-      icon={idx === 0 ? icon : null}
+    showResizeAnchors={state.allowResizeAnchors && state.insideGrid}
-      showResizeAnchors={allowResizeAnchors && insideGrid}
+    color={state.error ? errorColor : state.color}
-      color={errorState ? "var(--spectrum-global-color-static-red-600)" : color}
+    componentId={state.componentId}
-      {componentId}
+    zIndex={state.zIndex}
-      {transition}
+  />
-      {zIndex}
+{/each}
-    />
-  {/each}
-{/key}

packages/client/src/components/preview/SelectionIndicator.svelte

@@ -10,7 +10,6 @@
 <IndicatorSet
   componentId={$builderStore.selectedComponentId}
   {color}
-  zIndex="910"
+  zIndex={900}
-  transition
   allowResizeAnchors
 />

packages/client/src/index.js

@@ -98,7 +98,7 @@ const loadBudibase = async () => {
         context: stringifiedContext,
       })
     } else if (type === "hover-component") {
-      hoverStore.actions.hoverComponent(data)
+      hoverStore.actions.hoverComponent(data, false)
     } else if (type === "builder-meta") {
       builderStore.actions.setMetadata(data)
     }

packages/client/src/sdk.js

@@ -34,6 +34,8 @@ import {
   LuceneUtils,
   Constants,
   RowUtils,
+  memo,
+  derivedMemo,
 } from "@budibase/frontend-core"
 
 export default {
@@ -71,6 +73,8 @@ export default {
   makePropSafe,
   createContextStore,
   generateGoldenSample: RowUtils.generateGoldenSample,
+  memo,
+  derivedMemo,
 
   // Components
   Provider,

packages/client/src/stores/hover.js

@@ -5,13 +5,27 @@ const createHoverStore = () => {
   const store = writable({
     hoveredComponentId: null,
   })
+  let hoverTimeout
 
-  const hoverComponent = id => {
+  const hoverComponent = (id, notifyBuilder = true) => {
+    clearTimeout(hoverTimeout)
+    if (id) {
+      processHover(id, notifyBuilder)
+    } else {
+      hoverTimeout = setTimeout(() => {
+        processHover(id, notifyBuilder)
+      }, 10)
+    }
+  }
+
+  const processHover = (id, notifyBuilder = true) => {
     if (id === get(store).hoveredComponentId) {
       return
     }
     store.set({ hoveredComponentId: id })
-    eventStore.actions.dispatchEvent("hover-component", { id })
+    if (notifyBuilder) {
+      eventStore.actions.dispatchEvent("hover-component", { id })
+    }
   }
 
   return {

packages/server/__mocks__/mongodb.ts

@@ -1,40 +0,0 @@
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-module MongoMock {
-  const mongodb: any = {}
-
-  mongodb.MongoClient = function () {
-    this.connect = jest.fn()
-    this.close = jest.fn()
-    this.insertOne = jest.fn()
-    this.insertMany = jest.fn(() => ({ toArray: () => [] }))
-    this.find = jest.fn(() => ({ toArray: () => [] }))
-    this.findOne = jest.fn()
-    this.findOneAndUpdate = jest.fn()
-    this.count = jest.fn()
-    this.deleteOne = jest.fn()
-    this.deleteMany = jest.fn(() => ({ toArray: () => [] }))
-    this.updateOne = jest.fn()
-    this.updateMany = jest.fn(() => ({ toArray: () => [] }))
-
-    this.collection = jest.fn(() => ({
-      insertOne: this.insertOne,
-      find: this.find,
-      insertMany: this.insertMany,
-      findOne: this.findOne,
-      findOneAndUpdate: this.findOneAndUpdate,
-      count: this.count,
-      deleteOne: this.deleteOne,
-      deleteMany: this.deleteMany,
-      updateOne: this.updateOne,
-      updateMany: this.updateMany,
-    }))
-
-    this.db = () => ({
-      collection: this.collection,
-    })
-  }
-
-  mongodb.ObjectId = jest.requireActual("mongodb").ObjectId
-
-  module.exports = mongodb
-}

packages/server/src/api/controllers/query/index.ts

@@ -6,7 +6,7 @@ import { invalidateCachedVariable } from "../../../threads/utils"
 import env from "../../../environment"
 import { events, context, utils, constants } from "@budibase/backend-core"
 import sdk from "../../../sdk"
-import { QueryEvent } from "../../../threads/definitions"
+import { QueryEvent, QueryEventParameters } from "../../../threads/definitions"
 import {
   ConfigType,
   Query,
@@ -18,7 +18,6 @@ import {
   FieldType,
   ExecuteQueryRequest,
   ExecuteQueryResponse,
-  QueryParameter,
   PreviewQueryRequest,
   PreviewQueryResponse,
 } from "@budibase/types"
@@ -29,7 +28,7 @@ const Runner = new Thread(ThreadType.QUERY, {
   timeoutMs: env.QUERY_THREAD_TIMEOUT,
 })
 
-function validateQueryInputs(parameters: Record<string, string>) {
+function validateQueryInputs(parameters: QueryEventParameters) {
   for (let entry of Object.entries(parameters)) {
     const [key, value] = entry
     if (typeof value !== "string") {
@@ -100,10 +99,18 @@ export async function save(ctx: UserCtx<Query, Query>) {
   const datasource = await sdk.datasources.get(query.datasourceId)
 
   let eventFn
-  if (!query._id) {
+  if (!query._id && !query._rev) {
     query._id = generateQueryID(query.datasourceId)
+    // flag to state whether the default bindings are empty strings (old behaviour) or null
+    query.nullDefaultSupport = true
     eventFn = () => events.query.created(datasource, query)
   } else {
+    // check if flag has previously been set, don't let it change
+    // allow it to be explicitly set to false via API incase this is ever needed
+    const existingQuery = await db.get<Query>(query._id)
+    if (existingQuery.nullDefaultSupport && query.nullDefaultSupport == null) {
+      query.nullDefaultSupport = true
+    }
     eventFn = () => events.query.updated(datasource, query)
   }
   const response = await db.put(query)
@@ -135,16 +142,20 @@ function getAuthConfig(ctx: UserCtx) {
 }
 
 function enrichParameters(
-  queryParameters: QueryParameter[],
+  query: Query,
-  requestParameters: Record<string, string> = {}
+  requestParameters: QueryEventParameters = {}
-): Record<string, string> {
+): QueryEventParameters {
+  const paramNotSet = (val: unknown) => val === "" || val == undefined
   // first check parameters are all valid
   validateQueryInputs(requestParameters)
   // make sure parameters are fully enriched with defaults
-  for (let parameter of queryParameters) {
+  for (const parameter of query.parameters) {
-    if (!requestParameters[parameter.name]) {
+    let value: string | null =
-      requestParameters[parameter.name] = parameter.default
+      requestParameters[parameter.name] || parameter.default
+    if (query.nullDefaultSupport && paramNotSet(value)) {
+      value = null
     }
+    requestParameters[parameter.name] = value
   }
   return requestParameters
 }
@@ -157,10 +168,15 @@ export async function preview(
   )
   // preview may not have a queryId as it hasn't been saved, but if it does
   // this stops dynamic variables from calling the same query
-  const { fields, parameters, queryVerb, transformer, queryId, schema } =
+  const queryId = ctx.request.body.queryId
-    ctx.request.body
+  // the body contains the makings of a query, which has not been saved yet
+  const query: Query = ctx.request.body
+  // hasn't been saved, new query
+  if (!queryId && !query._id) {
+    query.nullDefaultSupport = true
+  }
 
-  let existingSchema = schema
+  let existingSchema = query.schema
   if (queryId && !existingSchema) {
     try {
       const db = context.getAppDB()
@@ -268,13 +284,14 @@ export async function preview(
   try {
     const inputs: QueryEvent = {
       appId: ctx.appId,
-      datasource,
+      queryVerb: query.queryVerb,
-      queryVerb,
+      fields: query.fields,
-      fields,
+      parameters: enrichParameters(query),
-      parameters: enrichParameters(parameters),
+      transformer: query.transformer,
-      transformer,
+      schema: query.schema,
+      nullDefaultSupport: query.nullDefaultSupport,
       queryId,
-      schema,
+      datasource,
       // have to pass down to the thread runner - can't put into context now
       environmentVariables: envVars,
       ctx: {
@@ -336,14 +353,12 @@ async function execute(
       queryVerb: query.queryVerb,
       fields: query.fields,
       pagination: ctx.request.body.pagination,
-      parameters: enrichParameters(
+      parameters: enrichParameters(query, ctx.request.body.parameters),
-        query.parameters,
-        ctx.request.body.parameters
-      ),
       transformer: query.transformer,
       queryId: ctx.params.queryId,
       // have to pass down to the thread runner - can't put into context now
       environmentVariables: envVars,
+      nullDefaultSupport: query.nullDefaultSupport,
       ctx: {
         user: ctx.user,
         auth: { ...authConfigCtx },

packages/server/src/api/controllers/table/utils.ts

@@ -30,6 +30,8 @@ import {
   View,
   RelationshipFieldMetadata,
   FieldType,
+  FieldTypeSubtypes,
+  AttachmentFieldMetadata,
 } from "@budibase/types"
 
 export async function clearColumns(table: Table, columnNames: string[]) {
@@ -88,6 +90,27 @@ export async function checkForColumnUpdates(
     // Update views
     await checkForViewUpdates(updatedTable, deletedColumns, columnRename)
   }
+
+  const changedAttachmentSubtypeColumns = Object.values(
+    updatedTable.schema
+  ).filter(
+    (column): column is AttachmentFieldMetadata =>
+      column.type === FieldType.ATTACHMENT &&
+      column.subtype !== oldTable?.schema[column.name]?.subtype
+  )
+  for (const attachmentColumn of changedAttachmentSubtypeColumns) {
+    if (attachmentColumn.subtype === FieldTypeSubtypes.ATTACHMENT.SINGLE) {
+      attachmentColumn.constraints ??= { length: {} }
+      attachmentColumn.constraints.length ??= {}
+      attachmentColumn.constraints.length.maximum = 1
+      attachmentColumn.constraints.length.message =
+        "cannot contain multiple files"
+    } else {
+      delete attachmentColumn.constraints?.length?.maximum
+      delete attachmentColumn.constraints?.length?.message
+    }
+  }
+
   return { rows: updatedRows, table: updatedTable }
 }
 

packages/server/src/api/routes/tests/environmentVariables.spec.ts

@@ -143,7 +143,10 @@ describe("/api/env/variables", () => {
     delete response.body.datasource.config
     expect(events.query.previewed).toHaveBeenCalledWith(
       response.body.datasource,
-      queryPreview
+      {
+        ...queryPreview,
+        nullDefaultSupport: true,
+      }
     )
     expect(pg.Client).toHaveBeenCalledWith({ password: "test", ssl: undefined })
   })

packages/server/src/api/routes/tests/queries/generic-sql.spec.ts

@@ -18,19 +18,22 @@ const createTableSQL: Record<string, string> = {
     CREATE TABLE test_table (
         id serial PRIMARY KEY,
         name VARCHAR ( 50 ) NOT NULL,
-        birthday TIMESTAMP
+        birthday TIMESTAMP,
+        number INT
     );`,
   [SourceName.MYSQL]: `
     CREATE TABLE test_table (
         id INT AUTO_INCREMENT PRIMARY KEY,
         name VARCHAR(50) NOT NULL,
-        birthday TIMESTAMP
+        birthday TIMESTAMP,
+        number INT
     );`,
   [SourceName.SQL_SERVER]: `
     CREATE TABLE test_table (
         id INT IDENTITY(1,1) PRIMARY KEY,
         name NVARCHAR(50) NOT NULL,
-        birthday DATETIME
+        birthday DATETIME,
+        number INT
     );`,
 }
 
@@ -42,7 +45,7 @@ describe.each([
   ["mysql", databaseTestProviders.mysql],
   ["mssql", databaseTestProviders.mssql],
   ["mariadb", databaseTestProviders.mariadb],
-])("queries (%s)", (__, dsProvider) => {
+])("queries (%s)", (dbName, dsProvider) => {
   const config = setup.getConfig()
   let datasource: Datasource
 
@@ -576,26 +579,31 @@ describe.each([
             id: 1,
             name: "one",
             birthday: null,
+            number: null,
           },
           {
             id: 2,
             name: "two",
             birthday: null,
+            number: null,
           },
           {
             id: 3,
             name: "three",
             birthday: null,
+            number: null,
           },
           {
             id: 4,
             name: "four",
             birthday: null,
+            number: null,
           },
           {
             id: 5,
             name: "five",
             birthday: null,
+            number: null,
           },
         ])
       })
@@ -606,9 +614,9 @@ describe.each([
             sql: "SELECT * FROM test_table WHERE id = 1",
           },
           transformer: `
-        data[0].id = data[0].id + 1;
+            data[0].id = data[0].id + 1;
-        return data;
+            return data;
-      `,
+          `,
         })
 
         const result = await config.api.query.execute(query._id!)
@@ -646,6 +654,7 @@ describe.each([
             id: 1,
             name: "one",
             birthday: null,
+            number: null,
           },
         ])
       })
@@ -777,6 +786,79 @@ describe.each([
         id: 2,
         name: "two",
       })
+
+      const rows = await rawQuery("SELECT * FROM test_table WHERE id = 1")
+      expect(rows).toEqual([
+        { id: 1, name: "foo", birthday: null, number: null },
+      ])
+    })
+
+    it("should be able to execute an update that updates no rows", async () => {
+      const query = await createQuery({
+        fields: {
+          sql: "UPDATE test_table SET name = 'updated' WHERE id = 100",
+        },
+        queryVerb: "update",
+      })
+
+      const result = await config.api.query.execute(query._id!, {})
+
+      expect(result.data).toEqual([
+        {
+          updated: true,
+        },
+      ])
+    })
+  })
+
+  // this parameter really only impacts SQL queries
+  describe("confirm nullDefaultSupport", () => {
+    const queryParams = {
+      fields: {
+        sql: "INSERT INTO test_table (name, number) VALUES ({{ bindingName }}, {{ bindingNumber }})",
+      },
+      parameters: [
+        {
+          name: "bindingName",
+          default: "",
+        },
+        {
+          name: "bindingNumber",
+          default: "",
+        },
+      ],
+      queryVerb: "create",
+    }
+
+    it("should error for old queries", async () => {
+      const query = await createQuery(queryParams)
+      await config.api.query.save({ ...query, nullDefaultSupport: false })
+      let error: string | undefined
+      try {
+        await config.api.query.execute(query._id!, {
+          parameters: {
+            bindingName: "testing",
+          },
+        })
+      } catch (err: any) {
+        error = err.message
+      }
+      if (dbName === "mssql") {
+        expect(error).toBeUndefined()
+      } else {
+        expect(error).toBeDefined()
+        expect(error).toContain("integer")
+      }
+    })
+
+    it("should not error for new queries", async () => {
+      const query = await createQuery(queryParams)
+      const results = await config.api.query.execute(query._id!, {
+        parameters: {
+          bindingName: "testing",
+        },
+      })
+      expect(results).toEqual({ data: [{ created: true }] })
     })
   })
 })

packages/server/src/api/routes/tests/queries/mongodb.spec.ts

@@ -4,8 +4,6 @@ import { databaseTestProviders } from "../../../../integrations/tests/utils"
 import { MongoClient, type Collection, BSON } from "mongodb"
 import { generator } from "@budibase/backend-core/tests"
 
-jest.unmock("mongodb")
-
 const collection = "test_collection"
 
 const expectValidId = expect.stringMatching(/^\w{24}$/)
@@ -249,6 +247,44 @@ describe("/queries", () => {
       expect(result.data).toEqual([{ value: 5 }])
     })
 
+    it("should be able to updateOne by ObjectId", async () => {
+      const insertResult = await withCollection(c =>
+        c.insertOne({ name: "one" })
+      )
+      const query = await createQuery({
+        fields: {
+          json: {
+            filter: { _id: { $eq: `ObjectId("${insertResult.insertedId}")` } },
+            update: { $set: { name: "newName" } },
+          },
+          extra: {
+            actionType: "updateOne",
+          },
+        },
+        queryVerb: "update",
+      })
+
+      const result = await config.api.query.execute(query._id!)
+
+      expect(result.data).toEqual([
+        {
+          acknowledged: true,
+          matchedCount: 1,
+          modifiedCount: 1,
+          upsertedCount: 0,
+          upsertedId: null,
+        },
+      ])
+
+      await withCollection(async collection => {
+        const doc = await collection.findOne({ name: { $eq: "newName" } })
+        expect(doc).toEqual({
+          _id: insertResult.insertedId,
+          name: "newName",
+        })
+      })
+    })
+
     it("a count query with a transformer", async () => {
       const query = await createQuery({
         fields: {
@@ -535,4 +571,85 @@ describe("/queries", () => {
       })
     })
   })
+
+  it("should throw an error if the incorrect actionType is specified", async () => {
+    const verbs = ["read", "create", "update", "delete"]
+    for (const verb of verbs) {
+      const query = await createQuery({
+        fields: { json: {}, extra: { actionType: "invalid" } },
+        queryVerb: verb,
+      })
+      await config.api.query.execute(query._id!, undefined, { status: 400 })
+    }
+  })
+
+  it("should ignore extra brackets in query", async () => {
+    const query = await createQuery({
+      fields: {
+        json: { foo: "te}st" },
+        extra: {
+          actionType: "insertOne",
+        },
+      },
+      queryVerb: "create",
+    })
+
+    const result = await config.api.query.execute(query._id!)
+    expect(result.data).toEqual([
+      {
+        acknowledged: true,
+        insertedId: expectValidId,
+      },
+    ])
+
+    await withCollection(async collection => {
+      const doc = await collection.findOne({ foo: { $eq: "te}st" } })
+      expect(doc).toEqual({
+        _id: expectValidBsonObjectId,
+        foo: "te}st",
+      })
+    })
+  })
+
+  it("should be able to save deeply nested data", async () => {
+    const data = {
+      foo: "bar",
+      data: [
+        { cid: 1 },
+        { cid: 2 },
+        {
+          nested: {
+            name: "test",
+            ary: [1, 2, 3],
+            aryOfObjects: [{ a: 1 }, { b: 2 }],
+          },
+        },
+      ],
+    }
+    const query = await createQuery({
+      fields: {
+        json: data,
+        extra: {
+          actionType: "insertOne",
+        },
+      },
+      queryVerb: "create",
+    })
+
+    const result = await config.api.query.execute(query._id!)
+    expect(result.data).toEqual([
+      {
+        acknowledged: true,
+        insertedId: expectValidId,
+      },
+    ])
+
+    await withCollection(async collection => {
+      const doc = await collection.findOne({ foo: { $eq: "bar" } })
+      expect(doc).toEqual({
+        _id: expectValidBsonObjectId,
+        ...data,
+      })
+    })
+  })
 })

packages/server/src/api/routes/tests/queries/query.seq.spec.ts

@@ -0,0 +1,774 @@
+import tk from "timekeeper"
+
+const pg = require("pg")
+
+// Mock out postgres for this
+jest.mock("pg")
+jest.mock("node-fetch")
+
+// Mock isProdAppID to we can later mock the implementation and pretend we are
+// using prod app IDs
+jest.mock("@budibase/backend-core", () => {
+  const core = jest.requireActual("@budibase/backend-core")
+  return {
+    ...core,
+    db: {
+      ...core.db,
+      isProdAppID: jest.fn(),
+    },
+  }
+})
+import * as setup from "../utilities"
+import { checkBuilderEndpoint } from "../utilities/TestFunctions"
+import { checkCacheForDynamicVariable } from "../../../../threads/utils"
+
+const { basicQuery, basicDatasource } = setup.structures
+import { events, db as dbCore } from "@budibase/backend-core"
+import {
+  Datasource,
+  Query,
+  SourceName,
+  QueryPreview,
+  QueryParameter,
+} from "@budibase/types"
+
+tk.freeze(Date.now())
+
+const mockIsProdAppID = dbCore.isProdAppID as jest.MockedFunction<
+  typeof dbCore.isProdAppID
+>
+
+describe("/queries", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+  let datasource: Datasource & Required<Pick<Datasource, "_id">>, query: Query
+
+  afterAll(setup.afterAll)
+
+  const setupTest = async () => {
+    await config.init()
+    datasource = await config.createDatasource()
+    query = await config.createQuery()
+  }
+
+  beforeAll(async () => {
+    await setupTest()
+  })
+
+  const createQuery = async (query: Query) => {
+    return request
+      .post(`/api/queries`)
+      .send(query)
+      .set(config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+  }
+
+  describe("create", () => {
+    it("should create a new query", async () => {
+      const { _id } = await config.createDatasource()
+      const query = basicQuery(_id)
+      jest.clearAllMocks()
+      const res = await createQuery(query)
+
+      expect((res as any).res.statusMessage).toEqual(
+        `Query ${query.name} saved successfully.`
+      )
+      expect(res.body).toEqual({
+        _rev: res.body._rev,
+        _id: res.body._id,
+        ...query,
+        nullDefaultSupport: true,
+        createdAt: new Date().toISOString(),
+        updatedAt: new Date().toISOString(),
+      })
+      expect(events.query.created).toHaveBeenCalledTimes(1)
+      expect(events.query.updated).not.toHaveBeenCalled()
+    })
+  })
+
+  describe("update", () => {
+    it("should update query", async () => {
+      const { _id } = await config.createDatasource()
+      const query = basicQuery(_id)
+      const res = await createQuery(query)
+      jest.clearAllMocks()
+      query._id = res.body._id
+      query._rev = res.body._rev
+      await createQuery(query)
+
+      expect((res as any).res.statusMessage).toEqual(
+        `Query ${query.name} saved successfully.`
+      )
+      expect(res.body).toEqual({
+        _rev: res.body._rev,
+        _id: res.body._id,
+        ...query,
+        nullDefaultSupport: true,
+        createdAt: new Date().toISOString(),
+        updatedAt: new Date().toISOString(),
+      })
+      expect(events.query.created).not.toHaveBeenCalled()
+      expect(events.query.updated).toHaveBeenCalledTimes(1)
+    })
+  })
+
+  describe("fetch", () => {
+    beforeEach(async () => {
+      await setupTest()
+    })
+
+    it("returns all the queries from the server", async () => {
+      const res = await request
+        .get(`/api/queries`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+
+      const queries = res.body
+      expect(queries).toEqual([
+        {
+          _rev: query._rev,
+          _id: query._id,
+          createdAt: new Date().toISOString(),
+          ...basicQuery(datasource._id),
+          nullDefaultSupport: true,
+          updatedAt: new Date().toISOString(),
+          readable: true,
+        },
+      ])
+    })
+
+    it("should apply authorization to endpoint", async () => {
+      await checkBuilderEndpoint({
+        config,
+        method: "GET",
+        url: `/api/datasources`,
+      })
+    })
+  })
+
+  describe("find", () => {
+    it("should find a query in builder", async () => {
+      const query = await config.createQuery()
+      const res = await request
+        .get(`/api/queries/${query._id}`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      expect(res.body._id).toEqual(query._id)
+    })
+
+    it("should find a query in cloud", async () => {
+      await config.withEnv({ SELF_HOSTED: "true" }, async () => {
+        const query = await config.createQuery()
+        const res = await request
+          .get(`/api/queries/${query._id}`)
+          .set(await config.defaultHeaders())
+          .expect(200)
+          .expect("Content-Type", /json/)
+        expect(res.body.fields).toBeDefined()
+        expect(res.body.parameters).toBeDefined()
+        expect(res.body.schema).toBeDefined()
+      })
+    })
+
+    it("should remove sensitive info for prod apps", async () => {
+      // Mock isProdAppID to pretend we are using a prod app
+      mockIsProdAppID.mockClear()
+      mockIsProdAppID.mockImplementation(() => true)
+
+      const query = await config.createQuery()
+      const res = await request
+        .get(`/api/queries/${query._id}`)
+        .set(await config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      expect(res.body._id).toEqual(query._id)
+      expect(res.body.fields).toBeUndefined()
+      expect(res.body.parameters).toBeUndefined()
+      expect(res.body.schema).toBeDefined()
+
+      // Reset isProdAppID mock
+      expect(dbCore.isProdAppID).toHaveBeenCalledTimes(1)
+      mockIsProdAppID.mockImplementation(() => false)
+    })
+  })
+
+  describe("destroy", () => {
+    beforeEach(async () => {
+      await setupTest()
+    })
+
+    it("deletes a query and returns a success message", async () => {
+      await request
+        .delete(`/api/queries/${query._id}/${query._rev}`)
+        .set(config.defaultHeaders())
+        .expect(200)
+
+      const res = await request
+        .get(`/api/queries`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+
+      expect(res.body).toEqual([])
+      expect(events.query.deleted).toHaveBeenCalledTimes(1)
+      expect(events.query.deleted).toHaveBeenCalledWith(datasource, query)
+    })
+
+    it("should apply authorization to endpoint", async () => {
+      const query = await config.createQuery()
+      await checkBuilderEndpoint({
+        config,
+        method: "DELETE",
+        url: `/api/queries/${query._id}/${query._rev}`,
+      })
+    })
+  })
+
+  describe("preview", () => {
+    it("should be able to preview the query", async () => {
+      const queryPreview: QueryPreview = {
+        datasourceId: datasource._id,
+        queryVerb: "read",
+        fields: {},
+        parameters: [],
+        transformer: "return data",
+        name: datasource.name!,
+        schema: {},
+        readable: true,
+      }
+      const responseBody = await config.api.query.previewQuery(queryPreview)
+      // these responses come from the mock
+      expect(responseBody.schema).toEqual({
+        a: { type: "string", name: "a" },
+        b: { type: "number", name: "b" },
+      })
+      expect(responseBody.rows.length).toEqual(1)
+      expect(events.query.previewed).toHaveBeenCalledTimes(1)
+      delete datasource.config
+      expect(events.query.previewed).toHaveBeenCalledWith(datasource, {
+        ...queryPreview,
+        nullDefaultSupport: true,
+      })
+    })
+
+    it("should apply authorization to endpoint", async () => {
+      await checkBuilderEndpoint({
+        config,
+        method: "POST",
+        url: `/api/queries/preview`,
+      })
+    })
+
+    it("should not error when trying to generate a nested schema for an empty array", async () => {
+      const queryPreview: QueryPreview = {
+        datasourceId: datasource._id,
+        parameters: [],
+        fields: {},
+        queryVerb: "read",
+        name: datasource.name!,
+        transformer: "return data",
+        schema: {},
+        readable: true,
+      }
+      const rows = [
+        {
+          contacts: [],
+        },
+      ]
+      pg.queryMock.mockImplementation(() => ({
+        rows,
+      }))
+
+      const responseBody = await config.api.query.previewQuery(queryPreview)
+      expect(responseBody).toEqual({
+        nestedSchemaFields: {},
+        rows,
+        schema: {
+          contacts: { type: "array", name: "contacts" },
+        },
+      })
+      expect(responseBody.rows.length).toEqual(1)
+      delete datasource.config
+    })
+
+    it("should generate a nested schema based on all the nested items", async () => {
+      const queryPreview: QueryPreview = {
+        datasourceId: datasource._id,
+        parameters: [],
+        fields: {},
+        queryVerb: "read",
+        name: datasource.name!,
+        transformer: "return data",
+        schema: {},
+        readable: true,
+      }
+      const rows = [
+        {
+          contacts: [
+            {
+              address: "123 Lane",
+            },
+            {
+              address: "456 Drive",
+            },
+            {
+              postcode: "BT1 12N",
+              lat: 54.59,
+              long: -5.92,
+            },
+            {
+              city: "Belfast",
+            },
+            {
+              address: "789 Avenue",
+              phoneNumber: "0800-999-5555",
+            },
+            {
+              name: "Name",
+              isActive: false,
+            },
+          ],
+        },
+      ]
+
+      pg.queryMock.mockImplementation(() => ({
+        rows,
+      }))
+
+      const responseBody = await config.api.query.previewQuery(queryPreview)
+      expect(responseBody).toEqual({
+        nestedSchemaFields: {
+          contacts: {
+            address: {
+              type: "string",
+              name: "address",
+            },
+            postcode: {
+              type: "string",
+              name: "postcode",
+            },
+            lat: {
+              type: "number",
+              name: "lat",
+            },
+            long: {
+              type: "number",
+              name: "long",
+            },
+            city: {
+              type: "string",
+              name: "city",
+            },
+            phoneNumber: {
+              type: "string",
+              name: "phoneNumber",
+            },
+            name: {
+              type: "string",
+              name: "name",
+            },
+            isActive: {
+              type: "boolean",
+              name: "isActive",
+            },
+          },
+        },
+        rows,
+        schema: {
+          contacts: { type: "json", name: "contacts", subtype: "array" },
+        },
+      })
+      expect(responseBody.rows.length).toEqual(1)
+      delete datasource.config
+    })
+  })
+
+  describe("execute", () => {
+    beforeEach(async () => {
+      await setupTest()
+    })
+
+    it("should be able to execute the query", async () => {
+      const res = await request
+        .post(`/api/queries/${query._id}`)
+        .send({
+          parameters: {},
+        })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      expect(res.body.length).toEqual(1)
+    })
+
+    it("should fail with invalid integration type", async () => {
+      const datasource: Datasource = {
+        ...basicDatasource().datasource,
+        source: "INVALID_INTEGRATION" as SourceName,
+      }
+      await config.api.datasource.create(datasource, {
+        status: 500,
+        body: {
+          message: "No datasource implementation found.",
+        },
+      })
+    })
+
+    it("shouldn't allow handlebars to be passed as parameters", async () => {
+      const res = await request
+        .post(`/api/queries/${query._id}`)
+        .send({
+          parameters: {
+            a: "{{ 'test' }}",
+          },
+        })
+        .set(config.defaultHeaders())
+        .expect(400)
+      expect(res.body.message).toEqual(
+        "Parameter 'a' input contains a handlebars binding - this is not allowed."
+      )
+    })
+  })
+
+  describe("variables", () => {
+    async function preview(datasource: Datasource, fields: any) {
+      const queryPreview: QueryPreview = {
+        datasourceId: datasource._id!,
+        parameters: [],
+        fields,
+        queryVerb: "read",
+        name: datasource.name!,
+        transformer: "return data",
+        schema: {},
+        readable: true,
+      }
+      return await config.api.query.previewQuery(queryPreview)
+    }
+
+    it("should work with static variables", async () => {
+      const datasource = await config.restDatasource({
+        staticVariables: {
+          variable: "google",
+          variable2: "1",
+        },
+      })
+      const responseBody = await preview(datasource, {
+        path: "www.{{ variable }}.com",
+        queryString: "test={{ variable2 }}",
+      })
+      // these responses come from the mock
+      expect(responseBody.schema).toEqual({
+        opts: { type: "json", name: "opts" },
+        url: { type: "string", name: "url" },
+        value: { type: "string", name: "value" },
+      })
+      expect(responseBody.rows[0].url).toEqual("http://www.google.com?test=1")
+    })
+
+    it("should work with dynamic variables", async () => {
+      const { datasource } = await config.dynamicVariableDatasource()
+      const responseBody = await preview(datasource, {
+        path: "www.google.com",
+        queryString: "test={{ variable3 }}",
+      })
+      expect(responseBody.schema).toEqual({
+        opts: { type: "json", name: "opts" },
+        url: { type: "string", name: "url" },
+        value: { type: "string", name: "value" },
+      })
+      expect(responseBody.rows[0].url).toContain("doctype%20html")
+    })
+
+    it("check that it automatically retries on fail with cached dynamics", async () => {
+      const { datasource, query: base } =
+        await config.dynamicVariableDatasource()
+      // preview once to cache
+      await preview(datasource, {
+        path: "www.google.com",
+        queryString: "test={{ variable3 }}",
+      })
+      // check its in cache
+      const contents = await checkCacheForDynamicVariable(
+        base._id!,
+        "variable3"
+      )
+      expect(contents.rows.length).toEqual(1)
+      const responseBody = await preview(datasource, {
+        path: "www.failonce.com",
+        queryString: "test={{ variable3 }}",
+      })
+      expect(responseBody.schema).toEqual({
+        fails: { type: "number", name: "fails" },
+        opts: { type: "json", name: "opts" },
+        url: { type: "string", name: "url" },
+      })
+      expect(responseBody.rows[0].fails).toEqual(1)
+    })
+
+    it("deletes variables when linked query is deleted", async () => {
+      const { datasource, query: base } =
+        await config.dynamicVariableDatasource()
+      // preview once to cache
+      await preview(datasource, {
+        path: "www.google.com",
+        queryString: "test={{ variable3 }}",
+      })
+      // check its in cache
+      let contents = await checkCacheForDynamicVariable(base._id!, "variable3")
+      expect(contents.rows.length).toEqual(1)
+
+      // delete the query
+      await request
+        .delete(`/api/queries/${base._id}/${base._rev}`)
+        .set(config.defaultHeaders())
+        .expect(200)
+
+      // check variables no longer in cache
+      contents = await checkCacheForDynamicVariable(base._id!, "variable3")
+      expect(contents).toBe(null)
+    })
+  })
+
+  describe("Current User Request Mapping", () => {
+    async function previewGet(
+      datasource: Datasource,
+      fields: any,
+      params: QueryParameter[]
+    ) {
+      const queryPreview: QueryPreview = {
+        datasourceId: datasource._id!,
+        parameters: params,
+        fields,
+        queryVerb: "read",
+        name: datasource.name!,
+        transformer: "return data",
+        schema: {},
+        readable: true,
+      }
+      return await config.api.query.previewQuery(queryPreview)
+    }
+
+    async function previewPost(
+      datasource: Datasource,
+      fields: any,
+      params: QueryParameter[]
+    ) {
+      const queryPreview: QueryPreview = {
+        datasourceId: datasource._id!,
+        parameters: params,
+        fields,
+        queryVerb: "create",
+        name: datasource.name!,
+        transformer: null,
+        schema: {},
+        readable: false,
+      }
+      return await config.api.query.previewQuery(queryPreview)
+    }
+
+    it("should parse global and query level header mappings", async () => {
+      const userDetails = config.getUserDetails()
+
+      const datasource = await config.restDatasource({
+        defaultHeaders: {
+          test: "headerVal",
+          emailHdr: "{{[user].[email]}}",
+        },
+      })
+      const responseBody = await previewGet(
+        datasource,
+        {
+          path: "www.google.com",
+          queryString: "email={{[user].[email]}}",
+          headers: {
+            queryHdr: "{{[user].[firstName]}}",
+            secondHdr: "1234",
+          },
+        },
+        []
+      )
+
+      const parsedRequest = JSON.parse(responseBody.extra.raw)
+      expect(parsedRequest.opts.headers).toEqual({
+        test: "headerVal",
+        emailHdr: userDetails.email,
+        queryHdr: userDetails.firstName,
+        secondHdr: "1234",
+      })
+      expect(responseBody.rows[0].url).toEqual(
+        "http://www.google.com?email=" + userDetails.email.replace("@", "%40")
+      )
+    })
+
+    it("should bind the current user to query parameters", async () => {
+      const userDetails = config.getUserDetails()
+
+      const datasource = await config.restDatasource()
+
+      const responseBody = await previewGet(
+        datasource,
+        {
+          path: "www.google.com",
+          queryString:
+            "test={{myEmail}}&testName={{myName}}&testParam={{testParam}}",
+        },
+        [
+          { name: "myEmail", default: "{{[user].[email]}}" },
+          { name: "myName", default: "{{[user].[firstName]}}" },
+          { name: "testParam", default: "1234" },
+        ]
+      )
+
+      expect(responseBody.rows[0].url).toEqual(
+        "http://www.google.com?test=" +
+          userDetails.email.replace("@", "%40") +
+          "&testName=" +
+          userDetails.firstName +
+          "&testParam=1234"
+      )
+    })
+
+    it("should bind the current user the request body - plain text", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+
+      const responseBody = await previewPost(
+        datasource,
+        {
+          path: "www.google.com",
+          queryString: "testParam={{testParam}}",
+          requestBody:
+            "This is plain text and this is my email: {{[user].[email]}}. This is a test param: {{testParam}}",
+          bodyType: "text",
+        },
+        [{ name: "testParam", default: "1234" }]
+      )
+
+      const parsedRequest = JSON.parse(responseBody.extra.raw)
+      expect(parsedRequest.opts.body).toEqual(
+        `This is plain text and this is my email: ${userDetails.email}. This is a test param: 1234`
+      )
+      expect(responseBody.rows[0].url).toEqual(
+        "http://www.google.com?testParam=1234"
+      )
+    })
+
+    it("should bind the current user the request body - json", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+
+      const responseBody = await previewPost(
+        datasource,
+        {
+          path: "www.google.com",
+          queryString: "testParam={{testParam}}",
+          requestBody:
+            '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}',
+          bodyType: "json",
+        },
+        [
+          { name: "testParam", default: "1234" },
+          { name: "userRef", default: "{{[user].[firstName]}}" },
+        ]
+      )
+
+      const parsedRequest = JSON.parse(responseBody.extra.raw)
+      const test = `{"email":"${userDetails.email}","queryCode":1234,"userRef":"${userDetails.firstName}"}`
+      expect(parsedRequest.opts.body).toEqual(test)
+      expect(responseBody.rows[0].url).toEqual(
+        "http://www.google.com?testParam=1234"
+      )
+    })
+
+    it("should bind the current user the request body - xml", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+
+      const responseBody = await previewPost(
+        datasource,
+        {
+          path: "www.google.com",
+          queryString: "testParam={{testParam}}",
+          requestBody:
+            "<note> <email>{{[user].[email]}}</email> <code>{{testParam}}</code> " +
+            "<ref>{{userId}}</ref> <somestring>testing</somestring> </note>",
+          bodyType: "xml",
+        },
+        [
+          { name: "testParam", default: "1234" },
+          { name: "userId", default: "{{[user].[firstName]}}" },
+        ]
+      )
+
+      const parsedRequest = JSON.parse(responseBody.extra.raw)
+      const test = `<note> <email>${userDetails.email}</email> <code>1234</code> <ref>${userDetails.firstName}</ref> <somestring>testing</somestring> </note>`
+
+      expect(parsedRequest.opts.body).toEqual(test)
+      expect(responseBody.rows[0].url).toEqual(
+        "http://www.google.com?testParam=1234"
+      )
+    })
+
+    it("should bind the current user the request body - form-data", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+
+      const responseBody = await previewPost(
+        datasource,
+        {
+          path: "www.google.com",
+          queryString: "testParam={{testParam}}",
+          requestBody:
+            '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}',
+          bodyType: "form",
+        },
+        [
+          { name: "testParam", default: "1234" },
+          { name: "userRef", default: "{{[user].[firstName]}}" },
+        ]
+      )
+
+      const parsedRequest = JSON.parse(responseBody.extra.raw)
+
+      const emailData = parsedRequest.opts.body._streams[1]
+      expect(emailData).toEqual(userDetails.email)
+
+      const queryCodeData = parsedRequest.opts.body._streams[4]
+      expect(queryCodeData).toEqual("1234")
+
+      const userRef = parsedRequest.opts.body._streams[7]
+      expect(userRef).toEqual(userDetails.firstName)
+
+      expect(responseBody.rows[0].url).toEqual(
+        "http://www.google.com?testParam=1234"
+      )
+    })
+
+    it("should bind the current user the request body - encoded", async () => {
+      const userDetails = config.getUserDetails()
+      const datasource = await config.restDatasource()
+
+      const responseBody = await previewPost(
+        datasource,
+        {
+          path: "www.google.com",
+          queryString: "testParam={{testParam}}",
+          requestBody:
+            '{"email":"{{[user].[email]}}","queryCode":{{testParam}},"userRef":"{{userRef}}"}',
+          bodyType: "encoded",
+        },
+        [
+          { name: "testParam", default: "1234" },
+          { name: "userRef", default: "{{[user].[firstName]}}" },
+        ]
+      )
+      const parsedRequest = JSON.parse(responseBody.extra.raw)
+
+      expect(parsedRequest.opts.body.email).toEqual(userDetails.email)
+      expect(parsedRequest.opts.body.queryCode).toEqual("1234")
+      expect(parsedRequest.opts.body.userRef).toEqual(userDetails.firstName)
+    })
+  })
+})

packages/server/src/integrations/base/sqlTable.ts

@@ -60,7 +60,7 @@ function generateSchema(
         schema.text(key)
         break
       case FieldType.BB_REFERENCE: {
-        const subtype = column.subtype as FieldSubtype
+        const subtype = column.subtype
         switch (subtype) {
           case FieldSubtype.USER:
             schema.text(key)

packages/server/src/integrations/mongodb.ts

@@ -23,7 +23,7 @@ import {
 } from "mongodb"
 import environment from "../environment"
 
-interface MongoDBConfig {
+export interface MongoDBConfig {
   connectionString: string
   db: string
   tlsCertificateKeyFile: string
@@ -348,7 +348,7 @@ const getSchema = () => {
 
 const SCHEMA: Integration = getSchema()
 
-class MongoIntegration implements IntegrationBase {
+export class MongoIntegration implements IntegrationBase {
   private config: MongoDBConfig
   private client: MongoClient
 

packages/server/src/integrations/queries/sql.ts

@@ -5,9 +5,10 @@ import sdk from "../../sdk"
 const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g")
 
 export async function interpolateSQL(
-  fields: { [key: string]: any },
+  fields: { sql: string; bindings: any[] },
   parameters: { [key: string]: any },
-  integration: DatasourcePlus
+  integration: DatasourcePlus,
+  opts: { nullDefaultSupport: boolean }
 ) {
   let sql = fields.sql
   if (!sql || typeof sql !== "string") {
@@ -64,7 +65,14 @@ export async function interpolateSQL(
   }
   // replicate the knex structure
   fields.sql = sql
-  fields.bindings = await sdk.queries.enrichContext(variables, parameters)
+  fields.bindings = await sdk.queries.enrichArrayContext(variables, parameters)
+  if (opts.nullDefaultSupport) {
+    for (let index in fields.bindings) {
+      if (fields.bindings[index] === "") {
+        fields.bindings[index] = null
+      }
+    }
+  }
   // check for arrays in the data
   let updated: string[] = []
   for (let i = 0; i < variables.length; i++) {

packages/server/src/integrations/tests/mongo.spec.ts

@@ -1,325 +0,0 @@
-const mongo = require("mongodb")
-
-import { default as MongoDBIntegration } from "../mongodb"
-
-jest.mock("mongodb")
-
-class TestConfiguration {
-  integration: any
-
-  constructor(config: any = {}) {
-    this.integration = new MongoDBIntegration.integration(config)
-  }
-}
-
-describe("MongoDB Integration", () => {
-  let config: any
-  let indexName = "Users"
-
-  beforeEach(() => {
-    config = new TestConfiguration()
-  })
-
-  it("calls the create method with the correct params", async () => {
-    const body = {
-      name: "Hello",
-    }
-    await config.integration.create({
-      index: indexName,
-      json: body,
-      extra: { collection: "testCollection", actionType: "insertOne" },
-    })
-    expect(config.integration.client.insertOne).toHaveBeenCalledWith(body)
-  })
-
-  it("calls the read method with the correct params", async () => {
-    const query = {
-      json: {
-        address: "test",
-      },
-      extra: { collection: "testCollection", actionType: "find" },
-    }
-    const response = await config.integration.read(query)
-    expect(config.integration.client.find).toHaveBeenCalledWith(query.json)
-    expect(response).toEqual(expect.any(Array))
-  })
-
-  it("calls the delete method with the correct params", async () => {
-    const query = {
-      json: {
-        filter: {
-          id: "test",
-        },
-        options: {
-          opt: "option",
-        },
-      },
-      extra: { collection: "testCollection", actionType: "deleteOne" },
-    }
-    await config.integration.delete(query)
-    expect(config.integration.client.deleteOne).toHaveBeenCalledWith(
-      query.json.filter,
-      query.json.options
-    )
-  })
-
-  it("calls the update method with the correct params", async () => {
-    const query = {
-      json: {
-        filter: {
-          id: "test",
-        },
-        update: {
-          name: "TestName",
-        },
-        options: {
-          upsert: false,
-        },
-      },
-      extra: { collection: "testCollection", actionType: "updateOne" },
-    }
-    await config.integration.update(query)
-    expect(config.integration.client.updateOne).toHaveBeenCalledWith(
-      query.json.filter,
-      query.json.update,
-      query.json.options
-    )
-  })
-
-  it("throws an error when an invalid query.extra.actionType is passed for each method", async () => {
-    const query = {
-      extra: { collection: "testCollection", actionType: "deleteOne" },
-    }
-
-    let error = null
-    try {
-      await config.integration.read(query)
-    } catch (err) {
-      error = err
-    }
-    expect(error).toBeDefined()
-  })
-
-  it("creates ObjectIds if the field contains a match on ObjectId", async () => {
-    const query = {
-      json: {
-        filter: {
-          _id: "ObjectId('ACBD12345678ABCD12345678')",
-          name: "ObjectId('BBBB12345678ABCD12345678')",
-        },
-        update: {
-          _id: "ObjectId('FFFF12345678ABCD12345678')",
-          name: "ObjectId('CCCC12345678ABCD12345678')",
-        },
-        options: {
-          upsert: false,
-        },
-      },
-      extra: { collection: "testCollection", actionType: "updateOne" },
-    }
-    await config.integration.update(query)
-    expect(config.integration.client.updateOne).toHaveBeenCalled()
-
-    const args = config.integration.client.updateOne.mock.calls[0]
-    expect(args[0]).toEqual({
-      _id: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"),
-      name: mongo.ObjectId.createFromHexString("BBBB12345678ABCD12345678"),
-    })
-    expect(args[1]).toEqual({
-      _id: mongo.ObjectId.createFromHexString("FFFF12345678ABCD12345678"),
-      name: mongo.ObjectId.createFromHexString("CCCC12345678ABCD12345678"),
-    })
-    expect(args[2]).toEqual({
-      upsert: false,
-    })
-  })
-
-  it("creates ObjectIds if the $ operator fields contains a match on ObjectId", async () => {
-    const query = {
-      json: {
-        filter: {
-          _id: {
-            $eq: "ObjectId('ACBD12345678ABCD12345678')",
-          },
-        },
-        update: {
-          $set: {
-            _id: "ObjectId('FFFF12345678ABCD12345678')",
-          },
-        },
-        options: {
-          upsert: true,
-        },
-      },
-      extra: { collection: "testCollection", actionType: "updateOne" },
-    }
-    await config.integration.update(query)
-    expect(config.integration.client.updateOne).toHaveBeenCalled()
-
-    const args = config.integration.client.updateOne.mock.calls[0]
-    expect(args[0]).toEqual({
-      _id: {
-        $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"),
-      },
-    })
-    expect(args[1]).toEqual({
-      $set: {
-        _id: mongo.ObjectId.createFromHexString("FFFF12345678ABCD12345678"),
-      },
-    })
-    expect(args[2]).toEqual({
-      upsert: true,
-    })
-  })
-
-  it("supports findOneAndUpdate", async () => {
-    const query = {
-      json: {
-        filter: {
-          _id: {
-            $eq: "ObjectId('ACBD12345678ABCD12345678')",
-          },
-        },
-        update: {
-          $set: {
-            name: "UPDATED",
-            age: 99,
-          },
-        },
-        options: {
-          upsert: false,
-        },
-      },
-      extra: { collection: "testCollection", actionType: "findOneAndUpdate" },
-    }
-    await config.integration.read(query)
-    expect(config.integration.client.findOneAndUpdate).toHaveBeenCalled()
-
-    const args = config.integration.client.findOneAndUpdate.mock.calls[0]
-    expect(args[0]).toEqual({
-      _id: {
-        $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"),
-      },
-    })
-    expect(args[1]).toEqual({
-      $set: {
-        name: "UPDATED",
-        age: 99,
-      },
-    })
-    expect(args[2]).toEqual({
-      upsert: false,
-      includeResultMetadata: true,
-    })
-  })
-
-  it("can parse nested objects with arrays", async () => {
-    const query = {
-      json: `{
-          "_id": {
-            "$eq": "ObjectId('ACBD12345678ABCD12345678')"
-          }
-        },
-        {
-          "$set": {
-            "value": {
-              "data": [
-                { "cid": 1 },
-                { "cid": 2 },
-                { "nested": {
-                  "name": "test"
-                }}
-              ]
-            }
-          }
-        },
-        {
-          "upsert": true
-        }`,
-      extra: { collection: "testCollection", actionType: "updateOne" },
-    }
-    await config.integration.update(query)
-    expect(config.integration.client.updateOne).toHaveBeenCalled()
-
-    const args = config.integration.client.updateOne.mock.calls[0]
-    expect(args[0]).toEqual({
-      _id: {
-        $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"),
-      },
-    })
-    expect(args[1]).toEqual({
-      $set: {
-        value: {
-          data: [
-            { cid: 1 },
-            { cid: 2 },
-            {
-              nested: {
-                name: "test",
-              },
-            },
-          ],
-        },
-      },
-    })
-    expect(args[2]).toEqual({
-      upsert: true,
-    })
-  })
-
-  it("ignores braces within strings when parsing nested objects", async () => {
-    const query = {
-      json: `{
-          "_id": {
-            "$eq": "ObjectId('ACBD12345678ABCD12345678')"
-          }
-        },
-        {
-          "$set": {
-            "value": {
-              "data": [
-                { "cid": 1 },
-                { "cid": 2 },
-                { "nested": {
-                  "name": "te}st"
-                }}
-              ]
-            }
-          }
-        },
-        {
-          "upsert": true,
-          "extra": "ad\\"{\\"d"
-        }`,
-      extra: { collection: "testCollection", actionType: "updateOne" },
-    }
-    await config.integration.update(query)
-    expect(config.integration.client.updateOne).toHaveBeenCalled()
-
-    const args = config.integration.client.updateOne.mock.calls[0]
-    expect(args[0]).toEqual({
-      _id: {
-        $eq: mongo.ObjectId.createFromHexString("ACBD12345678ABCD12345678"),
-      },
-    })
-    expect(args[1]).toEqual({
-      $set: {
-        value: {
-          data: [
-            { cid: 1 },
-            { cid: 2 },
-            {
-              nested: {
-                name: "te}st",
-              },
-            },
-          ],
-        },
-      },
-    })
-    expect(args[2]).toEqual({
-      upsert: true,
-      extra: 'ad"{"d',
-    })
-  })
-})

packages/server/src/jsRunner/bundles/index-helpers.ts

@@ -1,6 +1,5 @@
-const {
+// eslint-disable-next-line local-rules/no-budibase-imports
-  getJsHelperList,
+import { getJsHelperList } from "@budibase/string-templates/src/helpers/list"
-} = require("../../../../string-templates/src/helpers/list.js")
 
 export default {
   ...getJsHelperList(),

packages/server/src/jsRunner/bundles/snippets.ivm.bundle.js

@@ -1,3 +1,3 @@
-"use strict";var snippets=(()=>{var u=Object.create;var n=Object.defineProperty;var a=Object.getOwnPropertyDescriptor;var h=Object.getOwnPropertyNames;var x=Object.getPrototypeOf,C=Object.prototype.hasOwnProperty;var l=(i,e)=>()=>(e||i((e={exports:{}}).exports,e),e.exports),W=(i,e)=>{for(var p in e)n(i,p,{get:e[p],enumerable:!0})},f=(i,e,p,r)=>{if(e&&typeof e=="object"||typeof e=="function")for(let t of h(e))!C.call(i,t)&&t!==p&&n(i,t,{get:()=>e[t],enumerable:!(r=a(e,t))||r.enumerable});return i};var d=(i,e,p)=>(p=i!=null?u(x(i)):{},f(e||!i||!i.__esModule?n(p,"default",{value:i,enumerable:!0}):p,i)),g=i=>f(n({},"__esModule",{value:!0}),i);var s=l((D,o)=>{o.exports.iifeWrapper=i=>`(function(){
+"use strict";var snippets=(()=>{var p=Object.defineProperty;var f=Object.getOwnPropertyDescriptor;var s=Object.getOwnPropertyNames;var c=Object.prototype.hasOwnProperty;var u=(i,e)=>{for(var n in e)p(i,n,{get:e[n],enumerable:!0})},a=(i,e,n,r)=>{if(e&&typeof e=="object"||typeof e=="function")for(let t of s(e))!c.call(i,t)&&t!==n&&p(i,t,{get:()=>e[t],enumerable:!(r=f(e,t))||r.enumerable});return i};var h=i=>a(p({},"__esModule",{value:!0}),i);var C={};u(C,{default:()=>x});var o=i=>`(function(){
 ${i}
-})();`});var w={};W(w,{default:()=>v});var c=d(s()),v=new Proxy({},{get:function(i,e){return e in snippetCache||(snippetCache[e]=[eval][0]((0,c.iifeWrapper)(snippetDefinitions[e]))),snippetCache[e]}});return g(w);})();
+})();`;var x=new Proxy({},{get:function(i,e){return e in snippetCache||(snippetCache[e]=[eval][0](o(snippetDefinitions[e]))),snippetCache[e]}});return h(C);})();

packages/server/src/sdk/app/queries/queries.ts

@@ -65,14 +65,33 @@ export async function fetch(opts: { enrich: boolean } = { enrich: true }) {
   return updateSchemas(queries)
 }
 
+export async function enrichArrayContext(
+  fields: any[],
+  inputs = {}
+): Promise<any[]> {
+  const map: Record<string, any> = {}
+  for (let index in fields) {
+    map[index] = fields[index]
+  }
+  const output = await enrichContext(map, inputs)
+  const outputArray: any[] = []
+  for (let [key, value] of Object.entries(output)) {
+    outputArray[parseInt(key)] = value
+  }
+  return outputArray
+}
+
 export async function enrichContext(
   fields: Record<string, any>,
   inputs = {}
 ): Promise<Record<string, any>> {
-  const enrichedQuery: Record<string, any> = Array.isArray(fields) ? [] : {}
+  const enrichedQuery: Record<string, any> = {}
   if (!fields || !inputs) {
     return enrichedQuery
   }
+  if (Array.isArray(fields)) {
+    return enrichArrayContext(fields, inputs)
+  }
   const env = await getEnvironmentVariables()
   const parameters = { ...inputs, env }
   // enrich the fields with dynamic parameters

packages/server/src/sdk/app/rows/search/utils.ts

@@ -67,7 +67,7 @@ export function searchInputMapping(table: Table, options: SearchParams) {
   for (let [key, column] of Object.entries(table.schema)) {
     switch (column.type) {
       case FieldType.BB_REFERENCE: {
-        const subtype = column.subtype as FieldSubtype
+        const subtype = column.subtype
         switch (subtype) {
           case FieldSubtype.USER:
           case FieldSubtype.USERS:

packages/server/src/threads/definitions.ts

@@ -1,21 +1,20 @@
-import { Datasource, QuerySchema, Row } from "@budibase/types"
+import { Datasource, Row, Query } from "@budibase/types"
 
 export type WorkerCallback = (error: any, response?: any) => void
 
-export interface QueryEvent {
+export interface QueryEvent
+  extends Omit<Query, "datasourceId" | "name" | "parameters" | "readable"> {
   appId?: string
   datasource: Datasource
-  queryVerb: string
-  fields: { [key: string]: any }
-  parameters: { [key: string]: unknown }
   pagination?: any
-  transformer: any
   queryId?: string
   environmentVariables?: Record<string, string>
+  parameters: QueryEventParameters
   ctx?: any
-  schema?: Record<string, QuerySchema | string>
 }
 
+export type QueryEventParameters = Record<string, string | null>
+
 export interface QueryResponse {
   rows: Row[]
   keys: string[]

packages/server/src/threads/query.ts

@@ -26,10 +26,11 @@ class QueryRunner {
   fields: any
   parameters: any
   pagination: any
-  transformer: string
+  transformer: string | null
   cachedVariables: any[]
   ctx: any
   queryResponse: any
+  nullDefaultSupport: boolean
   noRecursiveQuery: boolean
   hasRerun: boolean
   hasRefreshedOAuth: boolean
@@ -45,6 +46,7 @@ class QueryRunner {
     this.transformer = input.transformer
     this.queryId = input.queryId!
     this.schema = input.schema
+    this.nullDefaultSupport = !!input.nullDefaultSupport
     this.noRecursiveQuery = flags.noRecursiveQuery
     this.cachedVariables = []
     // Additional context items for enrichment
@@ -59,7 +61,14 @@ class QueryRunner {
   }
 
   async execute(): Promise<QueryResponse> {
-    let { datasource, fields, queryVerb, transformer, schema } = this
+    let {
+      datasource,
+      fields,
+      queryVerb,
+      transformer,
+      schema,
+      nullDefaultSupport,
+    } = this
     let datasourceClone = cloneDeep(datasource)
     let fieldsClone = cloneDeep(fields)
 
@@ -100,10 +109,12 @@ class QueryRunner {
       )
     }
 
-    let query
+    let query: Record<string, any>
     // handle SQL injections by interpolating the variables
     if (isSQL(datasourceClone)) {
-      query = await interpolateSQL(fieldsClone, enrichedContext, integration)
+      query = await interpolateSQL(fieldsClone, enrichedContext, integration, {
+        nullDefaultSupport,
+      })
     } else {
       query = await sdk.queries.enrichContext(fieldsClone, enrichedContext)
     }
@@ -137,7 +148,9 @@ class QueryRunner {
         data: rows,
         params: enrichedParameters,
       }
-      rows = vm.withContext(ctx, () => vm.execute(transformer))
+      if (transformer != null) {
+        rows = vm.withContext(ctx, () => vm.execute(transformer!))
+      }
     }
 
     // if the request fails we retry once, invalidating the cached value
@@ -191,13 +204,15 @@ class QueryRunner {
     })
     return new QueryRunner(
       {
-        datasource,
+        schema: query.schema,
         queryVerb: query.queryVerb,
         fields: query.fields,
-        parameters,
         transformer: query.transformer,
-        queryId,
+        nullDefaultSupport: query.nullDefaultSupport,
         ctx: this.ctx,
+        parameters,
+        datasource,
+        queryId,
       },
       { noRecursiveQuery: true }
     ).execute()

packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts

@@ -7,7 +7,7 @@ const ROW_PREFIX = DocumentType.ROW + SEPARATOR
 
 export async function processInputBBReferences(
   value: string | string[] | { _id: string } | { _id: string }[],
-  subtype: FieldSubtype
+  subtype: FieldSubtype.USER | FieldSubtype.USERS
 ): Promise<string | string[] | null> {
   let referenceIds: string[] = []
 
@@ -61,7 +61,7 @@ export async function processInputBBReferences(
 
 export async function processOutputBBReferences(
   value: string | string[],
-  subtype: FieldSubtype
+  subtype: FieldSubtype.USER | FieldSubtype.USERS
 ) {
   if (value === null || value === undefined) {
     // Already processed or nothing to process

packages/server/src/utilities/rowProcessor/index.ts

@@ -6,7 +6,6 @@ import { TYPE_TRANSFORM_MAP } from "./map"
 import {
   FieldType,
   AutoFieldSubType,
-  FieldSubtype,
   Row,
   RowAttachment,
   Table,
@@ -159,10 +158,7 @@ export async function inputProcessing(
     }
 
     if (field.type === FieldType.BB_REFERENCE && value) {
-      clonedRow[key] = await processInputBBReferences(
+      clonedRow[key] = await processInputBBReferences(value, field.subtype)
-        value,
-        field.subtype as FieldSubtype
-      )
     }
   }
 
@@ -238,7 +234,7 @@ export async function outputProcessing<T extends Row[] | Row>(
       for (let row of enriched) {
         row[property] = await processOutputBBReferences(
           row[property],
-          column.subtype as FieldSubtype
+          column.subtype
         )
       }
     }

packages/server/src/utilities/schema.ts

@@ -1,26 +1,14 @@
-import { FieldType, FieldSubtype } from "@budibase/types"
+import {
+  FieldType,
+  FieldSubtype,
+  TableSchema,
+  FieldSchema,
+  Row,
+} from "@budibase/types"
 import { ValidColumnNameRegex, utils } from "@budibase/shared-core"
 import { db } from "@budibase/backend-core"
 import { parseCsvExport } from "../api/controllers/view/exporters"
 
-interface SchemaColumn {
-  readonly name: string
-  readonly type: FieldType
-  readonly subtype: FieldSubtype
-  readonly autocolumn?: boolean
-  readonly constraints?: {
-    presence: boolean
-  }
-}
-
-interface Schema {
-  readonly [index: string]: SchemaColumn
-}
-
-interface Row {
-  [index: string]: any
-}
-
 type Rows = Array<Row>
 
 interface SchemaValidation {
@@ -34,12 +22,10 @@ interface ValidationResults {
   errors: Record<string, string>
 }
 
-export function isSchema(schema: any): schema is Schema {
+export function isSchema(schema: any): schema is TableSchema {
   return (
     typeof schema === "object" &&
-    Object.values(schema).every(rawColumn => {
+    Object.values<FieldSchema>(schema).every(column => {
-      const column = rawColumn as SchemaColumn
-
       return (
         column !== null &&
         typeof column === "object" &&
@@ -54,7 +40,7 @@ export function isRows(rows: any): rows is Rows {
   return Array.isArray(rows) && rows.every(row => typeof row === "object")
 }
 
-export function validate(rows: Rows, schema: Schema): ValidationResults {
+export function validate(rows: Rows, schema: TableSchema): ValidationResults {
   const results: ValidationResults = {
     schemaValidation: {},
     allValid: false,
@@ -64,9 +50,11 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
 
   rows.forEach(row => {
     Object.entries(row).forEach(([columnName, columnData]) => {
-      const columnType = schema[columnName]?.type
+      const {
-      const columnSubtype = schema[columnName]?.subtype
+        type: columnType,
-      const isAutoColumn = schema[columnName]?.autocolumn
+        subtype: columnSubtype,
+        autocolumn: isAutoColumn,
+      } = schema[columnName]
 
       // If the column had an invalid value we don't want to override it
       if (results.schemaValidation[columnName] === false) {
@@ -123,7 +111,7 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
   return results
 }
 
-export function parse(rows: Rows, schema: Schema): Rows {
+export function parse(rows: Rows, schema: TableSchema): Rows {
   return rows.map(row => {
     const parsedRow: Row = {}
 
@@ -133,9 +121,7 @@ export function parse(rows: Rows, schema: Schema): Rows {
         return
       }
 
-      const columnType = schema[columnName].type
+      const { type: columnType, subtype: columnSubtype } = schema[columnName]
-      const columnSubtype = schema[columnName].subtype
-
       if (columnType === FieldType.NUMBER) {
         // If provided must be a valid number
         parsedRow[columnName] = columnData ? Number(columnData) : columnData
@@ -172,7 +158,7 @@ export function parse(rows: Rows, schema: Schema): Rows {
 
 function isValidBBReference(
   columnData: any,
-  columnSubtype: FieldSubtype
+  columnSubtype: FieldSubtype.USER | FieldSubtype.USERS
 ): boolean {
   switch (columnSubtype) {
     case FieldSubtype.USER:

packages/server/tsconfig.build.json

@@ -18,7 +18,8 @@
       "@budibase/backend-core/*": ["../backend-core/*"],
       "@budibase/shared-core": ["../shared-core/src"],
       "@budibase/pro": ["../pro/src"],
-      "@budibase/string-templates": ["../string-templates/src"]
+      "@budibase/string-templates": ["../string-templates/src"],
+      "@budibase/string-templates/*": ["../string-templates/*"]
     },
     "allowArbitraryExtensions": true
   },

packages/string-templates/src/helpers/list.ts

@@ -1,17 +1,29 @@
 import { date, duration } from "./date"
 
-import {
+/* 
-  math,
+@budibase/handlebars-helpers is not treeshakeable, so we can't use the barrel files. 
-  array,
+Otherwise, we have issues when generating the isolated-vm bundle because of the treeshaking
-  number,
+*/
-  url,
+/* eslint-disable local-rules/no-budibase-imports */
-  string,
+// @ts-expect-error
-  comparison,
+import math from "@budibase/handlebars-helpers/lib/math"
-  object,
+// @ts-expect-error
-  regex,
+import array from "@budibase/handlebars-helpers/lib/array"
-  uuid,
+// @ts-expect-error
-  // @ts-expect-error
+import number from "@budibase/handlebars-helpers/lib/number"
-} from "@budibase/handlebars-helpers"
+// @ts-expect-error
+import url from "@budibase/handlebars-helpers/lib/url"
+// @ts-expect-error
+import string from "@budibase/handlebars-helpers/lib/string"
+// @ts-expect-error
+import comparison from "@budibase/handlebars-helpers/lib/comparison"
+// @ts-expect-error
+import object from "@budibase/handlebars-helpers/lib/object"
+// @ts-expect-error
+import regex from "@budibase/handlebars-helpers/lib/regex"
+// @ts-expect-error
+import uuid from "@budibase/handlebars-helpers/lib/uuid"
+/* eslint-enable local-rules/no-budibase-imports */
 
 // https://github.com/evanw/esbuild/issues/56
 const externalCollections = {
@@ -42,14 +54,14 @@ export function getJsHelperList() {
 
   helpers = {}
   for (let collection of Object.values(externalCollections)) {
-    for (let [key, func] of Object.entries<any>(collection())) {
+    for (let [key, func] of Object.entries<any>(collection)) {
       // Handlebars injects the hbs options to the helpers by default. We are adding an empty {} as a last parameter to simulate it
       helpers[key] = (...props: any) => func(...props, {})
     }
   }
   helpers = {
     ...helpers,
-    addedHelpers,
+    ...addedHelpers,
   }
 
   for (const toRemove of helpersToRemoveForJs) {

packages/string-templates/src/index.ts

@@ -94,7 +94,7 @@ export async function processObject<T extends Record<string, any>>(
   for (const key of Object.keys(object || {})) {
     if (object[key] != null) {
       const val = object[key]
-      let parsedValue
+      let parsedValue = val
       if (typeof val === "string") {
         parsedValue = await processString(object[key], context, opts)
       } else if (typeof val === "object") {

packages/string-templates/test/basic.spec.ts

@@ -104,6 +104,26 @@ describe("Test that the object processing works correctly", () => {
     }
     expect(error).toBeNull()
   })
+
+  it("should be able to handle booleans", async () => {
+    const output = await processObject(
+      {
+        first: true,
+        second: "true",
+        third: "another string",
+        forth: "with {{ template }}",
+      },
+      {
+        template: "value",
+      }
+    )
+    expect(output).toEqual({
+      first: true,
+      second: "true",
+      third: "another string",
+      forth: "with value",
+    })
+  })
 })
 
 describe("check returning objects", () => {

packages/types/src/documents/app/query.ts

@@ -15,6 +15,8 @@ export interface Query extends Document {
   schema: Record<string, QuerySchema | string>
   readable: boolean
   queryVerb: string
+  // flag to state whether the default bindings are empty strings (old behaviour) or null
+  nullDefaultSupport?: boolean
 }
 
 export interface QueryPreview extends Omit<Query, "_id"> {

packages/types/src/documents/app/row.ts

@@ -38,11 +38,16 @@ export interface Row extends Document {
 export enum FieldSubtype {
   USER = "user",
   USERS = "users",
+  SINGLE = "single",
 }
 
+// The 'as' are required for typescript not to type the outputs as generic FieldSubtype
 export const FieldTypeSubtypes = {
   BB_REFERENCE: {
-    USER: FieldSubtype.USER,
+    USER: FieldSubtype.USER as FieldSubtype.USER,
-    USERS: FieldSubtype.USERS,
+    USERS: FieldSubtype.USERS as FieldSubtype.USERS,
+  },
+  ATTACHMENT: {
+    SINGLE: FieldSubtype.SINGLE as FieldSubtype.SINGLE,
   },
 }

packages/types/src/documents/app/table/schema.ts

@@ -112,6 +112,12 @@ export interface BBReferenceFieldMetadata
   relationshipType?: RelationshipType
 }
 
+export interface AttachmentFieldMetadata
+  extends Omit<BaseFieldSchema, "subtype"> {
+  type: FieldType.ATTACHMENT
+  subtype?: FieldSubtype.SINGLE
+}
+
 export interface FieldConstraints {
   type?: string
   email?: boolean
@@ -119,6 +125,7 @@ export interface FieldConstraints {
   length?: {
     minimum?: string | number | null
     maximum?: string | number | null
+    message?: string
   }
   numericality?: {
     greaterThanOrEqualTo: string | null
@@ -156,6 +163,8 @@ interface OtherFieldMetadata extends BaseFieldSchema {
     | FieldType.FORMULA
     | FieldType.NUMBER
     | FieldType.LONGFORM
+    | FieldType.BB_REFERENCE
+    | FieldType.ATTACHMENT
   >
 }
 
@@ -169,6 +178,7 @@ export type FieldSchema =
   | LongFormFieldMetadata
   | BBReferenceFieldMetadata
   | JsonFieldMetadata
+  | AttachmentFieldMetadata
 
 export interface TableSchema {
   [key: string]: FieldSchema
@@ -203,3 +213,9 @@ export function isBBReferenceField(
 ): field is BBReferenceFieldMetadata {
   return field.type === FieldType.BB_REFERENCE
 }
+
+export function isAttachmentField(
+  field: FieldSchema
+): field is AttachmentFieldMetadata {
+  return field.type === FieldType.ATTACHMENT
+}

packages/worker/__mocks__/aws-sdk.ts

@@ -1,15 +0,0 @@
-const mockS3 = {
-  headBucket: jest.fn().mockReturnThis(),
-  deleteObject: jest.fn().mockReturnThis(),
-  deleteObjects: jest.fn().mockReturnThis(),
-  createBucket: jest.fn().mockReturnThis(),
-  listObjects: jest.fn().mockReturnThis(),
-  promise: jest.fn().mockReturnThis(),
-  catch: jest.fn(),
-}
-
-const AWS = {
-  S3: jest.fn(() => mockS3),
-}
-
-export default AWS

packages/worker/__mocks__/node-fetch.ts

@@ -1,23 +0,0 @@
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-module FetchMock {
-  const fetch = jest.requireActual("node-fetch")
-
-  const func = async (url: any, opts: any) => {
-    if (url.includes("http://someconfigurl")) {
-      return {
-        ok: true,
-        json: () => ({
-          issuer: "test",
-          authorization_endpoint: "http://localhost/auth",
-          token_endpoint: "http://localhost/token",
-          userinfo_endpoint: "http://localhost/userinfo",
-        }),
-      }
-    }
-    return fetch(url, opts)
-  }
-
-  func.Headers = fetch.Headers
-
-  module.exports = func
-}

packages/worker/__mocks__/oauth.ts

@@ -1,57 +0,0 @@
-import * as jwt from "jsonwebtoken"
-
-const mockOAuth2 = {
-  getOAuthAccessToken: (code: string, p: any, cb: any) => {
-    const err = null
-    const accessToken = "access_token"
-    const refreshToken = "refresh_token"
-
-    const exp = new Date()
-    exp.setDate(exp.getDate() + 1)
-
-    const iat = new Date()
-    iat.setDate(iat.getDate() - 1)
-
-    const claims = {
-      iss: "test",
-      sub: "sub",
-      aud: "clientId",
-      exp: exp.getTime() / 1000,
-      iat: iat.getTime() / 1000,
-      email: "oauth@example.com",
-    }
-
-    const idToken = jwt.sign(claims, "secret")
-
-    const params = {
-      id_token: idToken,
-    }
-    return cb(err, accessToken, refreshToken, params)
-  },
-  _request: (
-    method: string,
-    url: string,
-    headers: any,
-    postBody: any,
-    accessToken: string,
-    cb: any
-  ) => {
-    const err = null
-    const body = {
-      sub: "sub",
-      user_id: "userId",
-      name: "OAuth",
-      family_name: "2",
-      given_name: "OAuth",
-      middle_name: "",
-    }
-    const res = {}
-    return cb(err, JSON.stringify(body), res)
-  },
-}
-
-const oauth = {
-  OAuth2: jest.fn(() => mockOAuth2),
-}
-
-export = oauth

packages/worker/package.json

@@ -86,6 +86,7 @@
     "@types/supertest": "2.0.14",
     "@types/uuid": "8.3.4",
     "jest": "29.7.0",
+    "nock": "^13.5.4",
     "nodemon": "2.0.15",
     "rimraf": "3.0.2",
     "supertest": "6.3.3",

packages/worker/src/api/routes/global/tests/auth.spec.ts

@@ -13,6 +13,8 @@ import { events, constants } from "@budibase/backend-core"
 import { Response } from "superagent"
 
 import * as userSdk from "../../../../sdk/users"
+import nock from "nock"
+import * as jwt from "jsonwebtoken"
 
 function getAuthCookie(response: Response) {
   return response.headers["set-cookie"]
@@ -274,45 +276,9 @@ describe("/api/global/auth", () => {
     })
   })
 
-  describe("init", () => {
-    describe("POST /api/global/auth/init", () => {})
-
-    describe("GET /api/global/auth/init", () => {})
-  })
-
-  describe("datasource", () => {
-    // MULTI TENANT
-
-    describe("GET /api/global/auth/:tenantId/datasource/:provider", () => {})
-
-    describe("GET /api/global/auth/:tenantId/datasource/:provider/callback", () => {})
-
-    // SINGLE TENANT
-
-    describe("GET /api/global/auth/datasource/:provider/callback", () => {})
-  })
-
-  describe("google", () => {
-    // MULTI TENANT
-
-    describe("GET /api/global/auth/:tenantId/google", () => {})
-
-    describe("GET /api/global/auth/:tenantId/google/callback", () => {})
-
-    // SINGLE TENANT
-
-    describe("GET /api/global/auth/google/callback", () => {})
-
-    describe("GET /api/admin/auth/google/callback", () => {})
-  })
-
   describe("oidc", () => {
-    beforeEach(async () => {
+    afterEach(() => {
-      jest.clearAllMocks()
+      nock.cleanAll()
-      mockGetWellKnownConfig()
-
-      // see: __mocks__/oauth
-      // for associated mocking inside passport
     })
 
     const generateOidcConfig = async () => {
@@ -321,21 +287,16 @@ describe("/api/global/auth", () => {
       return chosenConfig.uuid
     }
 
-    const mockGetWellKnownConfig = () => {
+    // MULTI TENANT
-      mocks.fetch.mockReturnValue({
+    describe("GET /api/global/auth/:tenantId/oidc/configs/:configId", () => {
-        ok: true,
+      it("redirects to auth provider", async () => {
-        json: () => ({
+        nock("http://someconfigurl").get("/").times(1).reply(200, {
           issuer: "test",
           authorization_endpoint: "http://localhost/auth",
           token_endpoint: "http://localhost/token",
           userinfo_endpoint: "http://localhost/userinfo",
-        }),
+        })
-      })
-    }
 
-    // MULTI TENANT
-    describe("GET /api/global/auth/:tenantId/oidc/configs/:configId", () => {
-      it("redirects to auth provider", async () => {
         const configId = await generateOidcConfig()
 
         const res = await config.api.configs.getOIDCConfig(configId)
@@ -352,10 +313,43 @@ describe("/api/global/auth", () => {
 
     describe("GET /api/global/auth/:tenantId/oidc/callback", () => {
       it("logs in", async () => {
+        nock("http://someconfigurl").get("/").times(2).reply(200, {
+          issuer: "test",
+          authorization_endpoint: "http://localhost/auth",
+          token_endpoint: "http://localhost/token",
+          userinfo_endpoint: "http://localhost/userinfo",
+        })
+
+        const token = jwt.sign(
+          {
+            iss: "test",
+            sub: "sub",
+            aud: "clientId",
+            exp: Math.floor(Date.now() / 1000) + 60 * 60,
+            email: "oauth@example.com",
+          },
+          "secret"
+        )
+
+        nock("http://localhost").post("/token").reply(200, {
+          access_token: "access",
+          refresh_token: "refresh",
+          id_token: token,
+        })
+
+        nock("http://localhost").get("/userinfo?schema=openid").reply(200, {
+          sub: "sub",
+          email: "oauth@example.com",
+        })
+
         const configId = await generateOidcConfig()
         const preAuthRes = await config.api.configs.getOIDCConfig(configId)
-
         const res = await config.api.configs.OIDCCallback(configId, preAuthRes)
+        if (res.status > 399) {
+          throw new Error(
+            `OIDC callback failed with status ${res.status}: ${res.text}`
+          )
+        }
 
         expect(events.auth.login).toHaveBeenCalledWith(
           "oidc",