Merge branch 'develop' into grid-mobile-scrolling

.github/workflows/budibase_ci.yml

@@ -20,6 +20,7 @@ env:
   PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
   NX_BASE_BRANCH: origin/${{ github.base_ref }}
   USE_NX_AFFECTED: ${{ github.event_name == 'pull_request' && github.base_ref != 'master'}}
+  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
 
 jobs:
   lint:

.github/workflows/close-featurebranch.yml

@@ -14,7 +14,7 @@ jobs:
       - uses: passeidireto/trigger-external-workflow-action@main
         env:
           PAYLOAD_BRANCH: ${{ github.head_ref }}
-          PAYLOAD_PR_NUMBER: ${{ github.ref }}
+          PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-close

.github/workflows/deploy-featurebranch.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - develop
+      - master
 
 jobs:
   release:
@@ -13,7 +14,7 @@ jobs:
       - uses: passeidireto/trigger-external-workflow-action@main
         env:
           PAYLOAD_BRANCH: ${{ github.head_ref }}
-          PAYLOAD_PR_NUMBER: ${{ github.ref }}
+          PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-deploy

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.10.16-alpha.11",
+  "version": "2.11.15-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/src/plugin/utils.ts

@@ -6,6 +6,7 @@ import {
   AutomationStepIdArray,
   AutomationIOType,
   AutomationCustomIOType,
+  DatasourceFeature,
 } from "@budibase/types"
 import joi from "joi"
 
@@ -67,9 +68,27 @@ function validateDatasource(schema: any) {
     version: joi.string().optional(),
     schema: joi.object({
       docs: joi.string(),
+      plus: joi.boolean().optional(),
+      isSQL: joi.boolean().optional(),
+      auth: joi
+        .object({
+          type: joi.string().required(),
+        })
+        .optional(),
+      features: joi
+        .object(
+          Object.fromEntries(
+            Object.values(DatasourceFeature).map(key => [
+              key,
+              joi.boolean().optional(),
+            ])
+          )
+        )
+        .optional(),
+      relationships: joi.boolean().optional(),
+      description: joi.string().required(),
       friendlyName: joi.string().required(),
       type: joi.string().allow(...DATASOURCE_TYPES),
-      description: joi.string().required(),
       datasource: joi.object().pattern(joi.string(), fieldValidator).required(),
       query: joi
         .object()

packages/bbui/src/OptionSelectDnD/OptionSelectDnD.svelte

@@ -21,14 +21,6 @@
     "hsla(240, 90%, 75%, 0.3)",
     "hsla(320, 90%, 75%, 0.3)",
   ]
-  $: {
-    if (constraints.inclusion.length) {
-      options = constraints.inclusion.map(value => ({
-        name: value,
-        id: Math.random(),
-      }))
-    }
-  }
   const removeInput = idx => {
     delete optionColors[options[idx].name]
     constraints.inclusion = constraints.inclusion.filter((e, i) => i !== idx)
@@ -80,6 +72,11 @@
     // Initialize anchor arrays on mount, assuming 'options' is already populated
     colorPopovers = constraints.inclusion.map(() => undefined)
     anchors = constraints.inclusion.map(() => undefined)
+
+    options = constraints.inclusion.map(value => ({
+      name: value,
+      id: Math.random(),
+    }))
   })
 </script>
 

packages/builder/src/builderStore/store/automation/index.js

@@ -221,18 +221,6 @@ const automationActions = store => ({
     newAutomation.definition.steps.splice(blockIdx, 0, block)
     await store.actions.save(newAutomation)
   },
-  /**
-   * "rowControl" appears to be the name of the flag used to determine whether
-   * a certain automation block uses values or bindings as inputs
-   */
-  toggleRowControl: async (block, rowControl) => {
-    const newBlock = { ...block, rowControl }
-    const newAutomation = store.actions.getUpdatedDefinition(
-      get(selectedAutomation),
-      newBlock
-    )
-    await store.actions.save(newAutomation)
-  },
   deleteAutomationBlock: async block => {
     const automation = get(selectedAutomation)
     let newAutomation = cloneDeep(automation)

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -7,7 +7,6 @@
     Detail,
     Modal,
     Button,
-    Select,
     ActionButton,
     notifications,
     Label,
@@ -39,9 +38,6 @@
     step => step.stepId === ActionStepID.COLLECT
   )
   $: automationId = $selectedAutomation?._id
-  $: showBindingPicker =
-    block.stepId === ActionStepID.CREATE_ROW ||
-    block.stepId === ActionStepID.UPDATE_ROW
   $: isTrigger = block.type === "TRIGGER"
   $: steps = $selectedAutomation?.definition?.steps ?? []
   $: blockIdx = steps.findIndex(step => step.id === block.id)
@@ -96,15 +92,6 @@
     }
   }
 
-  /**
-   * "rowControl" appears to be the name of the flag used to determine whether
-   * a certain automation block uses values or bindings as inputs
-   */
-  function toggleRowControl(evt) {
-    const rowControl = evt.detail !== "Use values"
-    automationStore.actions.toggleRowControl(block, rowControl)
-  }
-
   async function addLooping() {
     const loopDefinition = $automationStore.blockDefinitions.ACTION.LOOP
     const loopBlock = automationStore.actions.constructBlock(
@@ -189,16 +176,6 @@
                   Add Looping
                 </ActionButton>
               {/if}
-              {#if showBindingPicker}
-                <Select
-                  on:change={toggleRowControl}
-                  defaultValue="Use values"
-                  autoWidth
-                  value={block.rowControl ? "Use bindings" : "Use values"}
-                  options={["Use values", "Use bindings"]}
-                  placeholder={null}
-                />
-              {/if}
               <ActionButton
                 on:click={() => deleteStep()}
                 icon="DeleteOutline"

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -23,6 +23,7 @@
   import { environment, licensing } from "stores/portal"
   import WebhookDisplay from "../Shared/WebhookDisplay.svelte"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
+  import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import CodeEditorModal from "./CodeEditorModal.svelte"
   import QuerySelector from "./QuerySelector.svelte"
@@ -82,33 +83,6 @@
       ? [hbAutocomplete([...bindingsToCompletions(bindings, codeMode)])]
       : []
 
-  /**
-   * TODO - Remove after November 2023
-   * *******************************
-   * Code added to provide backwards compatibility between Values 1,2,3,4,5
-   * and the new JSON body.
-   */
-  let deprecatedSchemaProperties
-  $: {
-    if (block?.stepId === "integromat" || block?.stepId === "zapier") {
-      deprecatedSchemaProperties = schemaProperties.filter(
-        prop => !prop[0].startsWith("value")
-      )
-      if (!deprecatedSchemaProperties.map(entry => entry[0]).includes("body")) {
-        deprecatedSchemaProperties.push([
-          "body",
-          {
-            title: "Payload",
-            type: "json",
-          },
-        ])
-      }
-    } else {
-      deprecatedSchemaProperties = schemaProperties
-    }
-  }
-  /****************************************************/
-
   const getInputData = (testData, blockInputs) => {
     // Test data is not cloned for reactivity
     let newInputData = testData || cloneDeep(blockInputs)
@@ -118,30 +92,6 @@
       newInputData = cloneDeep(blockInputs)
     }
 
-    /**
-     * TODO - Remove after November 2023
-     * *******************************
-     * Code added to provide backwards compatibility between Values 1,2,3,4,5
-     * and the new JSON body.
-     */
-    if (
-      (block?.stepId === "integromat" || block?.stepId === "zapier") &&
-      !newInputData?.body?.value
-    ) {
-      let deprecatedValues = {
-        ...newInputData,
-      }
-      delete deprecatedValues.url
-      delete deprecatedValues.body
-      newInputData = {
-        url: newInputData.url,
-        body: {
-          value: JSON.stringify(deprecatedValues),
-        },
-      }
-    }
-    /**********************************/
-
     inputData = newInputData
     setDefaultEnumValues()
   }
@@ -337,7 +287,7 @@
 </script>
 
 <div class="fields">
-  {#each deprecatedSchemaProperties as [key, value]}
+  {#each schemaProperties as [key, value]}
     {#if canShowField(key, value)}
       <div class="block-field">
         {#if key !== "fields" && value.type !== "boolean"}
@@ -362,18 +312,6 @@
             mode="json"
             value={inputData[key]?.value}
             on:change={e => {
-              /**
-               * TODO - Remove after November 2023
-               * *******************************
-               * Code added to provide backwards compatibility between Values 1,2,3,4,5
-               * and the new JSON body.
-               */
-              delete inputData.value1
-              delete inputData.value2
-              delete inputData.value3
-              delete inputData.value4
-              delete inputData.value5
-              /***********************/
               onChange(e, key)
             }}
           />
@@ -386,10 +324,23 @@
             />
           </div>
         {:else if value.type === "date"}
-          <DatePicker
+          <DrawerBindableSlot
+            fillWidth
+            title={value.title}
+            panel={AutomationBindingPanel}
+            type={"date"}
             value={inputData[key]}
             on:change={e => onChange(e, key)}
-          />
+            {bindings}
+            allowJS={true}
+            updateOnChange={false}
+            drawerLeft="260px"
+          >
+            <DatePicker
+              value={inputData[key]}
+              on:change={e => onChange(e, key)}
+            />
+          </DrawerBindableSlot>
         {:else if value.customType === "column"}
           <Select
             on:change={e => onChange(e, key)}
@@ -469,7 +420,6 @@
           />
         {:else if value.customType === "row"}
           <RowSelector
-            {block}
             value={inputData[key]}
             meta={inputData["meta"] || {}}
             on:change={e => {

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -1,18 +1,16 @@
 <script>
   import { tables } from "stores/backend"
   import { Select, Checkbox } from "@budibase/bbui"
-  import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
-  import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
   import RowSelectorTypes from "./RowSelectorTypes.svelte"
-  import ModalBindableInput from "../../common/bindings/ModalBindableInput.svelte"
+  import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte"
+  import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
 
   const dispatch = createEventDispatcher()
 
   export let value
   export let meta
   export let bindings
-  export let block
   export let isTestModal
   export let isUpdateRow
 
@@ -25,16 +23,6 @@
   let table
   let schemaFields
 
-  let placeholders = {
-    number: 10,
-    boolean: "true",
-    datetime: "2022-02-16T12:00:00.000Z ",
-    options: "1",
-    array: "1 2 3 4",
-    link: "ro_ta_123_456",
-    longform: "long form text",
-  }
-  $: rowControl = block.rowControl
   $: {
     table = $tables.list.find(table => table._id === value?.tableId)
     schemaFields = Object.entries(table?.schema ?? {})
@@ -57,19 +45,13 @@
       return value
     }
 
-    if (type === "boolean") {
-      if (typeof value === "boolean") {
-        return value
-      }
-      return value === "true"
-    }
     if (type === "number") {
       if (typeof value === "number") {
         return value
       }
       return Number(value)
     }
-    if (type === "options") {
+    if (type === "options" || type === "boolean") {
       return value
     }
     if (type === "array") {
@@ -127,9 +109,31 @@
 {#if schemaFields.length}
   <div class="schema-fields">
     {#each schemaFields as [field, schema]}
-      {#if !schema.autocolumn}
+      {#if !schema.autocolumn && schema.type !== "attachment"}
-        {#if schema.type !== "attachment"}
+        {#if isTestModal}
-          {#if !rowControl}
+          <RowSelectorTypes
+            {isTestModal}
+            {field}
+            {schema}
+            bindings={parsedBindings}
+            {value}
+            {onChange}
+          />
+        {:else}
+          <DrawerBindableSlot
+            fillWidth
+            title={value.title}
+            label={field}
+            panel={AutomationBindingPanel}
+            type={schema.type}
+            {schema}
+            value={value[field]}
+            on:change={e => onChange(e, field)}
+            {bindings}
+            allowJS={true}
+            updateOnChange={false}
+            drawerLeft="260px"
+          >
             <RowSelectorTypes
               {isTestModal}
               {field}
@@ -138,37 +142,19 @@
               {value}
               {onChange}
             />
-          {:else}
+          </DrawerBindableSlot>
-            <div>
-              <svelte:component
-                this={isTestModal ? ModalBindableInput : DrawerBindableInput}
-                placeholder={placeholders[schema.type]}
-                panel={AutomationBindingPanel}
-                value={Array.isArray(value[field])
-                  ? value[field].join(",")
-                  : value[field]}
-                on:change={e => onChange(e, field, schema.type)}
-                label={field}
-                type="string"
-                bindings={parsedBindings}
-                fillWidth={true}
-                allowJS={true}
-                updateOnChange={false}
-              />
-              {#if isUpdateRow && schema.type === "link"}
-                <div class="checkbox-field">
-                  <Checkbox
-                    value={meta.fields?.[field]?.clearRelationships}
-                    text={"Clear relationships if empty?"}
-                    size={"S"}
-                    on:change={e => onChangeSetting(e, field)}
-                  />
-                </div>
-              {/if}
-            </div>
-          {/if}
         {/if}
       {/if}
+      {#if isUpdateRow && schema.type === "link"}
+        <div class="checkbox-field">
+          <Checkbox
+            value={meta.fields?.[field]?.clearRelationships}
+            text={"Clear relationships if empty?"}
+            size={"S"}
+            on:change={e => onChangeSetting(e, field)}
+          />
+        </div>
+      {/if}
     {/each}
   </div>
 {/if}

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -1,7 +1,6 @@
 <script>
   import {
     Select,
-    Toggle,
     DatePicker,
     Multiselect,
     TextArea,
@@ -45,10 +44,14 @@
     on:change={e => onChange(e, field)}
   />
 {:else if schema.type === "boolean"}
-  <Toggle
+  <Select
-    text={field}
-    value={value[field]}
     on:change={e => onChange(e, field)}
+    label={field}
+    value={value[field]}
+    options={[
+      { label: "True", value: "true" },
+      { label: "False", value: "false" },
+    ]}
   />
 {:else if schema.type === "array"}
   <Multiselect

packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte

@@ -13,7 +13,13 @@
   let modal
 
   $: tempValue = filters || []
-  $: schemaFields = Object.values(schema || {})
+  $: schemaFields = Object.entries(schema || {}).map(
+    ([fieldName, fieldSchema]) => ({
+      name: fieldName, // Using the key as name if not defined in the schema, for example in some autogenerated columns
+      ...fieldSchema,
+    })
+  )
+
   $: text = getText(filters)
   $: selected = tempValue.filter(x => !x.onEmptyFilter)?.length > 0
 

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -85,7 +85,7 @@
   }
 
   let relationshipMap = {
-    [RelationshipType.MANY_TO_ONE]: {
+    [RelationshipType.ONE_TO_MANY]: {
       part1: PrettyRelationshipDefinitions.MANY,
       part2: PrettyRelationshipDefinitions.ONE,
     },
@@ -93,7 +93,7 @@
       part1: PrettyRelationshipDefinitions.MANY,
       part2: PrettyRelationshipDefinitions.MANY,
     },
-    [RelationshipType.ONE_TO_MANY]: {
+    [RelationshipType.MANY_TO_ONE]: {
       part1: PrettyRelationshipDefinitions.ONE,
       part2: PrettyRelationshipDefinitions.MANY,
     },
@@ -277,10 +277,7 @@
       dispatch("updatecolumns")
       gridDispatch("close-edit-column")
 
-      if (
+      if (saveColumn.type === LINK_TYPE) {
-        saveColumn.type === LINK_TYPE &&
-        saveColumn.relationshipType === RelationshipType.MANY_TO_MANY
-      ) {
         // Fetching the new tables
         tables.fetch()
         // Fetching the new relationships
@@ -312,6 +309,11 @@
         confirmDeleteDialog.hide()
         dispatch("updatecolumns")
         gridDispatch("close-edit-column")
+
+        if (editableColumn.type === LINK_TYPE) {
+          // Updating the relationships
+          datasources.fetch()
+        }
       }
     } catch (error) {
       notifications.error(`Error deleting column: ${error.message}`)
@@ -658,7 +660,8 @@
       >Open schema editor</Button
     >
   {:else if editableColumn.type === USER_REFRENCE_TYPE}
-    <Toggle
+    <!-- Disabled temporally -->
+    <!-- <Toggle
       value={editableColumn.relationshipType === RelationshipType.MANY_TO_MANY}
       on:change={e =>
         (editableColumn.relationshipType = e.detail
@@ -667,7 +670,7 @@
       disabled={!isCreating}
       thin
       text="Allow multiple users"
-    />
+    /> -->
   {/if}
   {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
     <Select

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -57,7 +57,8 @@
     label: table.name,
     value: table._id,
   }))
-  $: valid = getErrorCount(errors) === 0 && allRequiredAttributesSet()
+  $: valid =
+    getErrorCount(errors) === 0 && allRequiredAttributesSet(relationshipType)
   $: isManyToMany = relationshipType === RelationshipType.MANY_TO_MANY
   $: isManyToOne = relationshipType === RelationshipType.MANY_TO_ONE
 
@@ -114,7 +115,7 @@
     return Object.entries(errors).filter(entry => !!entry[1]).length
   }
 
-  function allRequiredAttributesSet() {
+  function allRequiredAttributesSet(relationshipType) {
     const base = getTable(fromId) && getTable(toId) && fromColumn && toColumn
     if (relationshipType === RelationshipType.MANY_TO_ONE) {
       return base && fromPrimary && fromForeign
@@ -124,9 +125,10 @@
   }
 
   function validate() {
-    if (!allRequiredAttributesSet() && !hasValidated) {
+    if (!allRequiredAttributesSet(relationshipType) && !hasValidated) {
       return
     }
+
     hasValidated = true
     errorChecker.setType(relationshipType)
     const fromTable = getTable(fromId),

packages/builder/src/components/common/LinkedRowSelector.svelte

@@ -11,11 +11,11 @@
   const dispatch = createEventDispatcher()
 
   let rows = []
-  let linkedIds = (Array.isArray(linkedRows) ? linkedRows : [])?.map(
+  let linkedIds = []
+
+  $: linkedIds = (Array.isArray(linkedRows) ? linkedRows : [])?.map(
     row => row?._id || row
   )
-
-  $: linkedRows = linkedIds
   $: label = capitalise(schema.name)
   $: linkedTableId = schema.tableId
   $: linkedTable = $tables.list.find(table => table._id === linkedTableId)

packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte

@@ -0,0 +1,250 @@
+<script>
+  import { Icon, Input, Drawer, Button } from "@budibase/bbui"
+  import {
+    readableToRuntimeBinding,
+    runtimeToReadableBinding,
+  } from "builderStore/dataBinding"
+
+  import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
+  import { createEventDispatcher, setContext } from "svelte"
+  import { isJSBinding } from "@budibase/string-templates"
+
+  export let panel = ClientBindingPanel
+  export let value = ""
+  export let bindings = []
+  export let title = "Bindings"
+  export let placeholder
+  export let label
+  export let disabled = false
+  export let fillWidth
+  export let allowJS = true
+  export let allowHelpers = true
+  export let updateOnChange = true
+  export let drawerLeft
+  export let type
+  export let schema
+
+  const dispatch = createEventDispatcher()
+  let bindingDrawer
+  let valid = true
+  let currentVal = value
+
+  $: readableValue = runtimeToReadableBinding(bindings, value)
+  $: tempValue = readableValue
+  $: isJS = isJSBinding(value)
+
+  const saveBinding = () => {
+    onChange(tempValue)
+    onBlur()
+    bindingDrawer.hide()
+  }
+
+  setContext("binding-drawer-actions", {
+    save: saveBinding,
+  })
+
+  const onChange = value => {
+    if (type === "link" && value && hasValidLinks(value)) {
+      currentVal = value.split(",")
+    } else if (type === "array" && value && hasValidOptions(value)) {
+      currentVal = value.split(",")
+    } else {
+      currentVal = readableToRuntimeBinding(bindings, value)
+    }
+    dispatch("change", currentVal)
+  }
+
+  const onBlur = () => {
+    dispatch("blur", currentVal)
+  }
+
+  const isValidDate = value => {
+    return !value || !isNaN(new Date(value).valueOf())
+  }
+
+  const hasValidLinks = value => {
+    let links = []
+    if (Array.isArray(value)) {
+      links = value
+    } else if (value && typeof value === "string") {
+      links = value.split(",")
+    } else {
+      return !value
+    }
+
+    return links.every(link => link.startsWith("ro_"))
+  }
+
+  const hasValidOptions = value => {
+    let links = []
+    if (Array.isArray(value)) {
+      links = value
+    } else if (value && typeof value === "string") {
+      links = value.split(",")
+    } else {
+      return !value
+    }
+    return links.every(link => schema?.constraints?.inclusion?.includes(link))
+  }
+
+  const isValidBoolean = value => {
+    return value === "false" || value === "true" || value == ""
+  }
+
+  const validationMap = {
+    date: isValidDate,
+    datetime: isValidDate,
+    link: hasValidLinks,
+    array: hasValidOptions,
+    longform: value => !isJSBinding(value),
+    json: value => !isJSBinding(value),
+    boolean: isValidBoolean,
+  }
+
+  const isValid = value => {
+    const validate = validationMap[type]
+    return validate ? validate(value) : true
+  }
+
+  const getIconClass = (value, type) => {
+    if (type === "longform" && !isJSBinding(value)) {
+      return "text-area-slot-icon"
+    }
+    if (type === "json" && !isJSBinding(value)) {
+      return "json-slot-icon"
+    }
+    if (type !== "string" && type !== "number") {
+      return "slot-icon"
+    }
+    return ""
+  }
+</script>
+
+<div class="control" class:disabled>
+  {#if !isValid(value)}
+    <Input
+      {label}
+      {disabled}
+      readonly={isJS}
+      value={isJS ? "(JavaScript function)" : readableValue}
+      on:change={event => onChange(event.detail)}
+      on:blur={onBlur}
+      {placeholder}
+      {updateOnChange}
+    />
+    <div
+      class="icon"
+      on:click={() => {
+        if (!isJS) {
+          dispatch("change", "")
+        }
+      }}
+    >
+      <Icon disabled={isJS} size="S" name="Close" />
+    </div>
+  {:else}
+    <slot
+      {label}
+      {disabled}
+      readonly={isJS}
+      value={isJS ? "(JavaScript function)" : readableValue}
+      {placeholder}
+      {updateOnChange}
+    />
+  {/if}
+  {#if !disabled && type !== "formula"}
+    <div
+      class={`icon ${getIconClass(value, type)}`}
+      on:click={() => {
+        bindingDrawer.show()
+      }}
+    >
+      <Icon size="S" name="FlashOn" />
+    </div>
+  {/if}
+</div>
+<Drawer
+  on:drawerHide
+  on:drawerShow
+  {fillWidth}
+  bind:this={bindingDrawer}
+  {title}
+  left={drawerLeft}
+  headless
+>
+  <svelte:fragment slot="description">
+    Add the objects on the left to enrich your text.
+  </svelte:fragment>
+  <Button cta slot="buttons" disabled={!valid} on:click={saveBinding}>
+    Save
+  </Button>
+  <svelte:component
+    this={panel}
+    slot="body"
+    bind:valid
+    value={readableValue}
+    on:change={event => (tempValue = event.detail)}
+    {bindings}
+    {allowJS}
+    {allowHelpers}
+  />
+</Drawer>
+
+<style>
+  .control {
+    flex: 1;
+    position: relative;
+  }
+
+  .slot-icon {
+    right: 31px !important;
+    border-right: 1px solid var(--spectrum-alias-border-color);
+    border-top-right-radius: 0px !important;
+    border-bottom-right-radius: 0px !important;
+  }
+
+  .text-area-slot-icon {
+    border-bottom: 1px solid var(--spectrum-alias-border-color);
+    border-bottom-right-radius: 0px !important;
+    top: 26px !important;
+  }
+  .json-slot-icon {
+    border-bottom: 1px solid var(--spectrum-alias-border-color);
+    border-bottom-right-radius: 0px !important;
+    top: 23px !important;
+    right: 0px !important;
+  }
+
+  .icon {
+    right: 1px;
+    bottom: 1px;
+    position: absolute;
+    justify-content: center;
+    align-items: center;
+    display: flex;
+    flex-direction: row;
+    box-sizing: border-box;
+    border-left: 1px solid var(--spectrum-alias-border-color);
+    border-top-right-radius: var(--spectrum-alias-border-radius-regular);
+    border-bottom-right-radius: var(--spectrum-alias-border-radius-regular);
+    width: 31px;
+    color: var(--spectrum-alias-text-color);
+    background-color: var(--spectrum-global-color-gray-75);
+    transition: background-color
+        var(--spectrum-global-animation-duration-100, 130ms),
+      box-shadow var(--spectrum-global-animation-duration-100, 130ms),
+      border-color var(--spectrum-global-animation-duration-100, 130ms);
+    height: calc(var(--spectrum-alias-item-height-m) - 2px);
+  }
+
+  .icon:hover {
+    cursor: pointer;
+    color: var(--spectrum-alias-text-color-hover);
+    background-color: var(--spectrum-global-color-gray-50);
+    border-color: var(--spectrum-alias-border-color-hover);
+  }
+
+  .control:not(.disabled) :global(.spectrum-Textfield-input) {
+    padding-right: 40px;
+  }
+</style>

packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte

@@ -3,21 +3,24 @@
     Body,
     Button,
     Combobox,
-    Multiselect,
     DatePicker,
     DrawerContent,
     Icon,
     Input,
-    Layout,
-    Select,
     Label,
+    Layout,
+    Multiselect,
+    Select,
   } from "@budibase/bbui"
   import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
   import { generate } from "shortid"
-  import { LuceneUtils, Constants } from "@budibase/frontend-core"
+  import { Constants, LuceneUtils } from "@budibase/frontend-core"
   import { getFields } from "helpers/searchFields"
+  import { FieldType } from "@budibase/types"
   import { createEventDispatcher, onMount } from "svelte"
+  import FilterUsers from "./FilterUsers.svelte"
+  import { RelationshipType } from "constants/backend"
 
   export let schemaFields
   export let filters = []
@@ -29,7 +32,6 @@
 
   const dispatch = createEventDispatcher()
   const { OperatorOptions } = Constants
-  const { getValidOperatorsForType } = LuceneUtils
   const KeyedFieldRegex = /\d[0-9]*:/g
   const behaviourOptions = [
     { value: "and", label: "Match all filters" },
@@ -120,7 +122,7 @@
     return enrichedSchemaFields.find(field => field.name === filter.field)
   }
 
-  const santizeTypes = filter => {
+  const sanitizeTypes = filter => {
     // Update type based on field
     const fieldSchema = enrichedSchemaFields.find(x => x.name === filter.field)
     filter.type = fieldSchema?.type
@@ -129,13 +131,9 @@
     filter.externalType = getSchema(filter)?.externalType
   }
 
-  const santizeOperator = filter => {
+  const sanitizeOperator = filter => {
     // Ensure a valid operator is selected
-    const operators = getValidOperatorsForType(
+    const operators = getValidOperatorsForType(filter).map(x => x.value)
-      filter.type,
-      filter.field,
-      datasource
-    ).map(x => x.value)
     if (!operators.includes(filter.operator)) {
       filter.operator = operators[0] ?? OperatorOptions.Equals.value
     }
@@ -148,7 +146,7 @@
     filter.noValue = noValueOptions.includes(filter.operator)
   }
 
-  const santizeValue = filter => {
+  const sanitizeValue = (filter, previousType) => {
     // Check if the operator allows a value at all
     if (filter.noValue) {
       filter.value = null
@@ -162,28 +160,47 @@
       }
     } else if (filter.type === "array" && filter.valueType === "Value") {
       filter.value = []
+    } else if (
+      previousType !== filter.type &&
+      (previousType === FieldType.BB_REFERENCE ||
+        filter.type === FieldType.BB_REFERENCE)
+    ) {
+      filter.value = filter.type === "array" ? [] : null
     }
   }
 
   const onFieldChange = filter => {
-    santizeTypes(filter)
+    const previousType = filter.type
-    santizeOperator(filter)
+    sanitizeTypes(filter)
-    santizeValue(filter)
+    sanitizeOperator(filter)
+    sanitizeValue(filter, previousType)
   }
 
   const onOperatorChange = filter => {
-    santizeOperator(filter)
+    sanitizeOperator(filter)
-    santizeValue(filter)
+    sanitizeValue(filter, filter.type)
   }
 
   const onValueTypeChange = filter => {
-    santizeValue(filter)
+    sanitizeValue(filter)
   }
 
   const getFieldOptions = field => {
     const schema = enrichedSchemaFields.find(x => x.name === field)
     return schema?.constraints?.inclusion || []
   }
+
+  const getValidOperatorsForType = filter => {
+    if (!filter?.field) {
+      return []
+    }
+
+    return LuceneUtils.getValidOperatorsForType(
+      filter.type,
+      filter.field,
+      datasource
+    )
+  }
 </script>
 
 <DrawerContent>
@@ -228,11 +245,7 @@
               />
               <Select
                 disabled={!filter.field}
-                options={getValidOperatorsForType(
+                options={getValidOperatorsForType(filter)}
-                  filter.type,
-                  filter.field,
-                  datasource
-                )}
                 bind:value={filter.operator}
                 on:change={() => onOperatorChange(filter)}
                 placeholder={null}
@@ -285,6 +298,14 @@
                   timeOnly={getSchema(filter)?.timeOnly}
                   bind:value={filter.value}
                 />
+              {:else if filter.type === FieldType.BB_REFERENCE}
+                <FilterUsers
+                  bind:value={filter.value}
+                  multiselect={getSchema(filter).relationshipType ===
+                    RelationshipType.MANY_TO_MANY ||
+                    filter.operator === OperatorOptions.In.value}
+                  disabled={filter.noValue}
+                />
               {:else}
                 <DrawerBindableInput disabled />
               {/if}

packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte

@@ -0,0 +1,34 @@
+<script>
+  import { Select, Multiselect } from "@budibase/bbui"
+  import { fetchData } from "@budibase/frontend-core"
+
+  import { API } from "api"
+
+  export let value = null
+  export let disabled
+  export let multiselect = false
+
+  $: fetch = fetchData({
+    API,
+    datasource: {
+      type: "user",
+    },
+    options: {
+      limit: 100,
+    },
+  })
+
+  $: options = $fetch.rows
+
+  $: component = multiselect ? Multiselect : Select
+</script>
+
+<svelte:component
+  this={component}
+  bind:value
+  autocomplete
+  {options}
+  getOptionLabel={option => option.email}
+  getOptionValue={option => option._id}
+  {disabled}
+/>

packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte

@@ -20,7 +20,9 @@
 
   const getSortableFields = schema => {
     return Object.entries(schema || {})
-      .filter(entry => !UNSORTABLE_TYPES.includes(entry[1].type))
+      .filter(
+        entry => !UNSORTABLE_TYPES.includes(entry[1].type) && entry[1].sortable
+      )
       .map(entry => entry[0])
   }
 

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/EditDatasourceConfigButton.svelte

@@ -62,7 +62,14 @@
       </div>
     {/if}
     <div class="truncate">
-      <Body>{getSubtitle(datasource)}</Body>
+      <Body>
+        {@const subtitle = getSubtitle(datasource)}
+        {#if subtitle}
+          {subtitle}
+        {:else}
+          {Object.values(datasource.config).join(" / ")}
+        {/if}
+      </Body>
     </div>
   </div>
   <div class="right">

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Relationships.svelte

@@ -21,15 +21,22 @@
   function getRelationships(tables) {
     const relatedColumns = {}
 
-    tables.forEach(({ name: tableName, schema }) => {
+    tables.forEach(({ name: tableName, schema, _id: tableId }) => {
       Object.values(schema).forEach(column => {
         if (column.type !== "link") return
 
-        relatedColumns[column._id] ??= {}
+        const columnId =
-        relatedColumns[column._id].through =
+          column.through ||
-          relatedColumns[column._id].through || column.through
+          column._id ||
+          (column.main
+            ? `${tableId}_${column.fieldName}__${column.tableId}_${column.foreignKey}`
+            : `${column.tableId}_${column.foreignKey}__${tableId}_${column.fieldName}`)
 
-        relatedColumns[column._id][column.main ? "from" : "to"] = {
+        relatedColumns[columnId] ??= {}
+        relatedColumns[columnId].through =
+          relatedColumns[columnId].through || column.through
+
+        relatedColumns[columnId][column.main ? "from" : "to"] = {
           ...column,
           tableName,
         }

packages/builder/src/stores/backend/datasources.js

@@ -136,6 +136,7 @@ export function createDatasourcesStore() {
       config,
       name: `${integration.friendlyName}${nameModifier}`,
       plus: integration.plus && integration.name !== IntegrationTypes.REST,
+      isSQL: integration.isSQL,
     }
 
     if (await checkDatasourceValidity(integration, datasource)) {

packages/cli/src/hosting/utils.ts

@@ -57,7 +57,8 @@ export async function checkDockerConfigured() {
     "docker/docker-compose has not been installed, please follow instructions at: https://docs.budibase.com/docs/docker-compose"
   const docker = await lookpath("docker")
   const compose = await lookpath("docker-compose")
-  if (!docker || !compose) {
+  const composeV2 = await lookpath("docker compose")
+  if (!docker || (!compose && !composeV2)) {
     throw error
   }
 }

packages/cli/src/prebuilds.ts

@@ -12,6 +12,10 @@ if (!process.argv[0].includes("node")) {
   checkForBinaries()
 }
 
+function localPrebuildPath() {
+  return join(process.execPath, "..", PREBUILDS)
+}
+
 function checkForBinaries() {
   const readDir = join(__filename, "..", "..", "..", "cli", PREBUILDS, ARCH)
   if (fs.existsSync(PREBUILD_DIR) || !fs.existsSync(readDir)) {
@@ -19,17 +23,21 @@ function checkForBinaries() {
   }
   const natives = fs.readdirSync(readDir)
   if (fs.existsSync(readDir)) {
-    const writePath = join(process.execPath, PREBUILDS, ARCH)
+    const writePath = join(localPrebuildPath(), ARCH)
     fs.mkdirSync(writePath, { recursive: true })
     for (let native of natives) {
       const filename = `${native.split(".fake")[0]}.node`
       fs.cpSync(join(readDir, native), join(writePath, filename))
     }
-    console.log("copied something")
   }
 }
 
 function cleanup(evt?: number) {
+  // cleanup prebuilds first
+  const path = localPrebuildPath()
+  if (fs.existsSync(path)) {
+    fs.rmSync(path, { recursive: true })
+  }
   if (evt && !isNaN(evt)) {
     return
   }
@@ -41,10 +49,6 @@ function cleanup(evt?: number) {
     )
     console.error(error(evt))
   }
-  const path = join(process.execPath, PREBUILDS)
-  if (fs.existsSync(path)) {
-    fs.rmSync(path, { recursive: true })
-  }
 }
 
 const events = ["exit", "SIGINT", "SIGUSR1", "SIGUSR2", "uncaughtException"]

packages/client/manifest.json

@@ -5598,6 +5598,21 @@
           }
         ]
       },
+      {
+        "type": "event",
+        "label": "On row click",
+        "key": "onRowClick",
+        "context": [
+          {
+            "label": "Clicked row",
+            "key": "row"
+          }
+        ],
+        "dependsOn": {
+          "setting": "allowEditRows",
+          "value": false
+        }
+      },
       {
         "type": "boolean",
         "label": "Add rows",

packages/client/src/components/app/GridBlock.svelte

@@ -14,12 +14,14 @@
   export let initialSortOrder = null
   export let fixedRowHeight = null
   export let columns = null
+  export let onRowClick = null
 
   const component = getContext("component")
   const { styleable, API, builderStore, notificationStore } = getContext("sdk")
 
   $: columnWhitelist = columns?.map(col => col.name)
   $: schemaOverrides = getSchemaOverrides(columns)
+  $: handleRowClick = allowEditRows ? undefined : onRowClick
 
   const getSchemaOverrides = columns => {
     let overrides = {}
@@ -56,6 +58,7 @@
     showControls={false}
     notifySuccess={notificationStore.actions.success}
     notifyError={notificationStore.actions.error}
+    on:rowclick={e => handleRowClick?.({ row: e.detail })}
   />
 </div>
 

packages/client/src/components/app/Layout.svelte

@@ -3,6 +3,7 @@
   import { writable } from "svelte/store"
   import { Heading, Icon, clickOutside } from "@budibase/bbui"
   import { FieldTypes } from "constants"
+  import { Constants } from "@budibase/frontend-core"
   import active from "svelte-spa-router/active"
 
   const sdk = getContext("sdk")
@@ -103,7 +104,8 @@
     let validLinks = (allLinks || []).filter(link => link.text && link.url)
     // Filter to only links allowed by the current role
     return validLinks.filter(link => {
-      return userRoleHierarchy?.find(roleId => roleId === link.roleId)
+      const role = link.roleId || Constants.Roles.BASIC
+      return userRoleHierarchy?.find(roleId => roleId === role)
     })
   }
 

packages/client/src/components/app/forms/RelationshipField.svelte

@@ -60,6 +60,12 @@
       // even if they are not in the inital fetch results
       initialValuesProcessed = true
       optionsObj = (fieldState?.value || []).reduce((accumulator, value) => {
+        // fieldState has to be an array of strings to be valid for an update
+        // therefore we cannot guarantee value will be an object
+        // https://linear.app/budibase/issue/BUDI-7577/refactor-the-relationshipfield-component-to-have-better-support-for
+        if (!value._id) {
+          return accumulator
+        }
         accumulator[value._id] = {
           _id: value._id,
           [primaryDisplay]: value.primaryDisplay,

packages/client/src/components/devtools/DevToolsHeader.svelte

@@ -25,7 +25,6 @@
         value: roleId,
       })
     }
-    devToolsStore.actions.changeRole(SELF_ROLE)
     return list
   }
 

packages/client/src/stores/devTools.js

@@ -2,6 +2,7 @@ import { createLocalStorageStore } from "@budibase/frontend-core"
 import { initialise } from "./initialise"
 import { authStore } from "./auth"
 import { API } from "../api"
+import { get } from "svelte/store"
 
 const initialState = {
   visible: false,
@@ -27,9 +28,15 @@ const createDevToolStore = () => {
   }
 
   const changeRole = async role => {
+    if (role === "self") {
+      role = null
+    }
+    if (role === get(store).role) {
+      return
+    }
     store.update(state => ({
       ...state,
-      role: role === "self" ? null : role,
+      role,
     }))
     API.invalidateCache()
     await authStore.actions.fetchUser()

packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte

@@ -3,6 +3,8 @@
   import RelationshipCell from "./RelationshipCell.svelte"
   import { FieldSubtype } from "@budibase/types"
 
+  export let api
+
   const { API } = getContext("grid")
   const { subtype } = $$props.schema
 
@@ -17,8 +19,11 @@
       throw `Search for '${subtype}' not implemented`
     }
 
+    // As we are overriding the search function from RelationshipCell, we want to map one shape to the expected one for the specific API
+    const email = Object.values(searchParams.query.string)[0]
+
     const results = await API.searchUsers({
-      ...searchParams,
+      email,
     })
 
     // Mapping to the expected data within RelationshipCell
@@ -31,6 +36,7 @@
 </script>
 
 <RelationshipCell
+  bind:api
   {...$$props}
   {schema}
   {searchFunction}

packages/frontend-core/src/components/grid/cells/GutterCell.svelte

@@ -17,13 +17,24 @@
   const { config, dispatch, selectedRows } = getContext("grid")
   const svelteDispatch = createEventDispatcher()
 
-  const select = () => {
+  const select = e => {
+    e.stopPropagation()
     svelteDispatch("select")
     const id = row?._id
     if (id) {
       selectedRows.actions.toggleRow(id)
     }
   }
+
+  const bulkDelete = e => {
+    e.stopPropagation()
+    dispatch("request-bulk-delete")
+  }
+
+  const expand = e => {
+    e.stopPropagation()
+    svelteDispatch("expand")
+  }
 </script>
 
 <GridCell
@@ -56,7 +67,7 @@
       {/if}
     {/if}
     {#if rowSelected && $config.canDeleteRows}
-      <div class="delete" on:click={() => dispatch("request-bulk-delete")}>
+      <div class="delete" on:click={bulkDelete}>
         <Icon
           name="Delete"
           size="S"
@@ -65,12 +76,7 @@
       </div>
     {:else}
       <div class="expand" class:visible={$config.canExpandRows && expandable}>
-        <Icon
+        <Icon size="S" name="Maximize" hoverable on:click={expand} />
-          size="S"
-          name="Maximize"
-          hoverable
-          on:click={() => svelteDispatch("expand")}
-        />
       </div>
     {/if}
   </div>

packages/frontend-core/src/components/grid/layout/GridRow.svelte

@@ -17,6 +17,7 @@
     columnHorizontalInversionIndex,
     contentLines,
     isDragging,
+    dispatch,
   } = getContext("grid")
 
   $: rowSelected = !!$selectedRows[row._id]
@@ -30,6 +31,7 @@
   on:focus
   on:mouseenter={$isDragging ? null : () => ($hoveredRowId = row._id)}
   on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)}
+  on:click={() => dispatch("rowclick", row)}
 >
   {#each $renderedColumns as column, columnIdx (column.name)}
     {@const cellId = `${row._id}-${column.name}`}

packages/frontend-core/src/components/grid/layout/StickyColumn.svelte

@@ -74,6 +74,7 @@
           class="row"
           on:mouseenter={$isDragging ? null : () => ($hoveredRowId = row._id)}
           on:mouseleave={$isDragging ? null : () => ($hoveredRowId = null)}
+          on:click={() => dispatch("rowclick", row)}
         >
           <GutterCell {row} {rowFocused} {rowHovered} {rowSelected} />
           {#if $stickyColumn}

packages/sdk/scripts/generate-sdk.sh

@@ -19,7 +19,7 @@ docker run --rm \
   -v ${PWD}/generated:/generated \
   -v ${PWD}/config.json:/config.json \
   -u $(id -u):$(id -g) \
-  swaggerapi/swagger-codegen-cli-v3 generate \
+  swaggerapi/swagger-codegen-cli-v3:3.0.46 generate \
   -i /openapi.yml \
   -l javascript \
   -o /generated \

packages/server/scripts/integrations/postgres/docker-compose.yml

@@ -2,7 +2,7 @@ version: "3.8"
 services:
   db:
     container_name: postgres
-    image: postgres
+    image: postgres:15
     restart: unless-stopped
     environment:
       POSTGRES_USER: root
@@ -25,4 +25,4 @@ services:
       - "5050:80"
 
 volumes:
- pg_data:
+  pg_data:

packages/server/src/api/controllers/query/index.ts

@@ -9,7 +9,7 @@ import { quotas } from "@budibase/pro"
 import { events, context, utils, constants } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import { QueryEvent } from "../../../threads/definitions"
-import { Query } from "@budibase/types"
+import { ConfigType, Query, UserCtx } from "@budibase/types"
 import { ValidQueryNameRegex } from "@budibase/shared-core"
 
 const Runner = new Thread(ThreadType.QUERY, {
@@ -28,11 +28,11 @@ function enrichQueries(input: any) {
   return wasArray ? queries : queries[0]
 }
 
-export async function fetch(ctx: any) {
+export async function fetch(ctx: UserCtx) {
   ctx.body = await sdk.queries.fetch()
 }
 
-const _import = async (ctx: any) => {
+const _import = async (ctx: UserCtx) => {
   const body = ctx.request.body
   const data = body.data
 
@@ -73,7 +73,7 @@ const _import = async (ctx: any) => {
 }
 export { _import as import }
 
-export async function save(ctx: any) {
+export async function save(ctx: UserCtx) {
   const db = context.getAppDB()
   const query = ctx.request.body
 
@@ -100,19 +100,19 @@ export async function save(ctx: any) {
   ctx.message = `Query ${query.name} saved successfully.`
 }
 
-export async function find(ctx: any) {
+export async function find(ctx: UserCtx) {
   const queryId = ctx.params.queryId
   ctx.body = await sdk.queries.find(queryId)
 }
 
 //Required to discern between OIDC OAuth config entries
-function getOAuthConfigCookieId(ctx: any) {
+function getOAuthConfigCookieId(ctx: UserCtx) {
-  if (ctx.user.providerType === constants.Config.OIDC) {
+  if (ctx.user.providerType === ConfigType.OIDC) {
     return utils.getCookie(ctx, constants.Cookie.OIDC_CONFIG)
   }
 }
 
-function getAuthConfig(ctx: any) {
+function getAuthConfig(ctx: UserCtx) {
   const authCookie = utils.getCookie(ctx, constants.Cookie.Auth)
   let authConfigCtx: any = {}
   authConfigCtx["configId"] = getOAuthConfigCookieId(ctx)
@@ -120,7 +120,7 @@ function getAuthConfig(ctx: any) {
   return authConfigCtx
 }
 
-export async function preview(ctx: any) {
+export async function preview(ctx: UserCtx) {
   const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
     ctx.request.body.datasourceId
   )
@@ -129,6 +129,19 @@ export async function preview(ctx: any) {
   // this stops dynamic variables from calling the same query
   const { fields, parameters, queryVerb, transformer, queryId, schema } = query
 
+  let existingSchema = schema
+  if (queryId && !existingSchema) {
+    try {
+      const db = context.getAppDB()
+      const existing = (await db.get(queryId)) as Query
+      existingSchema = existing.schema
+    } catch (err: any) {
+      if (err.status !== 404) {
+        ctx.throw(500, "Unable to retrieve existing query")
+      }
+    }
+  }
+
   const authConfigCtx: any = getAuthConfig(ctx)
 
   try {
@@ -180,6 +193,14 @@ export async function preview(ctx: any) {
         schemaFields[key] = fieldType
       }
     }
+    // if existing schema, update to include any previous schema keys
+    if (existingSchema) {
+      for (let key of Object.keys(schemaFields)) {
+        if (existingSchema[key]?.type) {
+          schemaFields[key] = existingSchema[key].type
+        }
+      }
+    }
     // remove configuration before sending event
     delete datasource.config
     await events.query.previewed(datasource, query)
@@ -189,13 +210,13 @@ export async function preview(ctx: any) {
       info,
       extra,
     }
-  } catch (err) {
+  } catch (err: any) {
     ctx.throw(400, err)
   }
 }
 
 async function execute(
-  ctx: any,
+  ctx: UserCtx,
   opts: any = { rowsOnly: false, isAutomation: false }
 ) {
   const db = context.getAppDB()
@@ -255,17 +276,17 @@ async function execute(
     } else {
       ctx.body = { data: rows, pagination, ...extra, ...info }
     }
-  } catch (err) {
+  } catch (err: any) {
     ctx.throw(400, err)
   }
 }
 
-export async function executeV1(ctx: any) {
+export async function executeV1(ctx: UserCtx) {
   return execute(ctx, { rowsOnly: true, isAutomation: false })
 }
 
 export async function executeV2(
-  ctx: any,
+  ctx: UserCtx,
   { isAutomation }: { isAutomation?: boolean } = {}
 ) {
   return execute(ctx, { rowsOnly: false, isAutomation })
@@ -292,7 +313,7 @@ const removeDynamicVariables = async (queryId: any) => {
   }
 }
 
-export async function destroy(ctx: any) {
+export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
   const queryId = ctx.params.queryId
   await removeDynamicVariables(queryId)

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -340,10 +340,16 @@ export class ExternalRequest<T extends Operation> {
       // one to many
       if (isOneSide(field)) {
         let id = row[key][0]
-        if (typeof row[key] === "string") {
+        if (id) {
-          id = decodeURIComponent(row[key]).match(/\[(.*?)\]/)?.[1]
+          if (typeof row[key] === "string") {
+            id = decodeURIComponent(row[key]).match(/\[(.*?)\]/)?.[1]
+          }
+          newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(id)[0]
+        } else {
+          // Removing from both new and row, as we don't know if it has already been processed
+          row[field.foreignKey || linkTablePrimary] = null
+          newRow[field.foreignKey || linkTablePrimary] = null
         }
-        newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(id)[0]
       }
       // many to many
       else if (field.through) {
@@ -830,10 +836,7 @@ export class ExternalRequest<T extends Operation> {
     // can't really use response right now
     const response = await getDatasourceAndQuery(json)
     // handle many to many relationships now if we know the ID (could be auto increment)
-    if (
+    if (operation !== Operation.READ) {
-      operation !== Operation.READ &&
-      processed.manyRelationships?.length > 0
-    ) {
       await this.handleManyRelationships(
         table._id || "",
         response[0],

packages/server/src/api/controllers/row/external.ts

@@ -108,13 +108,11 @@ export async function save(ctx: UserCtx) {
     row,
   })
 
-  const responseRow = response as { row: Row }
-
   if (!isEqual(table, updatedTable)) {
     await sdk.tables.saveTable(updatedTable)
   }
 
-  const rowId = responseRow.row._id
+  const rowId = response.row._id
   if (rowId) {
     const row = await sdk.rows.external.getRow(tableId, rowId, {
       relationships: true,

packages/server/src/api/controllers/table/index.ts

@@ -14,7 +14,6 @@ import {
   Table,
   TableResponse,
   UserCtx,
-  Datasource,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { jsonFromCsvString } from "../../../utilities/csv"

packages/server/src/api/routes/tests/row.spec.ts

@@ -18,7 +18,6 @@ import {
   SortType,
   StaticQuotaName,
   Table,
-  User,
 } from "@budibase/types"
 import {
   expectAnyExternalColsAttributes,
@@ -1515,9 +1514,82 @@ describe.each([
     })
   })
 
-  describe("bb reference fields", () => {
+  let o2mTable: Table
+  let m2mTable: Table
+  beforeAll(async () => {
+    o2mTable = await config.createTable(
+      { ...generateTableConfig(), name: "o2m" },
+      {
+        skipReassigning: true,
+      }
+    )
+    m2mTable = await config.createTable(
+      { ...generateTableConfig(), name: "m2m" },
+      {
+        skipReassigning: true,
+      }
+    )
+  })
+
+  describe.each([
+    [
+      "relationship fields",
+      () => ({
+        user: {
+          name: "user",
+          relationshipType: RelationshipType.ONE_TO_MANY,
+          type: FieldType.LINK,
+          tableId: o2mTable._id!,
+          fieldName: "fk_o2m",
+        },
+        users: {
+          name: "users",
+          relationshipType: RelationshipType.MANY_TO_MANY,
+          type: FieldType.LINK,
+          tableId: m2mTable._id!,
+          fieldName: "fk_m2m",
+        },
+      }),
+      (tableId: string) =>
+        config.api.row.save(tableId, {
+          name: generator.word(),
+          description: generator.paragraph(),
+          tableId,
+        }),
+      (row: Row) => ({
+        _id: row._id,
+        primaryDisplay: row.name,
+      }),
+    ],
+    [
+      "bb reference fields",
+      () => ({
+        user: {
+          name: "user",
+          relationshipType: RelationshipType.ONE_TO_MANY,
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+        },
+        users: {
+          name: "users",
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+          relationshipType: RelationshipType.MANY_TO_MANY,
+        },
+      }),
+      () => config.createUser(),
+      (row: Row) => ({
+        _id: row._id,
+        email: row.email,
+        firstName: row.firstName,
+        lastName: row.lastName,
+        primaryDisplay: row.email,
+      }),
+    ],
+  ])("links - %s", (__, relSchema, dataGenerator, resultMapper) => {
     let tableId: string
-    let users: User[]
+    let o2mData: Row[]
+    let m2mData: Row[]
 
     beforeAll(async () => {
       const tableConfig = generateTableConfig()
@@ -1532,31 +1604,27 @@ describe.each([
         ...tableConfig,
         schema: {
           ...tableConfig.schema,
-          user: {
+          ...relSchema(),
-            name: "user",
-            type: FieldType.BB_REFERENCE,
-            subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
-            relationshipType: RelationshipType.ONE_TO_MANY,
-          },
-          users: {
-            name: "users",
-            type: FieldType.BB_REFERENCE,
-            subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
-            relationshipType: RelationshipType.MANY_TO_MANY,
-          },
         },
       })
       tableId = table._id!
 
-      users = [
+      o2mData = [
-        await config.createUser(),
+        await dataGenerator(o2mTable._id!),
-        await config.createUser(),
+        await dataGenerator(o2mTable._id!),
-        await config.createUser(),
+        await dataGenerator(o2mTable._id!),
-        await config.createUser(),
+        await dataGenerator(o2mTable._id!),
+      ]
+
+      m2mData = [
+        await dataGenerator(m2mTable._id!),
+        await dataGenerator(m2mTable._id!),
+        await dataGenerator(m2mTable._id!),
+        await dataGenerator(m2mTable._id!),
       ]
     })
 
-    it("can save a row when BB reference fields are empty", async () => {
+    it("can save a row when relationship fields are empty", async () => {
       const rowData = {
         ...basicRow(tableId),
         name: generator.name(),
@@ -1575,13 +1643,13 @@ describe.each([
       })
     })
 
-    it("can save a row with a single BB reference field", async () => {
+    it("can save a row with a single relationship field", async () => {
-      const user = _.sample(users)!
+      const user = _.sample(o2mData)!
       const rowData = {
         ...basicRow(tableId),
         name: generator.name(),
         description: generator.name(),
-        user: user,
+        user: [user],
       }
       const row = await config.api.row.save(tableId, rowData)
 
@@ -1589,24 +1657,17 @@ describe.each([
         name: rowData.name,
         description: rowData.description,
         tableId,
-        user: [
+        user: [user].map(u => resultMapper(u)),
-          {
-            _id: user._id,
-            email: user.email,
-            firstName: user.firstName,
-            lastName: user.lastName,
-            primaryDisplay: user.email,
-          },
-        ],
         _id: expect.any(String),
         _rev: expect.any(String),
         id: isInternal ? undefined : expect.any(Number),
         type: isInternal ? "row" : undefined,
+        [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id,
       })
     })
 
-    it("can save a row with a multiple BB reference field", async () => {
+    it("can save a row with a multiple relationship field", async () => {
-      const selectedUsers = _.sampleSize(users, 2)
+      const selectedUsers = _.sampleSize(m2mData, 2)
       const rowData = {
         ...basicRow(tableId),
         name: generator.name(),
@@ -1619,13 +1680,7 @@ describe.each([
         name: rowData.name,
         description: rowData.description,
         tableId,
-        users: selectedUsers.map(u => ({
+        users: expect.arrayContaining(selectedUsers.map(u => resultMapper(u))),
-          _id: u._id,
-          email: u.email,
-          firstName: u.firstName,
-          lastName: u.lastName,
-          primaryDisplay: u.email,
-        })),
         _id: expect.any(String),
         _rev: expect.any(String),
         id: isInternal ? undefined : expect.any(Number),
@@ -1633,7 +1688,7 @@ describe.each([
       })
     })
 
-    it("can retrieve rows with no populated BB references", async () => {
+    it("can retrieve rows with no populated relationships", async () => {
       const rowData = {
         ...basicRow(tableId),
         name: generator.name(),
@@ -1655,14 +1710,15 @@ describe.each([
       })
     })
 
-    it("can retrieve rows with populated BB references", async () => {
+    it("can retrieve rows with populated relationships", async () => {
-      const [user1, user2] = _.sampleSize(users, 2)
+      const user1 = _.sample(o2mData)!
+      const [user2, user3] = _.sampleSize(m2mData, 2)
 
       const rowData = {
         ...basicRow(tableId),
         name: generator.name(),
         description: generator.name(),
-        users: [user1, user2],
+        users: [user2, user3],
         user: [user1],
       }
       const row = await config.api.row.save(tableId, rowData)
@@ -1672,72 +1728,51 @@ describe.each([
         name: rowData.name,
         description: rowData.description,
         tableId,
-        user: [user1].map(u => ({
+        user: expect.arrayContaining([user1].map(u => resultMapper(u))),
-          _id: u._id,
+        users: expect.arrayContaining([user2, user3].map(u => resultMapper(u))),
-          email: u.email,
-          firstName: u.firstName,
-          lastName: u.lastName,
-          primaryDisplay: u.email,
-        })),
-        users: [user1, user2].map(u => ({
-          _id: u._id,
-          email: u.email,
-          firstName: u.firstName,
-          lastName: u.lastName,
-          primaryDisplay: u.email,
-        })),
         _id: row._id,
         _rev: expect.any(String),
         id: isInternal ? undefined : expect.any(Number),
+        [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user1.id,
         ...defaultRowFields,
       })
     })
 
     it("can update an existing populated row", async () => {
-      const [user1, user2, user3] = _.sampleSize(users, 3)
+      const user = _.sample(o2mData)!
+      const [users1, users2, users3] = _.sampleSize(m2mData, 3)
 
       const rowData = {
         ...basicRow(tableId),
         name: generator.name(),
         description: generator.name(),
-        users: [user1, user2],
+        users: [users1, users2],
       }
       const row = await config.api.row.save(tableId, rowData)
 
       const updatedRow = await config.api.row.save(tableId, {
         ...row,
-        user: [user3],
+        user: [user],
-        users: [user3, user2],
+        users: [users3, users1],
       })
       expect(updatedRow).toEqual({
         name: rowData.name,
         description: rowData.description,
         tableId,
-        user: [
+        user: expect.arrayContaining([user].map(u => resultMapper(u))),
-          {
+        users: expect.arrayContaining(
-            _id: user3._id,
+          [users3, users1].map(u => resultMapper(u))
-            email: user3.email,
+        ),
-            firstName: user3.firstName,
-            lastName: user3.lastName,
-            primaryDisplay: user3.email,
-          },
-        ],
-        users: [user3, user2].map(u => ({
-          _id: u._id,
-          email: u.email,
-          firstName: u.firstName,
-          lastName: u.lastName,
-          primaryDisplay: u.email,
-        })),
         _id: row._id,
         _rev: expect.any(String),
         id: isInternal ? undefined : expect.any(Number),
         type: isInternal ? "row" : undefined,
+        [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id,
       })
     })
 
-    it("can wipe an existing populated BB references in row", async () => {
+    it("can wipe an existing populated relationships in row", async () => {
-      const [user1, user2] = _.sampleSize(users, 2)
+      const [user1, user2] = _.sampleSize(m2mData, 2)
 
       const rowData = {
         ...basicRow(tableId),
@@ -1756,8 +1791,6 @@ describe.each([
         name: rowData.name,
         description: rowData.description,
         tableId,
-        user: isInternal ? null : undefined,
-        users: isInternal ? null : undefined,
         _id: row._id,
         _rev: expect.any(String),
         id: isInternal ? undefined : expect.any(Number),
@@ -1765,34 +1798,35 @@ describe.each([
       })
     })
 
-    it("fetch all will populate the BB references", async () => {
+    it("fetch all will populate the relationships", async () => {
-      const [user1, user2, user3] = _.sampleSize(users, 3)
+      const [user1] = _.sampleSize(o2mData, 1)
+      const [users1, users2, users3] = _.sampleSize(m2mData, 3)
 
       const rows: {
         name: string
         description: string
-        user?: User[]
+        user?: Row[]
-        users?: User[]
+        users?: Row[]
         tableId: string
       }[] = [
         {
           ...basicRow(tableId),
           name: generator.name(),
           description: generator.name(),
-          users: [user1, user2],
+          users: [users1, users2],
         },
         {
           ...basicRow(tableId),
           name: generator.name(),
           description: generator.name(),
           user: [user1],
-          users: [user1, user3],
+          users: [users1, users3],
         },
         {
           ...basicRow(tableId),
           name: generator.name(),
           description: generator.name(),
-          users: [user3],
+          users: [users3],
         },
       ]
 
@@ -1808,57 +1842,50 @@ describe.each([
             name: r.name,
             description: r.description,
             tableId,
-            user: r.user?.map(u => ({
+            user: r.user?.map(u => resultMapper(u)),
-              _id: u._id,
+            users: r.users?.length
-              email: u.email,
+              ? expect.arrayContaining(r.users?.map(u => resultMapper(u)))
-              firstName: u.firstName,
+              : undefined,
-              lastName: u.lastName,
-              primaryDisplay: u.email,
-            })),
-            users: r.users?.map(u => ({
-              _id: u._id,
-              email: u.email,
-              firstName: u.firstName,
-              lastName: u.lastName,
-              primaryDisplay: u.email,
-            })),
             _id: expect.any(String),
             _rev: expect.any(String),
             id: isInternal ? undefined : expect.any(Number),
+            [`fk_${o2mTable.name}_fk_o2m`]:
+              isInternal || !r.user?.length ? undefined : r.user[0].id,
             ...defaultRowFields,
           }))
         )
       )
     })
 
-    it("search all will populate the BB references", async () => {
+    it("search all will populate the relationships", async () => {
-      const [user1, user2, user3] = _.sampleSize(users, 3)
+      const [user1] = _.sampleSize(o2mData, 1)
+      const [users1, users2, users3] = _.sampleSize(m2mData, 3)
 
       const rows: {
         name: string
         description: string
-        user?: User[]
+        user?: Row[]
-        users?: User[]
+        users?: Row[]
         tableId: string
       }[] = [
         {
           ...basicRow(tableId),
           name: generator.name(),
           description: generator.name(),
-          users: [user1, user2],
+          users: [users1, users2],
         },
         {
           ...basicRow(tableId),
           name: generator.name(),
           description: generator.name(),
           user: [user1],
-          users: [user1, user3],
+          users: [users1, users3],
         },
         {
           ...basicRow(tableId),
           name: generator.name(),
           description: generator.name(),
-          users: [user3],
+          users: [users3],
         },
       ]
 
@@ -1874,23 +1901,15 @@ describe.each([
             name: r.name,
             description: r.description,
             tableId,
-            user: r.user?.map(u => ({
+            user: r.user?.map(u => resultMapper(u)),
-              _id: u._id,
+            users: r.users?.length
-              email: u.email,
+              ? expect.arrayContaining(r.users?.map(u => resultMapper(u)))
-              firstName: u.firstName,
+              : undefined,
-              lastName: u.lastName,
-              primaryDisplay: u.email,
-            })),
-            users: r.users?.map(u => ({
-              _id: u._id,
-              email: u.email,
-              firstName: u.firstName,
-              lastName: u.lastName,
-              primaryDisplay: u.email,
-            })),
             _id: expect.any(String),
             _rev: expect.any(String),
             id: isInternal ? undefined : expect.any(Number),
+            [`fk_${o2mTable.name}_fk_o2m`]:
+              isInternal || !r.user?.length ? undefined : r.user[0].id,
             ...defaultRowFields,
           }))
         ),

packages/server/src/automations/steps/make.ts

@@ -78,8 +78,7 @@ export const definition: AutomationStepSchema = {
 }
 
 export async function run({ inputs }: AutomationStepInput) {
-  //TODO - Remove deprecated values 1,2,3,4,5 after November 2023
+  const { url, body } = inputs
-  const { url, value1, value2, value3, value4, value5, body } = inputs
 
   let payload = {}
   try {
@@ -104,11 +103,6 @@ export async function run({ inputs }: AutomationStepInput) {
     response = await fetch(url, {
       method: "post",
       body: JSON.stringify({
-        value1,
-        value2,
-        value3,
-        value4,
-        value5,
         ...payload,
       }),
       headers: {

packages/server/src/automations/steps/zapier.ts

@@ -71,8 +71,7 @@ export const definition: AutomationStepSchema = {
 }
 
 export async function run({ inputs }: AutomationStepInput) {
-  //TODO - Remove deprecated values 1,2,3,4,5 after November 2023
+  const { url, body } = inputs
-  const { url, value1, value2, value3, value4, value5, body } = inputs
 
   let payload = {}
   try {
@@ -100,11 +99,6 @@ export async function run({ inputs }: AutomationStepInput) {
       method: "post",
       body: JSON.stringify({
         platform: "budibase",
-        value1,
-        value2,
-        value3,
-        value4,
-        value5,
         ...payload,
       }),
       headers: {

packages/server/src/db/linkedRows/LinkController.ts

@@ -308,12 +308,19 @@ class LinkController {
         }
       })
     )
-    // remove schema from other table
+    try {
-    let linkedTable = await this._db.get<Table>(field.tableId)
+      // remove schema from other table, if it exists
-    if (field.fieldName) {
+      let linkedTable = await this._db.get<Table>(field.tableId)
-      delete linkedTable.schema[field.fieldName]
+      if (field.fieldName) {
+        delete linkedTable.schema[field.fieldName]
+      }
+      await this._db.put(linkedTable)
+    } catch (error: any) {
+      // ignore missing to ensure broken relationship columns can be deleted
+      if (error.statusCode !== 404) {
+        throw error
+      }
     }
-    await this._db.put(linkedTable)
   }
 
   /**

packages/server/src/db/tests/linkController.spec.js

@@ -233,4 +233,19 @@ describe("test the link controller", () => {
     }
     await config.updateTable(table)
   })
+
+  it("should be able to remove a linked field from a table, even if the linked table does not exist", async () => {
+    await createLinkedRow()
+    await createLinkedRow("link2")
+    table1.schema["link"].tableId = "not_found"
+    const controller = await createLinkController(table1, null, table1)
+    await context.doInAppContext(appId, async () => {
+      let before = await controller.getTableLinkDocs()
+      await controller.removeFieldFromTable("link")
+      let after = await controller.getTableLinkDocs()
+      expect(before.length).toEqual(2)
+      // shouldn't delete the other field
+      expect(after.length).toEqual(1)
+    })
+  })
 })

packages/server/src/sdk/app/rows/search/external.ts

@@ -16,6 +16,7 @@ import { cleanExportRows } from "../utils"
 import { utils } from "@budibase/shared-core"
 import { ExportRowsParams, ExportRowsResult } from "../search"
 import { HTTPError, db } from "@budibase/backend-core"
+import { searchInputMapping } from "./utils"
 import pick from "lodash/pick"
 import { outputProcessing } from "../../../../utilities/rowProcessor"
 
@@ -50,7 +51,10 @@ export async function search(options: SearchParams) {
       [params.sort]: { direction },
     }
   }
+
   try {
+    const table = await sdk.tables.getTable(tableId)
+    options = searchInputMapping(table, options)
     let rows = (await handleRequest(Operation.READ, tableId, {
       filters: query,
       sort,
@@ -76,7 +80,6 @@ export async function search(options: SearchParams) {
       rows = rows.map((r: any) => pick(r, fields))
     }
 
-    const table = await sdk.tables.getTable(tableId)
     rows = await outputProcessing(table, rows, { preserveLinks: true })
 
     // need wrapper object for bookmarks etc when paginating

packages/server/src/sdk/app/rows/search/internal.ts

@@ -29,6 +29,7 @@ import {
 } from "../../../../api/controllers/view/utils"
 import sdk from "../../../../sdk"
 import { ExportRowsParams, ExportRowsResult } from "../search"
+import { searchInputMapping } from "./utils"
 import pick from "lodash/pick"
 
 export async function search(options: SearchParams) {
@@ -47,9 +48,9 @@ export async function search(options: SearchParams) {
     disableEscaping: options.disableEscaping,
   }
 
-  let table
+  let table = await sdk.tables.getTable(tableId)
+  options = searchInputMapping(table, options)
   if (params.sort && !params.sortType) {
-    table = await sdk.tables.getTable(tableId)
     const schema = table.schema
     const sortField = schema[params.sort]
     params.sortType = sortField.type === "number" ? "number" : "string"
@@ -68,7 +69,6 @@ export async function search(options: SearchParams) {
     if (tableId === InternalTables.USER_METADATA) {
       response.rows = await getGlobalUsersFromMetadata(response.rows)
     }
-    table = table || (await sdk.tables.getTable(tableId))
 
     if (options.fields) {
       const fields = [...options.fields, ...db.CONSTANT_INTERNAL_ROW_COLS]

packages/server/src/sdk/app/rows/search/tests/utils.spec.ts

@@ -0,0 +1,77 @@
+import { searchInputMapping } from "../utils"
+import { db as dbCore } from "@budibase/backend-core"
+import {
+  FieldType,
+  FieldTypeSubtypes,
+  Table,
+  SearchParams,
+} from "@budibase/types"
+
+const tableId = "ta_a"
+const tableWithUserCol: Table = {
+  _id: tableId,
+  name: "table",
+  schema: {
+    user: {
+      name: "user",
+      type: FieldType.BB_REFERENCE,
+      subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+    },
+  },
+}
+
+describe("searchInputMapping", () => {
+  const globalUserId = dbCore.generateGlobalUserID()
+  const userMedataId = dbCore.generateUserMetadataID(globalUserId)
+
+  it("should be able to map ro_ to global user IDs", () => {
+    const params: SearchParams = {
+      tableId,
+      query: {
+        equal: {
+          "1:user": userMedataId,
+        },
+      },
+    }
+    const output = searchInputMapping(tableWithUserCol, params)
+    expect(output.query.equal!["1:user"]).toBe(globalUserId)
+  })
+
+  it("should handle array of user IDs", () => {
+    const params: SearchParams = {
+      tableId,
+      query: {
+        oneOf: {
+          "1:user": [userMedataId, globalUserId],
+        },
+      },
+    }
+    const output = searchInputMapping(tableWithUserCol, params)
+    expect(output.query.oneOf!["1:user"]).toStrictEqual([
+      globalUserId,
+      globalUserId,
+    ])
+  })
+
+  it("shouldn't change any other input", () => {
+    const email = "test@test.com"
+    const params: SearchParams = {
+      tableId,
+      query: {
+        equal: {
+          "1:user": email,
+        },
+      },
+    }
+    const output = searchInputMapping(tableWithUserCol, params)
+    expect(output.query.equal!["1:user"]).toBe(email)
+  })
+
+  it("shouldn't error if no query supplied", () => {
+    const params: any = {
+      tableId,
+    }
+    const output = searchInputMapping(tableWithUserCol, params)
+    expect(output.query).toBeUndefined()
+  })
+})

packages/server/src/sdk/app/rows/search/utils.ts

@@ -0,0 +1,76 @@
+import {
+  FieldType,
+  FieldTypeSubtypes,
+  SearchParams,
+  Table,
+  DocumentType,
+  SEPARATOR,
+} from "@budibase/types"
+import { db as dbCore } from "@budibase/backend-core"
+
+function findColumnInQueries(
+  column: string,
+  options: SearchParams,
+  callback: (filter: any) => any
+) {
+  if (!options.query) {
+    return
+  }
+  for (let filterBlock of Object.values(options.query)) {
+    if (typeof filterBlock !== "object") {
+      continue
+    }
+    for (let [key, filter] of Object.entries(filterBlock)) {
+      if (key.endsWith(column)) {
+        filterBlock[key] = callback(filter)
+      }
+    }
+  }
+}
+
+function userColumnMapping(column: string, options: SearchParams) {
+  findColumnInQueries(column, options, (filterValue: any): any => {
+    const isArray = Array.isArray(filterValue),
+      isString = typeof filterValue === "string"
+    if (!isString && !isArray) {
+      return filterValue
+    }
+    const processString = (input: string) => {
+      const rowPrefix = DocumentType.ROW + SEPARATOR
+      if (input.startsWith(rowPrefix)) {
+        return dbCore.getGlobalIDFromUserMetadataID(input)
+      } else {
+        return input
+      }
+    }
+    if (isArray) {
+      return filterValue.map(el => {
+        if (typeof el === "string") {
+          return processString(el)
+        } else {
+          return el
+        }
+      })
+    } else {
+      return processString(filterValue)
+    }
+  })
+}
+
+// maps through the search parameters to check if any of the inputs are invalid
+// based on the table schema, converts them to something that is valid.
+export function searchInputMapping(table: Table, options: SearchParams) {
+  if (!table?.schema) {
+    return options
+  }
+  for (let [key, column] of Object.entries(table.schema)) {
+    switch (column.type) {
+      case FieldType.BB_REFERENCE:
+        if (column.subtype === FieldTypeSubtypes.BB_REFERENCE.USER) {
+          userColumnMapping(key, options)
+        }
+        break
+    }
+  }
+  return options
+}

packages/server/src/threads/definitions.ts

@@ -11,12 +11,7 @@ export interface QueryEvent {
   queryId: string
   environmentVariables?: Record<string, string>
   ctx?: any
-  schema?: {
+  schema?: Record<string, { name?: string; type: string }>
-    [key: string]: {
-      name: string
-      type: string
-    }
-  }
 }
 
 export interface QueryVariable {

packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts

@@ -48,7 +48,7 @@ export async function processOutputBBReferences(
 ) {
   if (typeof value !== "string") {
     // Already processed or nothing to process
-    return value
+    return value || undefined
   }
 
   const ids = value.split(",").filter(id => !!id)

packages/server/src/utilities/rowProcessor/index.ts

@@ -11,6 +11,7 @@ import {
   processInputBBReferences,
   processOutputBBReferences,
 } from "./bbReferenceProcessor"
+import { isExternalTable } from "../../integrations/utils"
 export * from "./utils"
 
 type AutoColumnProcessingOpts = {
@@ -234,9 +235,6 @@ export async function outputProcessing<T extends Row[] | Row>(
       }
     } else if (column.type == FieldTypes.BB_REFERENCE) {
       for (let row of enriched) {
-        if (row[property] == null) {
-          continue
-        }
         row[property] = await processOutputBBReferences(
           row[property],
           column.subtype as FieldSubtype
@@ -250,6 +248,16 @@ export async function outputProcessing<T extends Row[] | Row>(
       enriched
     )) as Row[]
   }
+  // remove null properties to match internal API
+  if (isExternalTable(table._id!)) {
+    for (let row of enriched) {
+      for (let key of Object.keys(row)) {
+        if (row[key] === null) {
+          delete row[key]
+        }
+      }
+    }
+  }
   return (wasArray ? enriched : enriched[0]) as T
 }
 

packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts

@@ -66,7 +66,7 @@ describe("rowProcessor - outputProcessing", () => {
     )
   })
 
-  it("does not fetch bb references when fields are empty", async () => {
+  it("process output even when the field is not empty", async () => {
     const table: Table = {
       _id: generator.guid(),
       name: "TestTable",
@@ -100,7 +100,7 @@ describe("rowProcessor - outputProcessing", () => {
 
     expect(result).toEqual({ name: "Jack" })
 
-    expect(bbReferenceProcessor.processOutputBBReferences).not.toBeCalled()
+    expect(bbReferenceProcessor.processOutputBBReferences).toBeCalledTimes(1)
   })
 
   it("does not fetch bb references when not in the schema", async () => {

packages/shared-core/src/filters.ts

@@ -14,7 +14,6 @@ const HBS_REGEX = /{{([^{].*?)}}/g
 
 /**
  * Returns the valid operator options for a certain data type
- * @param type the data type
  */
 export const getValidOperatorsForType = (
   type: FieldType,
@@ -44,22 +43,24 @@ export const getValidOperatorsForType = (
     value: string
     label: string
   }[] = []
-  if (type === "string") {
+  if (type === FieldType.STRING) {
     ops = stringOps
-  } else if (type === "number" || type === "bigint") {
+  } else if (type === FieldType.NUMBER || type === FieldType.BIGINT) {
     ops = numOps
-  } else if (type === "options") {
+  } else if (type === FieldType.OPTIONS) {
     ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In]
-  } else if (type === "array") {
+  } else if (type === FieldType.ARRAY) {
     ops = [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty, Op.ContainsAny]
-  } else if (type === "boolean") {
+  } else if (type === FieldType.BOOLEAN) {
     ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
-  } else if (type === "longform") {
+  } else if (type === FieldType.LONGFORM) {
     ops = stringOps
-  } else if (type === "datetime") {
+  } else if (type === FieldType.DATETIME) {
     ops = numOps
-  } else if (type === "formula") {
+  } else if (type === FieldType.FORMULA) {
     ops = stringOps.concat([Op.MoreThan, Op.LessThan])
+  } else if (type === FieldType.BB_REFERENCE) {
+    ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In]
   }
 
   // Only allow equal/not equal for _id in SQL tables

packages/shared-core/src/helpers/integrations.ts

@@ -14,5 +14,5 @@ export function isSQL(datasource: Datasource): boolean {
     SourceName.MYSQL,
     SourceName.ORACLE,
   ]
-  return SQL.indexOf(datasource.source) !== -1
+  return SQL.indexOf(datasource.source) !== -1 || datasource.isSQL === true
 }

packages/types/src/documents/app/datasource.ts

@@ -9,6 +9,7 @@ export interface Datasource extends Document {
   // the config is defined by the schema
   config?: Record<string, any>
   plus?: boolean
+  isSQL?: boolean
   entities?: {
     [key: string]: Table
   }

packages/types/src/documents/app/query.ts

@@ -6,7 +6,7 @@ export interface Query extends Document {
   parameters: QueryParameter[]
   fields: RestQueryFields | any
   transformer: string | null
-  schema: any
+  schema: Record<string, { name?: string; type: string }>
   readable: boolean
   queryVerb: string
 }

packages/types/src/sdk/datasources.ts

@@ -140,6 +140,7 @@ export interface DatasourceConfig {
 export interface Integration {
   docs: string
   plus?: boolean
+  isSQL?: boolean
   auth?: { type: string }
   features?: Partial<Record<DatasourceFeature, boolean>>
   relationships?: boolean

packages/types/src/sdk/koa.ts

@@ -1,5 +1,5 @@
 import { Context, Request } from "koa"
-import { User, Role, UserRoles, Account } from "../documents"
+import { User, Role, UserRoles, Account, ConfigType } from "../documents"
 import { FeatureFlag, License } from "../sdk"
 import { Files } from "formidable"
 
@@ -13,6 +13,7 @@ export interface ContextUser extends Omit<User, "roles"> {
   csrfToken?: string
   featureFlags?: FeatureFlag[]
   accountPortalAccess?: boolean
+  providerType?: ConfigType
   account?: Account
 }
 

packages/worker/src/api/routes/global/tests/scim.spec.ts

@@ -11,7 +11,7 @@ import { TestConfiguration } from "../../../../tests"
 import { events } from "@budibase/backend-core"
 
 // this test can 409 - retries reduce issues with this
-jest.retryTimes(2)
+jest.retryTimes(2, { logErrorsBeforeRetry: true })
 jest.setTimeout(30000)
 
 mocks.licenses.useScimIntegration()