Merge branch 'develop' into grid-clipboard

.github/workflows/close-featurebranch.yml

@@ -14,7 +14,7 @@ jobs:
       - uses: passeidireto/trigger-external-workflow-action@main
         env:
           PAYLOAD_BRANCH: ${{ github.head_ref }}
-          PAYLOAD_PR_NUMBER: ${{ github.ref }}
+          PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-close

.github/workflows/deploy-featurebranch.yml

@@ -13,7 +13,7 @@ jobs:
       - uses: passeidireto/trigger-external-workflow-action@main
         env:
           PAYLOAD_BRANCH: ${{ github.head_ref }}
-          PAYLOAD_PR_NUMBER: ${{ github.ref }}
+          PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-deploy

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.10.16-alpha.11",
+  "version": "2.11.5-alpha.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/src/plugin/utils.ts

@@ -6,6 +6,7 @@ import {
   AutomationStepIdArray,
   AutomationIOType,
   AutomationCustomIOType,
+  DatasourceFeature,
 } from "@budibase/types"
 import joi from "joi"
 
@@ -67,9 +68,27 @@ function validateDatasource(schema: any) {
     version: joi.string().optional(),
     schema: joi.object({
       docs: joi.string(),
+      plus: joi.boolean().optional(),
+      isSQL: joi.boolean().optional(),
+      auth: joi
+        .object({
+          type: joi.string().required(),
+        })
+        .optional(),
+      features: joi
+        .object(
+          Object.fromEntries(
+            Object.values(DatasourceFeature).map(key => [
+              key,
+              joi.boolean().optional(),
+            ])
+          )
+        )
+        .optional(),
+      relationships: joi.boolean().optional(),
+      description: joi.string().required(),
       friendlyName: joi.string().required(),
       type: joi.string().allow(...DATASOURCE_TYPES),
-      description: joi.string().required(),
       datasource: joi.object().pattern(joi.string(), fieldValidator).required(),
       query: joi
         .object()

packages/builder/src/builderStore/store/automation/index.js

@@ -221,18 +221,6 @@ const automationActions = store => ({
     newAutomation.definition.steps.splice(blockIdx, 0, block)
     await store.actions.save(newAutomation)
   },
-  /**
-   * "rowControl" appears to be the name of the flag used to determine whether
-   * a certain automation block uses values or bindings as inputs
-   */
-  toggleRowControl: async (block, rowControl) => {
-    const newBlock = { ...block, rowControl }
-    const newAutomation = store.actions.getUpdatedDefinition(
-      get(selectedAutomation),
-      newBlock
-    )
-    await store.actions.save(newAutomation)
-  },
   deleteAutomationBlock: async block => {
     const automation = get(selectedAutomation)
     let newAutomation = cloneDeep(automation)

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -7,7 +7,6 @@
     Detail,
     Modal,
     Button,
-    Select,
     ActionButton,
     notifications,
     Label,
@@ -39,9 +38,6 @@
     step => step.stepId === ActionStepID.COLLECT
   )
   $: automationId = $selectedAutomation?._id
-  $: showBindingPicker =
-    block.stepId === ActionStepID.CREATE_ROW ||
-    block.stepId === ActionStepID.UPDATE_ROW
   $: isTrigger = block.type === "TRIGGER"
   $: steps = $selectedAutomation?.definition?.steps ?? []
   $: blockIdx = steps.findIndex(step => step.id === block.id)
@@ -96,15 +92,6 @@
     }
   }
 
-  /**
-   * "rowControl" appears to be the name of the flag used to determine whether
-   * a certain automation block uses values or bindings as inputs
-   */
-  function toggleRowControl(evt) {
-    const rowControl = evt.detail !== "Use values"
-    automationStore.actions.toggleRowControl(block, rowControl)
-  }
-
   async function addLooping() {
     const loopDefinition = $automationStore.blockDefinitions.ACTION.LOOP
     const loopBlock = automationStore.actions.constructBlock(
@@ -189,16 +176,6 @@
                   Add Looping
                 </ActionButton>
               {/if}
-              {#if showBindingPicker}
-                <Select
-                  on:change={toggleRowControl}
-                  defaultValue="Use values"
-                  autoWidth
-                  value={block.rowControl ? "Use bindings" : "Use values"}
-                  options={["Use values", "Use bindings"]}
-                  placeholder={null}
-                />
-              {/if}
               <ActionButton
                 on:click={() => deleteStep()}
                 icon="DeleteOutline"

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -23,6 +23,7 @@
   import { environment, licensing } from "stores/portal"
   import WebhookDisplay from "../Shared/WebhookDisplay.svelte"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
+  import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte"
   import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import CodeEditorModal from "./CodeEditorModal.svelte"
   import QuerySelector from "./QuerySelector.svelte"
@@ -82,33 +83,6 @@
       ? [hbAutocomplete([...bindingsToCompletions(bindings, codeMode)])]
       : []
 
-  /**
-   * TODO - Remove after November 2023
-   * *******************************
-   * Code added to provide backwards compatibility between Values 1,2,3,4,5
-   * and the new JSON body.
-   */
-  let deprecatedSchemaProperties
-  $: {
-    if (block?.stepId === "integromat" || block?.stepId === "zapier") {
-      deprecatedSchemaProperties = schemaProperties.filter(
-        prop => !prop[0].startsWith("value")
-      )
-      if (!deprecatedSchemaProperties.map(entry => entry[0]).includes("body")) {
-        deprecatedSchemaProperties.push([
-          "body",
-          {
-            title: "Payload",
-            type: "json",
-          },
-        ])
-      }
-    } else {
-      deprecatedSchemaProperties = schemaProperties
-    }
-  }
-  /****************************************************/
-
   const getInputData = (testData, blockInputs) => {
     // Test data is not cloned for reactivity
     let newInputData = testData || cloneDeep(blockInputs)
@@ -118,30 +92,6 @@
       newInputData = cloneDeep(blockInputs)
     }
 
-    /**
-     * TODO - Remove after November 2023
-     * *******************************
-     * Code added to provide backwards compatibility between Values 1,2,3,4,5
-     * and the new JSON body.
-     */
-    if (
-      (block?.stepId === "integromat" || block?.stepId === "zapier") &&
-      !newInputData?.body?.value
-    ) {
-      let deprecatedValues = {
-        ...newInputData,
-      }
-      delete deprecatedValues.url
-      delete deprecatedValues.body
-      newInputData = {
-        url: newInputData.url,
-        body: {
-          value: JSON.stringify(deprecatedValues),
-        },
-      }
-    }
-    /**********************************/
-
     inputData = newInputData
     setDefaultEnumValues()
   }
@@ -337,7 +287,7 @@
 </script>
 
 <div class="fields">
-  {#each deprecatedSchemaProperties as [key, value]}
+  {#each schemaProperties as [key, value]}
     {#if canShowField(key, value)}
       <div class="block-field">
         {#if key !== "fields" && value.type !== "boolean"}
@@ -362,18 +312,6 @@
             mode="json"
             value={inputData[key]?.value}
             on:change={e => {
-              /**
-               * TODO - Remove after November 2023
-               * *******************************
-               * Code added to provide backwards compatibility between Values 1,2,3,4,5
-               * and the new JSON body.
-               */
-              delete inputData.value1
-              delete inputData.value2
-              delete inputData.value3
-              delete inputData.value4
-              delete inputData.value5
-              /***********************/
               onChange(e, key)
             }}
           />
@@ -386,10 +324,23 @@
             />
           </div>
         {:else if value.type === "date"}
-          <DatePicker
+          <DrawerBindableSlot
+            fillWidth
+            title={value.title}
+            panel={AutomationBindingPanel}
+            type={"date"}
             value={inputData[key]}
             on:change={e => onChange(e, key)}
-          />
+            {bindings}
+            allowJS={true}
+            updateOnChange={false}
+            drawerLeft="260px"
+          >
+            <DatePicker
+              value={inputData[key]}
+              on:change={e => onChange(e, key)}
+            />
+          </DrawerBindableSlot>
         {:else if value.customType === "column"}
           <Select
             on:change={e => onChange(e, key)}
@@ -469,7 +420,6 @@
           />
         {:else if value.customType === "row"}
           <RowSelector
-            {block}
             value={inputData[key]}
             meta={inputData["meta"] || {}}
             on:change={e => {

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -1,18 +1,16 @@
 <script>
   import { tables } from "stores/backend"
   import { Select, Checkbox } from "@budibase/bbui"
-  import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
-  import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
   import { createEventDispatcher } from "svelte"
   import RowSelectorTypes from "./RowSelectorTypes.svelte"
-  import ModalBindableInput from "../../common/bindings/ModalBindableInput.svelte"
+  import DrawerBindableSlot from "../../common/bindings/DrawerBindableSlot.svelte"
+  import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte"
 
   const dispatch = createEventDispatcher()
 
   export let value
   export let meta
   export let bindings
-  export let block
   export let isTestModal
   export let isUpdateRow
 
@@ -25,16 +23,6 @@
   let table
   let schemaFields
 
-  let placeholders = {
-    number: 10,
-    boolean: "true",
-    datetime: "2022-02-16T12:00:00.000Z ",
-    options: "1",
-    array: "1 2 3 4",
-    link: "ro_ta_123_456",
-    longform: "long form text",
-  }
-  $: rowControl = block.rowControl
   $: {
     table = $tables.list.find(table => table._id === value?.tableId)
     schemaFields = Object.entries(table?.schema ?? {})
@@ -57,19 +45,13 @@
       return value
     }
 
-    if (type === "boolean") {
-      if (typeof value === "boolean") {
-        return value
-      }
-      return value === "true"
-    }
     if (type === "number") {
       if (typeof value === "number") {
         return value
       }
       return Number(value)
     }
-    if (type === "options") {
+    if (type === "options" || type === "boolean") {
       return value
     }
     if (type === "array") {
@@ -127,47 +109,40 @@
 {#if schemaFields.length}
   <div class="schema-fields">
     {#each schemaFields as [field, schema]}
-      {#if !schema.autocolumn}
+      {#if !schema.autocolumn && schema.type !== "attachment"}
-        {#if schema.type !== "attachment"}
+        <DrawerBindableSlot
-          {#if !rowControl}
+          fillWidth
-            <RowSelectorTypes
+          title={value.title}
-              {isTestModal}
+          label={field}
-              {field}
+          panel={AutomationBindingPanel}
-              {schema}
+          type={schema.type}
-              bindings={parsedBindings}
+          {schema}
-              {value}
+          value={value[field]}
-              {onChange}
+          on:change={e => onChange(e, field)}
-            />
+          {bindings}
-          {:else}
+          allowJS={true}
-            <div>
+          updateOnChange={false}
-              <svelte:component
+          drawerLeft="260px"
-                this={isTestModal ? ModalBindableInput : DrawerBindableInput}
+        >
-                placeholder={placeholders[schema.type]}
+          <RowSelectorTypes
-                panel={AutomationBindingPanel}
+            {isTestModal}
-                value={Array.isArray(value[field])
+            {field}
-                  ? value[field].join(",")
+            {schema}
-                  : value[field]}
+            bindings={parsedBindings}
-                on:change={e => onChange(e, field, schema.type)}
+            {value}
-                label={field}
+            {onChange}
-                type="string"
+          />
-                bindings={parsedBindings}
+        </DrawerBindableSlot>
-                fillWidth={true}
+      {/if}
-                allowJS={true}
+      {#if isUpdateRow && schema.type === "link"}
-                updateOnChange={false}
+        <div class="checkbox-field">
-              />
+          <Checkbox
-              {#if isUpdateRow && schema.type === "link"}
+            value={meta.fields?.[field]?.clearRelationships}
-                <div class="checkbox-field">
+            text={"Clear relationships if empty?"}
-                  <Checkbox
+            size={"S"}
-                    value={meta.fields?.[field]?.clearRelationships}
+            on:change={e => onChangeSetting(e, field)}
-                    text={"Clear relationships if empty?"}
+          />
-                    size={"S"}
+        </div>
-                    on:change={e => onChangeSetting(e, field)}
-                  />
-                </div>
-              {/if}
-            </div>
-          {/if}
-        {/if}
       {/if}
     {/each}
   </div>

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -1,7 +1,6 @@
 <script>
   import {
     Select,
-    Toggle,
     DatePicker,
     Multiselect,
     TextArea,
@@ -45,10 +44,14 @@
     on:change={e => onChange(e, field)}
   />
 {:else if schema.type === "boolean"}
-  <Toggle
+  <Select
-    text={field}
-    value={value[field]}
     on:change={e => onChange(e, field)}
+    label={field}
+    value={value[field]}
+    options={[
+      { label: "True", value: "true" },
+      { label: "False", value: "false" },
+    ]}
   />
 {:else if schema.type === "array"}
   <Multiselect

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -85,7 +85,7 @@
   }
 
   let relationshipMap = {
-    [RelationshipType.MANY_TO_ONE]: {
+    [RelationshipType.ONE_TO_MANY]: {
       part1: PrettyRelationshipDefinitions.MANY,
       part2: PrettyRelationshipDefinitions.ONE,
     },
@@ -93,7 +93,7 @@
       part1: PrettyRelationshipDefinitions.MANY,
       part2: PrettyRelationshipDefinitions.MANY,
     },
-    [RelationshipType.ONE_TO_MANY]: {
+    [RelationshipType.MANY_TO_ONE]: {
       part1: PrettyRelationshipDefinitions.ONE,
       part2: PrettyRelationshipDefinitions.MANY,
     },
@@ -277,10 +277,7 @@
       dispatch("updatecolumns")
       gridDispatch("close-edit-column")
 
-      if (
+      if (saveColumn.type === LINK_TYPE) {
-        saveColumn.type === LINK_TYPE &&
-        saveColumn.relationshipType === RelationshipType.MANY_TO_MANY
-      ) {
         // Fetching the new tables
         tables.fetch()
         // Fetching the new relationships
@@ -312,6 +309,11 @@
         confirmDeleteDialog.hide()
         dispatch("updatecolumns")
         gridDispatch("close-edit-column")
+
+        if (editableColumn.type === LINK_TYPE) {
+          // Updating the relationships
+          datasources.fetch()
+        }
       }
     } catch (error) {
       notifications.error(`Error deleting column: ${error.message}`)

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -57,7 +57,8 @@
     label: table.name,
     value: table._id,
   }))
-  $: valid = getErrorCount(errors) === 0 && allRequiredAttributesSet()
+  $: valid =
+    getErrorCount(errors) === 0 && allRequiredAttributesSet(relationshipType)
   $: isManyToMany = relationshipType === RelationshipType.MANY_TO_MANY
   $: isManyToOne = relationshipType === RelationshipType.MANY_TO_ONE
 
@@ -114,7 +115,7 @@
     return Object.entries(errors).filter(entry => !!entry[1]).length
   }
 
-  function allRequiredAttributesSet() {
+  function allRequiredAttributesSet(relationshipType) {
     const base = getTable(fromId) && getTable(toId) && fromColumn && toColumn
     if (relationshipType === RelationshipType.MANY_TO_ONE) {
       return base && fromPrimary && fromForeign
@@ -124,9 +125,10 @@
   }
 
   function validate() {
-    if (!allRequiredAttributesSet() && !hasValidated) {
+    if (!allRequiredAttributesSet(relationshipType) && !hasValidated) {
       return
     }
+
     hasValidated = true
     errorChecker.setType(relationshipType)
     const fromTable = getTable(fromId),

packages/builder/src/components/common/LinkedRowSelector.svelte

@@ -11,11 +11,11 @@
   const dispatch = createEventDispatcher()
 
   let rows = []
-  let linkedIds = (Array.isArray(linkedRows) ? linkedRows : [])?.map(
+  let linkedIds = []
+
+  $: linkedIds = (Array.isArray(linkedRows) ? linkedRows : [])?.map(
     row => row?._id || row
   )
-
-  $: linkedRows = linkedIds
   $: label = capitalise(schema.name)
   $: linkedTableId = schema.tableId
   $: linkedTable = $tables.list.find(table => table._id === linkedTableId)

packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte

@@ -0,0 +1,250 @@
+<script>
+  import { Icon, Input, Drawer, Button } from "@budibase/bbui"
+  import {
+    readableToRuntimeBinding,
+    runtimeToReadableBinding,
+  } from "builderStore/dataBinding"
+
+  import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
+  import { createEventDispatcher, setContext } from "svelte"
+  import { isJSBinding } from "@budibase/string-templates"
+
+  export let panel = ClientBindingPanel
+  export let value = ""
+  export let bindings = []
+  export let title = "Bindings"
+  export let placeholder
+  export let label
+  export let disabled = false
+  export let fillWidth
+  export let allowJS = true
+  export let allowHelpers = true
+  export let updateOnChange = true
+  export let drawerLeft
+  export let type
+  export let schema
+
+  const dispatch = createEventDispatcher()
+  let bindingDrawer
+  let valid = true
+  let currentVal = value
+
+  $: readableValue = runtimeToReadableBinding(bindings, value)
+  $: tempValue = readableValue
+  $: isJS = isJSBinding(value)
+
+  const saveBinding = () => {
+    onChange(tempValue)
+    onBlur()
+    bindingDrawer.hide()
+  }
+
+  setContext("binding-drawer-actions", {
+    save: saveBinding,
+  })
+
+  const onChange = value => {
+    if (type === "link" && value && hasValidLinks(value)) {
+      currentVal = value.split(",")
+    } else if (type === "array" && value && hasValidOptions(value)) {
+      currentVal = value.split(",")
+    } else {
+      currentVal = readableToRuntimeBinding(bindings, value)
+    }
+    dispatch("change", currentVal)
+  }
+
+  const onBlur = () => {
+    dispatch("blur", currentVal)
+  }
+
+  const isValidDate = value => {
+    return !value || !isNaN(new Date(value).valueOf())
+  }
+
+  const hasValidLinks = value => {
+    let links = []
+    if (Array.isArray(value)) {
+      links = value
+    } else if (value && typeof value === "string") {
+      links = value.split(",")
+    } else {
+      return !value
+    }
+
+    return links.every(link => link.startsWith("ro_"))
+  }
+
+  const hasValidOptions = value => {
+    let links = []
+    if (Array.isArray(value)) {
+      links = value
+    } else if (value && typeof value === "string") {
+      links = value.split(",")
+    } else {
+      return !value
+    }
+    return links.every(link => schema?.constraints?.inclusion?.includes(link))
+  }
+
+  const isValidBoolean = value => {
+    return value === "false" || value === "true" || value == ""
+  }
+
+  const validationMap = {
+    date: isValidDate,
+    datetime: isValidDate,
+    link: hasValidLinks,
+    array: hasValidOptions,
+    longform: value => !isJSBinding(value),
+    json: value => !isJSBinding(value),
+    boolean: isValidBoolean,
+  }
+
+  const isValid = value => {
+    const validate = validationMap[type]
+    return validate ? validate(value) : true
+  }
+
+  const getIconClass = (value, type) => {
+    if (type === "longform" && !isJSBinding(value)) {
+      return "text-area-slot-icon"
+    }
+    if (type === "json" && !isJSBinding(value)) {
+      return "json-slot-icon"
+    }
+    if (type !== "string" && type !== "number") {
+      return "slot-icon"
+    }
+    return ""
+  }
+</script>
+
+<div class="control" class:disabled>
+  {#if !isValid(value)}
+    <Input
+      {label}
+      {disabled}
+      readonly={isJS}
+      value={isJS ? "(JavaScript function)" : readableValue}
+      on:change={event => onChange(event.detail)}
+      on:blur={onBlur}
+      {placeholder}
+      {updateOnChange}
+    />
+    <div
+      class="icon"
+      on:click={() => {
+        if (!isJS) {
+          dispatch("change", "")
+        }
+      }}
+    >
+      <Icon disabled={isJS} size="S" name="Close" />
+    </div>
+  {:else}
+    <slot
+      {label}
+      {disabled}
+      readonly={isJS}
+      value={isJS ? "(JavaScript function)" : readableValue}
+      {placeholder}
+      {updateOnChange}
+    />
+  {/if}
+  {#if !disabled && type !== "formula"}
+    <div
+      class={`icon ${getIconClass(value, type)}`}
+      on:click={() => {
+        bindingDrawer.show()
+      }}
+    >
+      <Icon size="S" name="FlashOn" />
+    </div>
+  {/if}
+</div>
+<Drawer
+  on:drawerHide
+  on:drawerShow
+  {fillWidth}
+  bind:this={bindingDrawer}
+  {title}
+  left={drawerLeft}
+  headless
+>
+  <svelte:fragment slot="description">
+    Add the objects on the left to enrich your text.
+  </svelte:fragment>
+  <Button cta slot="buttons" disabled={!valid} on:click={saveBinding}>
+    Save
+  </Button>
+  <svelte:component
+    this={panel}
+    slot="body"
+    bind:valid
+    value={readableValue}
+    on:change={event => (tempValue = event.detail)}
+    {bindings}
+    {allowJS}
+    {allowHelpers}
+  />
+</Drawer>
+
+<style>
+  .control {
+    flex: 1;
+    position: relative;
+  }
+
+  .slot-icon {
+    right: 31px !important;
+    border-right: 1px solid var(--spectrum-alias-border-color);
+    border-top-right-radius: 0px !important;
+    border-bottom-right-radius: 0px !important;
+  }
+
+  .text-area-slot-icon {
+    border-bottom: 1px solid var(--spectrum-alias-border-color);
+    border-bottom-right-radius: 0px !important;
+    top: 26px !important;
+  }
+  .json-slot-icon {
+    border-bottom: 1px solid var(--spectrum-alias-border-color);
+    border-bottom-right-radius: 0px !important;
+    top: 23px !important;
+    right: 0px !important;
+  }
+
+  .icon {
+    right: 1px;
+    bottom: 1px;
+    position: absolute;
+    justify-content: center;
+    align-items: center;
+    display: flex;
+    flex-direction: row;
+    box-sizing: border-box;
+    border-left: 1px solid var(--spectrum-alias-border-color);
+    border-top-right-radius: var(--spectrum-alias-border-radius-regular);
+    border-bottom-right-radius: var(--spectrum-alias-border-radius-regular);
+    width: 31px;
+    color: var(--spectrum-alias-text-color);
+    background-color: var(--spectrum-global-color-gray-75);
+    transition: background-color
+        var(--spectrum-global-animation-duration-100, 130ms),
+      box-shadow var(--spectrum-global-animation-duration-100, 130ms),
+      border-color var(--spectrum-global-animation-duration-100, 130ms);
+    height: calc(var(--spectrum-alias-item-height-m) - 2px);
+  }
+
+  .icon:hover {
+    cursor: pointer;
+    color: var(--spectrum-alias-text-color-hover);
+    background-color: var(--spectrum-global-color-gray-50);
+    border-color: var(--spectrum-alias-border-color-hover);
+  }
+
+  .control:not(.disabled) :global(.spectrum-Textfield-input) {
+    padding-right: 40px;
+  }
+</style>

packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte

@@ -20,7 +20,9 @@
 
   const getSortableFields = schema => {
     return Object.entries(schema || {})
-      .filter(entry => !UNSORTABLE_TYPES.includes(entry[1].type))
+      .filter(
+        entry => !UNSORTABLE_TYPES.includes(entry[1].type) && entry[1].sortable
+      )
       .map(entry => entry[0])
   }
 

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/EditDatasourceConfigButton.svelte

@@ -62,7 +62,14 @@
       </div>
     {/if}
     <div class="truncate">
-      <Body>{getSubtitle(datasource)}</Body>
+      <Body>
+        {@const subtitle = getSubtitle(datasource)}
+        {#if subtitle}
+          {subtitle}
+        {:else}
+          {Object.values(datasource.config).join(" / ")}
+        {/if}
+      </Body>
     </div>
   </div>
   <div class="right">

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Relationships.svelte

@@ -21,15 +21,22 @@
   function getRelationships(tables) {
     const relatedColumns = {}
 
-    tables.forEach(({ name: tableName, schema }) => {
+    tables.forEach(({ name: tableName, schema, _id: tableId }) => {
       Object.values(schema).forEach(column => {
         if (column.type !== "link") return
 
-        relatedColumns[column._id] ??= {}
+        const columnId =
-        relatedColumns[column._id].through =
+          column.through ||
-          relatedColumns[column._id].through || column.through
+          column._id ||
+          (column.main
+            ? `${tableId}_${column.fieldName}__${column.tableId}_${column.foreignKey}`
+            : `${column.tableId}_${column.foreignKey}__${tableId}_${column.fieldName}`)
 
-        relatedColumns[column._id][column.main ? "from" : "to"] = {
+        relatedColumns[columnId] ??= {}
+        relatedColumns[columnId].through =
+          relatedColumns[columnId].through || column.through
+
+        relatedColumns[columnId][column.main ? "from" : "to"] = {
           ...column,
           tableName,
         }

packages/builder/src/stores/backend/datasources.js

@@ -136,6 +136,7 @@ export function createDatasourcesStore() {
       config,
       name: `${integration.friendlyName}${nameModifier}`,
       plus: integration.plus && integration.name !== IntegrationTypes.REST,
+      isSQL: integration.isSQL,
     }
 
     if (await checkDatasourceValidity(integration, datasource)) {

packages/client/src/components/app/Layout.svelte

@@ -3,6 +3,7 @@
   import { writable } from "svelte/store"
   import { Heading, Icon, clickOutside } from "@budibase/bbui"
   import { FieldTypes } from "constants"
+  import { Constants } from "@budibase/frontend-core"
   import active from "svelte-spa-router/active"
 
   const sdk = getContext("sdk")
@@ -103,7 +104,8 @@
     let validLinks = (allLinks || []).filter(link => link.text && link.url)
     // Filter to only links allowed by the current role
     return validLinks.filter(link => {
-      return userRoleHierarchy?.find(roleId => roleId === link.roleId)
+      const role = link.roleId || Constants.Roles.BASIC
+      return userRoleHierarchy?.find(roleId => roleId === role)
     })
   }
 

packages/client/src/components/app/forms/RelationshipField.svelte

@@ -60,6 +60,12 @@
       // even if they are not in the inital fetch results
       initialValuesProcessed = true
       optionsObj = (fieldState?.value || []).reduce((accumulator, value) => {
+        // fieldState has to be an array of strings to be valid for an update
+        // therefore we cannot guarantee value will be an object
+        // https://linear.app/budibase/issue/BUDI-7577/refactor-the-relationshipfield-component-to-have-better-support-for
+        if (!value._id) {
+          return accumulator
+        }
         accumulator[value._id] = {
           _id: value._id,
           [primaryDisplay]: value.primaryDisplay,

packages/client/src/components/devtools/DevToolsHeader.svelte

@@ -25,7 +25,6 @@
         value: roleId,
       })
     }
-    devToolsStore.actions.changeRole(SELF_ROLE)
     return list
   }
 

packages/client/src/stores/devTools.js

@@ -2,6 +2,7 @@ import { createLocalStorageStore } from "@budibase/frontend-core"
 import { initialise } from "./initialise"
 import { authStore } from "./auth"
 import { API } from "../api"
+import { get } from "svelte/store"
 
 const initialState = {
   visible: false,
@@ -27,9 +28,15 @@ const createDevToolStore = () => {
   }
 
   const changeRole = async role => {
+    if (role === "self") {
+      role = null
+    }
+    if (role === get(store).role) {
+      return
+    }
     store.update(state => ({
       ...state,
-      role: role === "self" ? null : role,
+      role,
     }))
     API.invalidateCache()
     await authStore.actions.fetchUser()

packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte

@@ -3,6 +3,8 @@
   import RelationshipCell from "./RelationshipCell.svelte"
   import { FieldSubtype } from "@budibase/types"
 
+  export let api
+
   const { API } = getContext("grid")
   const { subtype } = $$props.schema
 
@@ -17,8 +19,11 @@
       throw `Search for '${subtype}' not implemented`
     }
 
+    // As we are overriding the search function from RelationshipCell, we want to map one shape to the expected one for the specific API
+    const email = Object.values(searchParams.query.string)[0]
+
     const results = await API.searchUsers({
-      ...searchParams,
+      email,
     })
 
     // Mapping to the expected data within RelationshipCell
@@ -31,6 +36,7 @@
 </script>
 
 <RelationshipCell
+  bind:api
   {...$$props}
   {schema}
   {searchFunction}

packages/sdk/scripts/generate-sdk.sh

@@ -19,7 +19,7 @@ docker run --rm \
   -v ${PWD}/generated:/generated \
   -v ${PWD}/config.json:/config.json \
   -u $(id -u):$(id -g) \
-  swaggerapi/swagger-codegen-cli-v3 generate \
+  swaggerapi/swagger-codegen-cli-v3:3.0.46 generate \
   -i /openapi.yml \
   -l javascript \
   -o /generated \
@@ -34,4 +34,4 @@ if [[ -f "openapi.yaml" ]]; then
 fi
 if [[ -d "generated" ]]; then
     rm -r generated
-fi
+fi

packages/server/src/api/controllers/query/index.ts

@@ -9,7 +9,7 @@ import { quotas } from "@budibase/pro"
 import { events, context, utils, constants } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import { QueryEvent } from "../../../threads/definitions"
-import { Query } from "@budibase/types"
+import { ConfigType, Query, UserCtx } from "@budibase/types"
 import { ValidQueryNameRegex } from "@budibase/shared-core"
 
 const Runner = new Thread(ThreadType.QUERY, {
@@ -28,11 +28,11 @@ function enrichQueries(input: any) {
   return wasArray ? queries : queries[0]
 }
 
-export async function fetch(ctx: any) {
+export async function fetch(ctx: UserCtx) {
   ctx.body = await sdk.queries.fetch()
 }
 
-const _import = async (ctx: any) => {
+const _import = async (ctx: UserCtx) => {
   const body = ctx.request.body
   const data = body.data
 
@@ -73,7 +73,7 @@ const _import = async (ctx: any) => {
 }
 export { _import as import }
 
-export async function save(ctx: any) {
+export async function save(ctx: UserCtx) {
   const db = context.getAppDB()
   const query = ctx.request.body
 
@@ -100,19 +100,19 @@ export async function save(ctx: any) {
   ctx.message = `Query ${query.name} saved successfully.`
 }
 
-export async function find(ctx: any) {
+export async function find(ctx: UserCtx) {
   const queryId = ctx.params.queryId
   ctx.body = await sdk.queries.find(queryId)
 }
 
 //Required to discern between OIDC OAuth config entries
-function getOAuthConfigCookieId(ctx: any) {
+function getOAuthConfigCookieId(ctx: UserCtx) {
-  if (ctx.user.providerType === constants.Config.OIDC) {
+  if (ctx.user.providerType === ConfigType.OIDC) {
     return utils.getCookie(ctx, constants.Cookie.OIDC_CONFIG)
   }
 }
 
-function getAuthConfig(ctx: any) {
+function getAuthConfig(ctx: UserCtx) {
   const authCookie = utils.getCookie(ctx, constants.Cookie.Auth)
   let authConfigCtx: any = {}
   authConfigCtx["configId"] = getOAuthConfigCookieId(ctx)
@@ -120,7 +120,7 @@ function getAuthConfig(ctx: any) {
   return authConfigCtx
 }
 
-export async function preview(ctx: any) {
+export async function preview(ctx: UserCtx) {
   const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
     ctx.request.body.datasourceId
   )
@@ -129,6 +129,19 @@ export async function preview(ctx: any) {
   // this stops dynamic variables from calling the same query
   const { fields, parameters, queryVerb, transformer, queryId, schema } = query
 
+  let existingSchema = schema
+  if (queryId && !existingSchema) {
+    try {
+      const db = context.getAppDB()
+      const existing = (await db.get(queryId)) as Query
+      existingSchema = existing.schema
+    } catch (err: any) {
+      if (err.status !== 404) {
+        ctx.throw(500, "Unable to retrieve existing query")
+      }
+    }
+  }
+
   const authConfigCtx: any = getAuthConfig(ctx)
 
   try {
@@ -180,6 +193,14 @@ export async function preview(ctx: any) {
         schemaFields[key] = fieldType
       }
     }
+    // if existing schema, update to include any previous schema keys
+    if (existingSchema) {
+      for (let key of Object.keys(schemaFields)) {
+        if (existingSchema[key]?.type) {
+          schemaFields[key] = existingSchema[key].type
+        }
+      }
+    }
     // remove configuration before sending event
     delete datasource.config
     await events.query.previewed(datasource, query)
@@ -189,13 +210,13 @@ export async function preview(ctx: any) {
       info,
       extra,
     }
-  } catch (err) {
+  } catch (err: any) {
     ctx.throw(400, err)
   }
 }
 
 async function execute(
-  ctx: any,
+  ctx: UserCtx,
   opts: any = { rowsOnly: false, isAutomation: false }
 ) {
   const db = context.getAppDB()
@@ -255,17 +276,17 @@ async function execute(
     } else {
       ctx.body = { data: rows, pagination, ...extra, ...info }
     }
-  } catch (err) {
+  } catch (err: any) {
     ctx.throw(400, err)
   }
 }
 
-export async function executeV1(ctx: any) {
+export async function executeV1(ctx: UserCtx) {
   return execute(ctx, { rowsOnly: true, isAutomation: false })
 }
 
 export async function executeV2(
-  ctx: any,
+  ctx: UserCtx,
   { isAutomation }: { isAutomation?: boolean } = {}
 ) {
   return execute(ctx, { rowsOnly: false, isAutomation })
@@ -292,7 +313,7 @@ const removeDynamicVariables = async (queryId: any) => {
   }
 }
 
-export async function destroy(ctx: any) {
+export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
   const queryId = ctx.params.queryId
   await removeDynamicVariables(queryId)

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -340,10 +340,16 @@ export class ExternalRequest<T extends Operation> {
       // one to many
       if (isOneSide(field)) {
         let id = row[key][0]
-        if (typeof row[key] === "string") {
+        if (id) {
-          id = decodeURIComponent(row[key]).match(/\[(.*?)\]/)?.[1]
+          if (typeof row[key] === "string") {
+            id = decodeURIComponent(row[key]).match(/\[(.*?)\]/)?.[1]
+          }
+          newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(id)[0]
+        } else {
+          // Removing from both new and row, as we don't know if it has already been processed
+          row[field.foreignKey || linkTablePrimary] = null
+          newRow[field.foreignKey || linkTablePrimary] = null
         }
-        newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(id)[0]
       }
       // many to many
       else if (field.through) {
@@ -830,10 +836,7 @@ export class ExternalRequest<T extends Operation> {
     // can't really use response right now
     const response = await getDatasourceAndQuery(json)
     // handle many to many relationships now if we know the ID (could be auto increment)
-    if (
+    if (operation !== Operation.READ) {
-      operation !== Operation.READ &&
-      processed.manyRelationships?.length > 0
-    ) {
       await this.handleManyRelationships(
         table._id || "",
         response[0],

packages/server/src/api/controllers/row/external.ts

@@ -108,13 +108,11 @@ export async function save(ctx: UserCtx) {
     row,
   })
 
-  const responseRow = response as { row: Row }
-
   if (!isEqual(table, updatedTable)) {
     await sdk.tables.saveTable(updatedTable)
   }
 
-  const rowId = responseRow.row._id
+  const rowId = response.row._id
   if (rowId) {
     const row = await sdk.rows.external.getRow(tableId, rowId, {
       relationships: true,

packages/server/src/api/controllers/table/index.ts

@@ -14,7 +14,6 @@ import {
   Table,
   TableResponse,
   UserCtx,
-  Datasource,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { jsonFromCsvString } from "../../../utilities/csv"

packages/server/src/api/routes/tests/row.spec.ts

@@ -18,7 +18,6 @@ import {
   SortType,
   StaticQuotaName,
   Table,
-  User,
 } from "@budibase/types"
 import {
   expectAnyExternalColsAttributes,
@@ -1515,9 +1514,82 @@ describe.each([
     })
   })
 
-  describe("bb reference fields", () => {
+  let o2mTable: Table
+  let m2mTable: Table
+  beforeAll(async () => {
+    o2mTable = await config.createTable(
+      { ...generateTableConfig(), name: "o2m" },
+      {
+        skipReassigning: true,
+      }
+    )
+    m2mTable = await config.createTable(
+      { ...generateTableConfig(), name: "m2m" },
+      {
+        skipReassigning: true,
+      }
+    )
+  })
+
+  describe.each([
+    [
+      "relationship fields",
+      () => ({
+        user: {
+          name: "user",
+          relationshipType: RelationshipType.ONE_TO_MANY,
+          type: FieldType.LINK,
+          tableId: o2mTable._id!,
+          fieldName: "fk_o2m",
+        },
+        users: {
+          name: "users",
+          relationshipType: RelationshipType.MANY_TO_MANY,
+          type: FieldType.LINK,
+          tableId: m2mTable._id!,
+          fieldName: "fk_m2m",
+        },
+      }),
+      (tableId: string) =>
+        config.api.row.save(tableId, {
+          name: generator.word(),
+          description: generator.paragraph(),
+          tableId,
+        }),
+      (row: Row) => ({
+        _id: row._id,
+        primaryDisplay: row.name,
+      }),
+    ],
+    [
+      "bb reference fields",
+      () => ({
+        user: {
+          name: "user",
+          relationshipType: RelationshipType.ONE_TO_MANY,
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+        },
+        users: {
+          name: "users",
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+          relationshipType: RelationshipType.MANY_TO_MANY,
+        },
+      }),
+      () => config.createUser(),
+      (row: Row) => ({
+        _id: row._id,
+        email: row.email,
+        firstName: row.firstName,
+        lastName: row.lastName,
+        primaryDisplay: row.email,
+      }),
+    ],
+  ])("links - %s", (__, relSchema, dataGenerator, resultMapper) => {
     let tableId: string
-    let users: User[]
+    let o2mData: Row[]
+    let m2mData: Row[]
 
     beforeAll(async () => {
       const tableConfig = generateTableConfig()
@@ -1532,31 +1604,27 @@ describe.each([
         ...tableConfig,
         schema: {
           ...tableConfig.schema,
-          user: {
+          ...relSchema(),
-            name: "user",
-            type: FieldType.BB_REFERENCE,
-            subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
-            relationshipType: RelationshipType.ONE_TO_MANY,
-          },
-          users: {
-            name: "users",
-            type: FieldType.BB_REFERENCE,
-            subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
-            relationshipType: RelationshipType.MANY_TO_MANY,
-          },
         },
       })
       tableId = table._id!
 
-      users = [
+      o2mData = [
-        await config.createUser(),
+        await dataGenerator(o2mTable._id!),
-        await config.createUser(),
+        await dataGenerator(o2mTable._id!),
-        await config.createUser(),
+        await dataGenerator(o2mTable._id!),
-        await config.createUser(),
+        await dataGenerator(o2mTable._id!),
+      ]
+
+      m2mData = [
+        await dataGenerator(m2mTable._id!),
+        await dataGenerator(m2mTable._id!),
+        await dataGenerator(m2mTable._id!),
+        await dataGenerator(m2mTable._id!),
       ]
     })
 
-    it("can save a row when BB reference fields are empty", async () => {
+    it("can save a row when relationship fields are empty", async () => {
       const rowData = {
         ...basicRow(tableId),
         name: generator.name(),
@@ -1575,13 +1643,13 @@ describe.each([
       })
     })
 
-    it("can save a row with a single BB reference field", async () => {
+    it("can save a row with a single relationship field", async () => {
-      const user = _.sample(users)!
+      const user = _.sample(o2mData)!
       const rowData = {
         ...basicRow(tableId),
         name: generator.name(),
         description: generator.name(),
-        user: user,
+        user: [user],
       }
       const row = await config.api.row.save(tableId, rowData)
 
@@ -1589,24 +1657,17 @@ describe.each([
         name: rowData.name,
         description: rowData.description,
         tableId,
-        user: [
+        user: [user].map(u => resultMapper(u)),
-          {
-            _id: user._id,
-            email: user.email,
-            firstName: user.firstName,
-            lastName: user.lastName,
-            primaryDisplay: user.email,
-          },
-        ],
         _id: expect.any(String),
         _rev: expect.any(String),
         id: isInternal ? undefined : expect.any(Number),
         type: isInternal ? "row" : undefined,
+        [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id,
       })
     })
 
-    it("can save a row with a multiple BB reference field", async () => {
+    it("can save a row with a multiple relationship field", async () => {
-      const selectedUsers = _.sampleSize(users, 2)
+      const selectedUsers = _.sampleSize(m2mData, 2)
       const rowData = {
         ...basicRow(tableId),
         name: generator.name(),
@@ -1619,13 +1680,7 @@ describe.each([
         name: rowData.name,
         description: rowData.description,
         tableId,
-        users: selectedUsers.map(u => ({
+        users: expect.arrayContaining(selectedUsers.map(u => resultMapper(u))),
-          _id: u._id,
-          email: u.email,
-          firstName: u.firstName,
-          lastName: u.lastName,
-          primaryDisplay: u.email,
-        })),
         _id: expect.any(String),
         _rev: expect.any(String),
         id: isInternal ? undefined : expect.any(Number),
@@ -1633,7 +1688,7 @@ describe.each([
       })
     })
 
-    it("can retrieve rows with no populated BB references", async () => {
+    it("can retrieve rows with no populated relationships", async () => {
       const rowData = {
         ...basicRow(tableId),
         name: generator.name(),
@@ -1655,14 +1710,15 @@ describe.each([
       })
     })
 
-    it("can retrieve rows with populated BB references", async () => {
+    it("can retrieve rows with populated relationships", async () => {
-      const [user1, user2] = _.sampleSize(users, 2)
+      const user1 = _.sample(o2mData)!
+      const [user2, user3] = _.sampleSize(m2mData, 2)
 
       const rowData = {
         ...basicRow(tableId),
         name: generator.name(),
         description: generator.name(),
-        users: [user1, user2],
+        users: [user2, user3],
         user: [user1],
       }
       const row = await config.api.row.save(tableId, rowData)
@@ -1672,72 +1728,51 @@ describe.each([
         name: rowData.name,
         description: rowData.description,
         tableId,
-        user: [user1].map(u => ({
+        user: expect.arrayContaining([user1].map(u => resultMapper(u))),
-          _id: u._id,
+        users: expect.arrayContaining([user2, user3].map(u => resultMapper(u))),
-          email: u.email,
-          firstName: u.firstName,
-          lastName: u.lastName,
-          primaryDisplay: u.email,
-        })),
-        users: [user1, user2].map(u => ({
-          _id: u._id,
-          email: u.email,
-          firstName: u.firstName,
-          lastName: u.lastName,
-          primaryDisplay: u.email,
-        })),
         _id: row._id,
         _rev: expect.any(String),
         id: isInternal ? undefined : expect.any(Number),
+        [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user1.id,
         ...defaultRowFields,
       })
     })
 
     it("can update an existing populated row", async () => {
-      const [user1, user2, user3] = _.sampleSize(users, 3)
+      const user = _.sample(o2mData)!
+      const [users1, users2, users3] = _.sampleSize(m2mData, 3)
 
       const rowData = {
         ...basicRow(tableId),
         name: generator.name(),
         description: generator.name(),
-        users: [user1, user2],
+        users: [users1, users2],
       }
       const row = await config.api.row.save(tableId, rowData)
 
       const updatedRow = await config.api.row.save(tableId, {
         ...row,
-        user: [user3],
+        user: [user],
-        users: [user3, user2],
+        users: [users3, users1],
       })
       expect(updatedRow).toEqual({
         name: rowData.name,
         description: rowData.description,
         tableId,
-        user: [
+        user: expect.arrayContaining([user].map(u => resultMapper(u))),
-          {
+        users: expect.arrayContaining(
-            _id: user3._id,
+          [users3, users1].map(u => resultMapper(u))
-            email: user3.email,
+        ),
-            firstName: user3.firstName,
-            lastName: user3.lastName,
-            primaryDisplay: user3.email,
-          },
-        ],
-        users: [user3, user2].map(u => ({
-          _id: u._id,
-          email: u.email,
-          firstName: u.firstName,
-          lastName: u.lastName,
-          primaryDisplay: u.email,
-        })),
         _id: row._id,
         _rev: expect.any(String),
         id: isInternal ? undefined : expect.any(Number),
         type: isInternal ? "row" : undefined,
+        [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id,
       })
     })
 
-    it("can wipe an existing populated BB references in row", async () => {
+    it("can wipe an existing populated relationships in row", async () => {
-      const [user1, user2] = _.sampleSize(users, 2)
+      const [user1, user2] = _.sampleSize(m2mData, 2)
 
       const rowData = {
         ...basicRow(tableId),
@@ -1756,8 +1791,6 @@ describe.each([
         name: rowData.name,
         description: rowData.description,
         tableId,
-        user: isInternal ? null : undefined,
-        users: isInternal ? null : undefined,
         _id: row._id,
         _rev: expect.any(String),
         id: isInternal ? undefined : expect.any(Number),
@@ -1765,34 +1798,35 @@ describe.each([
       })
     })
 
-    it("fetch all will populate the BB references", async () => {
+    it("fetch all will populate the relationships", async () => {
-      const [user1, user2, user3] = _.sampleSize(users, 3)
+      const [user1] = _.sampleSize(o2mData, 1)
+      const [users1, users2, users3] = _.sampleSize(m2mData, 3)
 
       const rows: {
         name: string
         description: string
-        user?: User[]
+        user?: Row[]
-        users?: User[]
+        users?: Row[]
         tableId: string
       }[] = [
         {
           ...basicRow(tableId),
           name: generator.name(),
           description: generator.name(),
-          users: [user1, user2],
+          users: [users1, users2],
         },
         {
           ...basicRow(tableId),
           name: generator.name(),
           description: generator.name(),
           user: [user1],
-          users: [user1, user3],
+          users: [users1, users3],
         },
         {
           ...basicRow(tableId),
           name: generator.name(),
           description: generator.name(),
-          users: [user3],
+          users: [users3],
         },
       ]
 
@@ -1808,57 +1842,50 @@ describe.each([
             name: r.name,
             description: r.description,
             tableId,
-            user: r.user?.map(u => ({
+            user: r.user?.map(u => resultMapper(u)),
-              _id: u._id,
+            users: r.users?.length
-              email: u.email,
+              ? expect.arrayContaining(r.users?.map(u => resultMapper(u)))
-              firstName: u.firstName,
+              : undefined,
-              lastName: u.lastName,
-              primaryDisplay: u.email,
-            })),
-            users: r.users?.map(u => ({
-              _id: u._id,
-              email: u.email,
-              firstName: u.firstName,
-              lastName: u.lastName,
-              primaryDisplay: u.email,
-            })),
             _id: expect.any(String),
             _rev: expect.any(String),
             id: isInternal ? undefined : expect.any(Number),
+            [`fk_${o2mTable.name}_fk_o2m`]:
+              isInternal || !r.user?.length ? undefined : r.user[0].id,
             ...defaultRowFields,
           }))
         )
       )
     })
 
-    it("search all will populate the BB references", async () => {
+    it("search all will populate the relationships", async () => {
-      const [user1, user2, user3] = _.sampleSize(users, 3)
+      const [user1] = _.sampleSize(o2mData, 1)
+      const [users1, users2, users3] = _.sampleSize(m2mData, 3)
 
       const rows: {
         name: string
         description: string
-        user?: User[]
+        user?: Row[]
-        users?: User[]
+        users?: Row[]
         tableId: string
       }[] = [
         {
           ...basicRow(tableId),
           name: generator.name(),
           description: generator.name(),
-          users: [user1, user2],
+          users: [users1, users2],
         },
         {
           ...basicRow(tableId),
           name: generator.name(),
           description: generator.name(),
           user: [user1],
-          users: [user1, user3],
+          users: [users1, users3],
         },
         {
           ...basicRow(tableId),
           name: generator.name(),
           description: generator.name(),
-          users: [user3],
+          users: [users3],
         },
       ]
 
@@ -1874,23 +1901,15 @@ describe.each([
             name: r.name,
             description: r.description,
             tableId,
-            user: r.user?.map(u => ({
+            user: r.user?.map(u => resultMapper(u)),
-              _id: u._id,
+            users: r.users?.length
-              email: u.email,
+              ? expect.arrayContaining(r.users?.map(u => resultMapper(u)))
-              firstName: u.firstName,
+              : undefined,
-              lastName: u.lastName,
-              primaryDisplay: u.email,
-            })),
-            users: r.users?.map(u => ({
-              _id: u._id,
-              email: u.email,
-              firstName: u.firstName,
-              lastName: u.lastName,
-              primaryDisplay: u.email,
-            })),
             _id: expect.any(String),
             _rev: expect.any(String),
             id: isInternal ? undefined : expect.any(Number),
+            [`fk_${o2mTable.name}_fk_o2m`]:
+              isInternal || !r.user?.length ? undefined : r.user[0].id,
             ...defaultRowFields,
           }))
         ),

packages/server/src/automations/steps/make.ts

@@ -78,8 +78,7 @@ export const definition: AutomationStepSchema = {
 }
 
 export async function run({ inputs }: AutomationStepInput) {
-  //TODO - Remove deprecated values 1,2,3,4,5 after November 2023
+  const { url, body } = inputs
-  const { url, value1, value2, value3, value4, value5, body } = inputs
 
   let payload = {}
   try {
@@ -104,11 +103,6 @@ export async function run({ inputs }: AutomationStepInput) {
     response = await fetch(url, {
       method: "post",
       body: JSON.stringify({
-        value1,
-        value2,
-        value3,
-        value4,
-        value5,
         ...payload,
       }),
       headers: {

packages/server/src/automations/steps/zapier.ts

@@ -71,8 +71,7 @@ export const definition: AutomationStepSchema = {
 }
 
 export async function run({ inputs }: AutomationStepInput) {
-  //TODO - Remove deprecated values 1,2,3,4,5 after November 2023
+  const { url, body } = inputs
-  const { url, value1, value2, value3, value4, value5, body } = inputs
 
   let payload = {}
   try {
@@ -100,11 +99,6 @@ export async function run({ inputs }: AutomationStepInput) {
       method: "post",
       body: JSON.stringify({
         platform: "budibase",
-        value1,
-        value2,
-        value3,
-        value4,
-        value5,
         ...payload,
       }),
       headers: {

packages/server/src/threads/definitions.ts

@@ -11,12 +11,7 @@ export interface QueryEvent {
   queryId: string
   environmentVariables?: Record<string, string>
   ctx?: any
-  schema?: {
+  schema?: Record<string, { name?: string; type: string }>
-    [key: string]: {
-      name: string
-      type: string
-    }
-  }
 }
 
 export interface QueryVariable {

packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts

@@ -48,7 +48,7 @@ export async function processOutputBBReferences(
 ) {
   if (typeof value !== "string") {
     // Already processed or nothing to process
-    return value
+    return value || undefined
   }
 
   const ids = value.split(",").filter(id => !!id)

packages/server/src/utilities/rowProcessor/index.ts

@@ -11,6 +11,7 @@ import {
   processInputBBReferences,
   processOutputBBReferences,
 } from "./bbReferenceProcessor"
+import { isExternalTable } from "../../integrations/utils"
 export * from "./utils"
 
 type AutoColumnProcessingOpts = {
@@ -234,9 +235,6 @@ export async function outputProcessing<T extends Row[] | Row>(
       }
     } else if (column.type == FieldTypes.BB_REFERENCE) {
       for (let row of enriched) {
-        if (row[property] == null) {
-          continue
-        }
         row[property] = await processOutputBBReferences(
           row[property],
           column.subtype as FieldSubtype
@@ -250,6 +248,16 @@ export async function outputProcessing<T extends Row[] | Row>(
       enriched
     )) as Row[]
   }
+  // remove null properties to match internal API
+  if (isExternalTable(table._id!)) {
+    for (let row of enriched) {
+      for (let key of Object.keys(row)) {
+        if (row[key] === null) {
+          delete row[key]
+        }
+      }
+    }
+  }
   return (wasArray ? enriched : enriched[0]) as T
 }
 

packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts

@@ -66,7 +66,7 @@ describe("rowProcessor - outputProcessing", () => {
     )
   })
 
-  it("does not fetch bb references when fields are empty", async () => {
+  it("process output even when the field is not empty", async () => {
     const table: Table = {
       _id: generator.guid(),
       name: "TestTable",
@@ -100,7 +100,7 @@ describe("rowProcessor - outputProcessing", () => {
 
     expect(result).toEqual({ name: "Jack" })
 
-    expect(bbReferenceProcessor.processOutputBBReferences).not.toBeCalled()
+    expect(bbReferenceProcessor.processOutputBBReferences).toBeCalledTimes(1)
   })
 
   it("does not fetch bb references when not in the schema", async () => {

packages/shared-core/src/helpers/integrations.ts

@@ -14,5 +14,5 @@ export function isSQL(datasource: Datasource): boolean {
     SourceName.MYSQL,
     SourceName.ORACLE,
   ]
-  return SQL.indexOf(datasource.source) !== -1
+  return SQL.indexOf(datasource.source) !== -1 || datasource.isSQL === true
 }

packages/types/src/documents/app/datasource.ts

@@ -9,6 +9,7 @@ export interface Datasource extends Document {
   // the config is defined by the schema
   config?: Record<string, any>
   plus?: boolean
+  isSQL?: boolean
   entities?: {
     [key: string]: Table
   }

packages/types/src/documents/app/query.ts

@@ -6,7 +6,7 @@ export interface Query extends Document {
   parameters: QueryParameter[]
   fields: RestQueryFields | any
   transformer: string | null
-  schema: any
+  schema: Record<string, { name?: string; type: string }>
   readable: boolean
   queryVerb: string
 }

packages/types/src/sdk/datasources.ts

@@ -140,6 +140,7 @@ export interface DatasourceConfig {
 export interface Integration {
   docs: string
   plus?: boolean
+  isSQL?: boolean
   auth?: { type: string }
   features?: Partial<Record<DatasourceFeature, boolean>>
   relationships?: boolean

packages/types/src/sdk/koa.ts

@@ -1,5 +1,5 @@
 import { Context, Request } from "koa"
-import { User, Role, UserRoles, Account } from "../documents"
+import { User, Role, UserRoles, Account, ConfigType } from "../documents"
 import { FeatureFlag, License } from "../sdk"
 import { Files } from "formidable"
 
@@ -13,6 +13,7 @@ export interface ContextUser extends Omit<User, "roles"> {
   csrfToken?: string
   featureFlags?: FeatureFlag[]
   accountPortalAccess?: boolean
+  providerType?: ConfigType
   account?: Account
 }
 

packages/worker/src/api/routes/global/tests/scim.spec.ts

@@ -11,7 +11,7 @@ import { TestConfiguration } from "../../../../tests"
 import { events } from "@budibase/backend-core"
 
 // this test can 409 - retries reduce issues with this
-jest.retryTimes(2)
+jest.retryTimes(2, { logErrorsBeforeRetry: true })
 jest.setTimeout(30000)
 
 mocks.licenses.useScimIntegration()