Merge branch 'master' of github.com:budibase/budibase into remove-worker-user-mock

packages/client/src/components/app/forms/RelationshipField.svelte

@@ -1,18 +1,15 @@
 <script lang="ts">
   import { CoreSelect, CoreMultiselect } from "@budibase/bbui"
-  import { FieldType } from "@budibase/types"
+  import { FieldType, InternalTable } from "@budibase/types"
   import { fetchData, Utils } from "@budibase/frontend-core"
   import { getContext } from "svelte"
   import Field from "./Field.svelte"
   import type {
     SearchFilter,
     RelationshipFieldMetadata,
-    Table,
     Row,
   } from "@budibase/types"
 
-  const { API } = getContext("sdk")
-
   export let field: string | undefined = undefined
   export let label: string | undefined = undefined
   export let placeholder: any = undefined
@@ -20,10 +17,10 @@
   export let readonly: boolean = false
   export let validation: any
   export let autocomplete: boolean = true
-  export let defaultValue: string | undefined = undefined
+  export let defaultValue: string | string[] | undefined = undefined
   export let onChange: any
   export let filter: SearchFilter[]
-  export let datasourceType: "table" | "user" | "groupUser" = "table"
+  export let datasourceType: "table" | "user" = "table"
   export let primaryDisplay: string | undefined = undefined
   export let span: number | undefined = undefined
   export let helpText: string | undefined = undefined
@@ -32,191 +29,305 @@
     | FieldType.BB_REFERENCE
     | FieldType.BB_REFERENCE_SINGLE = FieldType.LINK
 
-  type RelationshipValue = { _id: string; [key: string]: any }
+  type BasicRelatedRow = { _id: string; primaryDisplay: string }
-  type OptionObj = Record<string, RelationshipValue>
+  type OptionsMap = Record<string, BasicRelatedRow>
-  type OptionsObjType = Record<string, OptionObj>
 
+  const { API } = getContext("sdk")
+
+  // Field state
   let fieldState: any
   let fieldApi: any
   let fieldSchema: RelationshipFieldMetadata | undefined
-  let tableDefinition: Table | null | undefined
-  let searchTerm: any
-  let open: boolean
-  let selectedValue: string[] | string
 
-  // need a cast version of this for reactivity, components below aren't typed
+  // Local UI state
-  $: castSelectedValue = selectedValue as any
+  let searchTerm: any
+  let open: boolean = false
+
+  // Options state
+  let options: BasicRelatedRow[] = []
+  let optionsMap: OptionsMap = {}
+  let loadingMissingOptions: boolean = false
+
+  // Determine if we can select multiple rows or not
   $: multiselect =
     [FieldType.LINK, FieldType.BB_REFERENCE].includes(type) &&
     fieldSchema?.relationshipType !== "one-to-many"
-  $: linkedTableId = fieldSchema?.tableId!
+
-  $: fetch = fetchData({
+  // Get the proper string representation of the value
+  $: realValue = fieldState?.value
+  $: selectedValue = parseSelectedValue(realValue, multiselect)
+  $: selectedIDs = getSelectedIDs(selectedValue)
+
+  // If writable, we use a fetch to load options
+  $: linkedTableId = fieldSchema?.tableId
+  $: writable = !disabled && !readonly
+  $: fetch = createFetch(writable, datasourceType, filter, linkedTableId)
+
+  // Attempt to determine the primary display field to use
+  $: tableDefinition = $fetch?.definition
+  $: primaryDisplayField = primaryDisplay || tableDefinition?.primaryDisplay
+
+  // Build our options map
+  $: rows = $fetch?.rows || []
+  $: processOptions(realValue, rows, primaryDisplayField)
+
+  // If we ever have a value selected for which we don't have an option, we must
+  // fetch those rows to ensure we can render them as options
+  $: missingIDs = selectedIDs.filter(id => !optionsMap[id])
+  $: loadMissingOptions(missingIDs, linkedTableId, primaryDisplayField)
+
+  // Convert our options map into an array for display
+  $: updateOptions(optionsMap)
+  $: !open && sortOptions()
+
+  // Search for new options when search term changes
+  $: debouncedSearchOptions(searchTerm || "", primaryDisplayField)
+
+  // Ensure backwards compatibility
+  $: enrichedDefaultValue = enrichDefaultValue(defaultValue)
+
+  // We need to cast value to pass it down, as those components aren't typed
+  $: emptyValue = multiselect ? [] : null
+  $: displayValue = missingIDs.length ? emptyValue : (selectedValue as any)
+
+  // Ensures that we flatten any objects so that only the IDs of the selected
+  // rows are passed down. Not sure how this can be an object to begin with?
+  const parseSelectedValue = (
+    value: any,
+    multiselect: boolean
+  ): undefined | string | string[] => {
+    return multiselect ? flatten(value) : flatten(value)[0]
+  }
+
+  // Where applicable, creates the fetch instance to load row options
+  const createFetch = (
+    writable: boolean,
+    dsType: typeof datasourceType,
+    filter: SearchFilter[],
+    linkedTableId?: string
+  ) => {
+    if (!linkedTableId) {
+      return undefined
+    }
+    const datasource =
+      datasourceType === "table"
+        ? {
+            type: datasourceType,
+            tableId: fieldSchema?.tableId!,
+          }
+        : {
+            type: datasourceType,
+            tableId: InternalTable.USER_METADATA,
+          }
+    return fetchData({
       API,
-    datasource: {
+      datasource,
-      // typing here doesn't seem correct - we have the correct datasourceType options
-      // but when we configure the fetchData, it seems to think only "table" is valid
-      type: datasourceType as any,
-      tableId: linkedTableId,
-    },
       options: {
         filter,
-      limit: 100,
+        limit: writable ? 100 : 1,
       },
     })
-
-  $: tableDefinition = $fetch.definition
-  $: selectedValue = multiselect
-    ? flatten(fieldState?.value) ?? []
-    : flatten(fieldState?.value)?.[0]
-  $: component = multiselect ? CoreMultiselect : CoreSelect
-  $: primaryDisplay = primaryDisplay || tableDefinition?.primaryDisplay
-
-  let optionsObj: OptionsObjType = {}
-  const debouncedFetchRows = Utils.debounce(fetchRows, 250)
-
-  $: {
-    if (primaryDisplay && fieldState && !optionsObj) {
-      // Persist the initial values as options, allowing them to be present in the dropdown,
-      // even if they are not in the inital fetch results
-      let valueAsSafeArray = fieldState.value || []
-      if (!Array.isArray(valueAsSafeArray)) {
-        valueAsSafeArray = [fieldState.value]
   }
-      optionsObj = valueAsSafeArray.reduce(
+
-        (
+  // Small helper to represent the selected value as an array
-          accumulator: OptionObj,
+  const getSelectedIDs = (
-          value: { _id: string; primaryDisplay: any }
+    selectedValue: undefined | string | string[]
+  ): string[] => {
+    if (!selectedValue) {
+      return []
+    }
+    return Array.isArray(selectedValue) ? selectedValue : [selectedValue]
+  }
+
+  // Builds a map of all available options, in a consistent structure
+  const processOptions = (
+    realValue: any | any[],
+    rows: Row[],
+    primaryDisplay?: string
   ) => {
-          // fieldState has to be an array of strings to be valid for an update
+    // First ensure that all options included in the value are present as valid
-          // therefore we cannot guarantee value will be an object
+    // options. These can be basic related row shapes which already include
-          // https://linear.app/budibase/issue/BUDI-7577/refactor-the-relationshipfield-component-to-have-better-support-for
+    // a value for primary display
-          if (!value._id) {
+    if (realValue) {
-            return accumulator
+      const valueArray = Array.isArray(realValue) ? realValue : [realValue]
+      for (let val of valueArray) {
+        const option = parseOption(val, primaryDisplay)
+        if (option) {
+          optionsMap[option._id] = option
         }
-          accumulator[value._id] = {
-            _id: value._id,
-            [primaryDisplay]: value.primaryDisplay,
-          }
-          return accumulator
-        },
-        {}
-      )
       }
     }
 
-  $: enrichedOptions = enrichOptions(optionsObj, $fetch.rows)
+    // Process all rows loaded from our fetch
-  const enrichOptions = (optionsObj: OptionsObjType, fetchResults: Row[]) => {
+    for (let row of rows) {
-    const result = (fetchResults || [])?.reduce((accumulator, row) => {
+      const option = parseOption(row, primaryDisplay)
-      if (!accumulator[row._id!]) {
+      if (option) {
-        accumulator[row._id!] = row
+        optionsMap[option._id] = option
-      }
-      return accumulator
-    }, optionsObj || {})
-
-    return Object.values(result)
-  }
-  $: {
-    // We don't want to reorder while the dropdown is open, to avoid UX jumps
-    if (!open && primaryDisplay) {
-      enrichedOptions = enrichedOptions.sort((a: OptionObj, b: OptionObj) => {
-        const selectedValues = flatten(fieldState?.value) || []
-
-        const aIsSelected = selectedValues.find(
-          (v: RelationshipValue) => v === a._id
-        )
-        const bIsSelected = selectedValues.find(
-          (v: RelationshipValue) => v === b._id
-        )
-        if (aIsSelected && !bIsSelected) {
-          return -1
-        } else if (!aIsSelected && bIsSelected) {
-          return 1
-        }
-
-        return (a[primaryDisplay] > b[primaryDisplay]) as unknown as number
-      })
       }
     }
 
-  $: {
+    // Reassign to trigger reactivity
-    if (filter || defaultValue) {
+    optionsMap = optionsMap
-      forceFetchRows()
   }
-  }
-  $: debouncedFetchRows(searchTerm, primaryDisplay, defaultValue)
 
-  const forceFetchRows = async () => {
+  // Parses a row-like structure into a properly shaped option
-    // if the filter has changed, then we need to reset the options, clear the selection, and re-fetch
+  const parseOption = (
-    optionsObj = {}
+    option: any | BasicRelatedRow | Row,
-    fieldApi?.setValue([])
+    primaryDisplay?: string
-    selectedValue = []
+  ): BasicRelatedRow | null => {
-    debouncedFetchRows(searchTerm, primaryDisplay, defaultValue)
+    if (!option || typeof option !== "object" || !option?._id) {
+      return null
     }
-  async function fetchRows(
+    // If this is a basic related row shape (_id and PD only) then just use
-    searchTerm: any,
+    // that
-    primaryDisplay: string,
+    if (Object.keys(option).length === 2 && "primaryDisplay" in option) {
-    defaultVal: string | string[]
+      return {
+        _id: option._id,
+        primaryDisplay: ensureString(option.primaryDisplay),
+      }
+    }
+    // Otherwise use the primary display field specified
+    if (primaryDisplay) {
+      return {
+        _id: option._id,
+        primaryDisplay: ensureString(
+          option[primaryDisplay as keyof typeof option]
+        ),
+      }
+    } else {
+      return {
+        _id: option._id,
+        primaryDisplay: option._id,
+      }
+    }
+  }
+
+  // Loads any rows which are selected and aren't present in the currently
+  // available option set. This is typically only IDs specified as default
+  // values.
+  const loadMissingOptions = async (
+    missingIDs: string[],
+    linkedTableId?: string,
+    primaryDisplay?: string
+  ) => {
+    if (
+      loadingMissingOptions ||
+      !missingIDs.length ||
+      !linkedTableId ||
+      !primaryDisplay
     ) {
-    const allRowsFetched =
-      $fetch.loaded &&
-      !Object.keys($fetch.query?.string || {}).length &&
-      !$fetch.hasNextPage
-    // Don't request until we have the primary display or default value has been fetched
-    if (allRowsFetched || !primaryDisplay) {
       return
     }
-    // must be an array
+    loadingMissingOptions = true
-    const defaultValArray: string[] = !defaultVal
+    try {
-      ? []
+      const res = await API.searchTable(linkedTableId, {
-      : !Array.isArray(defaultVal)
+        query: {
-      ? defaultVal.split(",")
+          oneOf: {
-      : defaultVal
+            _id: missingIDs,
+          },
+        },
+      })
+      for (let row of res.rows) {
+        const option = parseOption(row, primaryDisplay)
+        if (option) {
+          optionsMap[option._id] = option
+        }
+      }
 
-    if (
+      // Reassign to trigger reactivity
-      defaultVal &&
+      optionsMap = optionsMap
-      optionsObj &&
+      updateOptions(optionsMap)
-      defaultValArray.some(val => !optionsObj[val])
+    } catch (error) {
-    ) {
+      console.error("Error loading missing row IDs", error)
-      await fetch.update({
+    } finally {
-        query: { oneOf: { _id: defaultValArray } },
+      // Ensure we have some sort of option for all IDs
+      for (let id of missingIDs) {
+        if (!optionsMap[id]) {
+          optionsMap[id] = {
+            _id: id,
+            primaryDisplay: id,
+          }
+        }
+      }
+      loadingMissingOptions = false
+    }
+  }
+
+  // Updates the options list to reflect the currently available options
+  const updateOptions = (optionsMap: OptionsMap) => {
+    let newOptions = Object.values(optionsMap)
+
+    // Only override options if the quantity of options changes
+    if (newOptions.length !== options.length) {
+      options = newOptions
+      sortOptions()
+    }
+  }
+
+  // Sorts the options list by selected state, then by primary display
+  const sortOptions = () => {
+    // Create a quick lookup map so we can test whether options are selected
+    const selectedMap: Record<string, boolean> = selectedIDs.reduce(
+      (map, id) => ({ ...map, [id]: true }),
+      {}
+    )
+    options.sort((a, b) => {
+      const aSelected = !!selectedMap[a._id]
+      const bSelected = !!selectedMap[b._id]
+      if (aSelected === bSelected) {
+        return a.primaryDisplay < b.primaryDisplay ? -1 : 1
+      } else {
+        return aSelected ? -1 : 1
+      }
     })
   }
 
-    if (
+  // Util to ensure a value is stringified
-      (Array.isArray(selectedValue) &&
+  const ensureString = (val: any): string => {
-        selectedValue.some(val => !optionsObj[val])) ||
+    return typeof val === "string" ? val : JSON.stringify(val)
-      (selectedValue && !optionsObj[selectedValue as string])
+  }
-    ) {
+
-      await fetch.update({
+  // We previously included logic to manually process default value, which
-        query: {
+  // should not be done as it is handled by the core form logic.
-          oneOf: {
+  // This logic included handling a comma separated list of IDs, so for
-            _id: Array.isArray(selectedValue) ? selectedValue : [selectedValue],
+  // backwards compatibility we must now unfortunately continue to handle that
-          },
+  // at this level.
-        },
+  const enrichDefaultValue = (val: any) => {
-      })
+    if (!val || typeof val !== "string") {
+      return val
+    }
+    return val.includes(",") ? val.split(",") : val
+  }
+
+  // Searches for new options matching the given term
+  async function searchOptions(searchTerm: string, primaryDisplay?: string) {
+    if (!primaryDisplay) {
+      return
     }
 
     // Ensure we match all filters, rather than any
+    let newFilter: any = filter
+    if (searchTerm) {
       // @ts-expect-error this doesn't fit types, but don't want to change it yet
-    const baseFilter: any = (filter || []).filter(x => x.operator !== "allOr")
+      newFilter = (newFilter || []).filter(x => x.operator !== "allOr")
-    await fetch.update({
+      newFilter.push({
-      filter: [
-        ...baseFilter,
-        {
         // Use a big numeric prefix to avoid clashing with an existing filter
         field: `999:${primaryDisplay}`,
         operator: "string",
         value: searchTerm,
-        },
-      ],
       })
     }
+    await fetch?.update({
+      filter: newFilter,
+    })
+  }
+  const debouncedSearchOptions = Utils.debounce(searchOptions, 250)
 
-  const flatten = (values: any | any[]) => {
+  // Flattens an array of row-like objects into a simple array of row IDs
+  const flatten = (values: any | any[]): string[] => {
     if (!values) {
       return []
     }
-
     if (!Array.isArray(values)) {
       values = [values]
     }
@@ -226,16 +337,11 @@
     return values
   }
 
-  const getDisplayName = (row: Row) => {
-    return row?.[primaryDisplay!] || "-"
-  }
-
   const handleChange = (e: any) => {
     let value = e.detail
     if (!multiselect) {
       value = value == null ? [] : [value]
     }
-
     if (
       type === FieldType.BB_REFERENCE_SINGLE &&
       value &&
@@ -243,7 +349,6 @@
     ) {
       value = value[0] || null
     }
-
     const changed = fieldApi.setValue(value)
     if (onChange && changed) {
       onChange({
@@ -251,12 +356,6 @@
       })
     }
   }
-
-  const loadMore = () => {
-    if (!$fetch.loading) {
-      fetch.nextPage()
-    }
-  }
 </script>
 
 <Field
@@ -265,31 +364,31 @@
   {disabled}
   {readonly}
   {validation}
-  {defaultValue}
   {type}
   {span}
   {helpText}
+  defaultValue={enrichedDefaultValue}
   bind:fieldState
   bind:fieldApi
   bind:fieldSchema
 >
   {#if fieldState}
     <svelte:component
-      this={component}
+      this={multiselect ? CoreMultiselect : CoreSelect}
-      options={enrichedOptions}
+      value={displayValue}
-      {autocomplete}
+      id={fieldState?.fieldId}
-      value={castSelectedValue}
+      disabled={fieldState?.disabled}
-      on:change={handleChange}
+      readonly={fieldState?.readonly}
-      on:loadMore={loadMore}
+      loading={!!$fetch?.loading}
-      id={fieldState.fieldId}
+      getOptionLabel={option => option.primaryDisplay}
-      disabled={fieldState.disabled}
-      readonly={fieldState.readonly}
-      getOptionLabel={getDisplayName}
       getOptionValue={option => option._id}
+      {options}
       {placeholder}
+      {autocomplete}
       bind:searchTerm
-      loading={$fetch.loading}
       bind:open
+      on:change={handleChange}
+      on:loadMore={() => fetch?.nextPage()}
     />
   {/if}
 </Field>

packages/frontend-core/src/fetch/GroupUserFetch.ts

@@ -4,7 +4,7 @@ import { GroupUserDatasource, InternalTable } from "@budibase/types"
 
 interface GroupUserQuery {
   groupId: string
-  emailSearch: string
+  emailSearch?: string
 }
 
 interface GroupUserDefinition {

packages/frontend-core/src/fetch/UserFetch.ts

@@ -9,8 +9,8 @@ import {
 } from "@budibase/types"
 
 interface UserFetchQuery {
-  appId: string
+  appId?: string
-  paginated: boolean
+  paginated?: boolean
 }
 
 interface UserDefinition {