Support converting nested UI filters into queries, add tests, improve types and fix relationship pickers with filters applied caching options incorrectly

packages/client/src/components/app/forms/RelationshipField.svelte

@@ -1,17 +1,27 @@
+<script context="module" lang="ts">
+  type ValueType = string | string[]
+  type BasicRelatedRow = { _id: string; primaryDisplay: string }
+  type OptionsMap = Record<string, BasicRelatedRow>
+</script>
+
 <script lang="ts">
   import { CoreSelect, CoreMultiselect } from "@budibase/bbui"
-  import { BasicOperator, FieldType, InternalTable } from "@budibase/types"
+  import {
+    BasicOperator,
+    EmptyFilterOption,
+    FieldType,
+    InternalTable,
+    UILogicalOperator,
+    type LegacyFilter,
+    type SearchFilterGroup,
+    type UISearchFilter,
+  } from "@budibase/types"
   import { fetchData, Utils } from "@budibase/frontend-core"
   import { getContext } from "svelte"
   import Field from "./Field.svelte"
-  import type {
+  import type { RelationshipFieldMetadata, Row } from "@budibase/types"
-    SearchFilter,
-    RelationshipFieldMetadata,
-    Row,
-  } from "@budibase/types"
   import type { FieldApi, FieldState, FieldValidation } from "@/types"
-
+  import { utils } from "@budibase/shared-core"
-  type ValueType = string | string[]
 
   export let field: string | undefined = undefined
   export let label: string | undefined = undefined
@@ -22,7 +32,7 @@
   export let autocomplete: boolean = true
   export let defaultValue: ValueType | undefined = undefined
   export let onChange: (_props: { value: ValueType }) => void
-  export let filter: SearchFilter[]
+  export let filter: UISearchFilter | LegacyFilter[] | undefined = undefined
   export let datasourceType: "table" | "user" = "table"
   export let primaryDisplay: string | undefined = undefined
   export let span: number | undefined = undefined
@@ -32,14 +42,10 @@
     | FieldType.BB_REFERENCE
     | FieldType.BB_REFERENCE_SINGLE = FieldType.LINK
 
-  type BasicRelatedRow = { _id: string; primaryDisplay: string }
-  type OptionsMap = Record<string, BasicRelatedRow>
-
   const { API } = getContext("sdk")
 
   // Field state
   let fieldState: FieldState<string | string[]> | undefined
-
   let fieldApi: FieldApi
   let fieldSchema: RelationshipFieldMetadata | undefined
 
@@ -52,6 +58,9 @@
   let optionsMap: OptionsMap = {}
   let loadingMissingOptions: boolean = false
 
+  // Reset the available options when our base filter changes
+  $: filter, (optionsMap = {})
+
   // Determine if we can select multiple rows or not
   $: multiselect =
     [FieldType.LINK, FieldType.BB_REFERENCE].includes(type) &&
@@ -65,7 +74,13 @@
   // If writable, we use a fetch to load options
   $: linkedTableId = fieldSchema?.tableId
   $: writable = !disabled && !readonly
-  $: fetch = createFetch(writable, datasourceType, filter, linkedTableId)
+  $: migratedFilter = migrateFilter(filter)
+  $: fetch = createFetch(
+    writable,
+    datasourceType,
+    migratedFilter,
+    linkedTableId
+  )
 
   // Attempt to determine the primary display field to use
   $: tableDefinition = $fetch?.definition
@@ -90,8 +105,8 @@
   // Ensure backwards compatibility
   $: enrichedDefaultValue = enrichDefaultValue(defaultValue)
 
-  $: emptyValue = multiselect ? [] : undefined
   // We need to cast value to pass it down, as those components aren't typed
+  $: emptyValue = multiselect ? [] : undefined
   $: displayValue = (missingIDs.length ? emptyValue : selectedValue) as any
 
   // Ensures that we flatten any objects so that only the IDs of the selected
@@ -107,7 +122,7 @@
   const createFetch = (
     writable: boolean,
     dsType: typeof datasourceType,
-    filter: SearchFilter[],
+    filter: UISearchFilter | undefined,
     linkedTableId?: string
   ) => {
     const datasource =
@@ -176,9 +191,16 @@
     option: string | BasicRelatedRow | Row,
     primaryDisplay?: string
   ): BasicRelatedRow | null => {
+    // For plain strings, check if we already have this option available
+    if (typeof option === "string" && optionsMap[option]) {
+      return optionsMap[option]
+    }
+
+    // Otherwise ensure we have a valid option object
     if (!option || typeof option !== "object" || !option?._id) {
       return null
     }
+
     // If this is a basic related row shape (_id and PD only) then just use
     // that
     if (Object.keys(option).length === 2 && "primaryDisplay" in option) {
@@ -300,24 +322,54 @@
     return val.includes(",") ? val.split(",") : val
   }
 
+  // We may need to migrate the filter structure, in the case of this being
+  // an old app with LegacyFilter[] saved
+  const migrateFilter = (
+    filter: UISearchFilter | LegacyFilter[] | undefined
+  ): UISearchFilter | undefined => {
+    if (Array.isArray(filter)) {
+      return utils.processSearchFilters(filter)
+    }
+    return filter
+  }
+
   // Searches for new options matching the given term
   async function searchOptions(searchTerm: string, primaryDisplay?: string) {
     if (!primaryDisplay) {
       return
     }
-
+    let newFilter: UISearchFilter | undefined = undefined
-    // Ensure we match all filters, rather than any
+    let searchFilter: SearchFilterGroup = {
-    let newFilter = filter
+      logicalOperator: UILogicalOperator.ALL,
-    if (searchTerm) {
+      filters: [
-      // @ts-expect-error this doesn't fit types, but don't want to change it yet
+        {
-      newFilter = (newFilter || []).filter(x => x.operator !== "allOr")
+          field: primaryDisplay,
-      newFilter.push({
+          operator: BasicOperator.STRING,
-        // Use a big numeric prefix to avoid clashing with an existing filter
+          value: searchTerm,
-        field: `999:${primaryDisplay}`,
+        },
-        operator: BasicOperator.STRING,
+      ],
-        value: searchTerm,
-      })
     }
+
+    // Determine the new filter to apply to the fetch
+    if (searchTerm && migratedFilter) {
+      // If we have both a search term and existing filter, filter by both
+      newFilter = {
+        logicalOperator: UILogicalOperator.ALL,
+        groups: [searchFilter, migratedFilter],
+        onEmptyFilter: EmptyFilterOption.RETURN_NONE,
+      }
+    } else if (searchTerm) {
+      // If we just have a search term them use that
+      newFilter = {
+        logicalOperator: UILogicalOperator.ALL,
+        groups: [searchFilter],
+        onEmptyFilter: EmptyFilterOption.RETURN_NONE,
+      }
+    } else {
+      // Otherwise use the supplied filter untouched
+      newFilter = migratedFilter
+    }
+
     await fetch?.update({
       filter: newFilter,
     })
@@ -389,7 +441,6 @@
       bind:searchTerm
       bind:open
       on:change={handleChange}
-      on:loadMore={() => fetch?.nextPage()}
     />
   {/if}
 </Field>

packages/shared-core/src/filters.ts

@@ -1,30 +1,30 @@
 import {
-  Datasource,
+  ArrayOperator,
+  BasicOperator,
   BBReferenceFieldSubType,
+  Datasource,
+  EmptyFilterOption,
+  FieldConstraints,
   FieldType,
   FormulaType,
+  isArraySearchOperator,
+  isBasicSearchOperator,
+  isLogicalSearchOperator,
+  isRangeSearchOperator,
   LegacyFilter,
+  LogicalOperator,
+  RangeOperator,
+  RowSearchParams,
+  SearchFilter,
+  SearchFilterOperator,
   SearchFilters,
   SearchQueryFields,
-  ArrayOperator,
-  SearchFilterOperator,
-  SortType,
-  FieldConstraints,
-  SortOrder,
-  RowSearchParams,
-  EmptyFilterOption,
   SearchResponse,
+  SortOrder,
+  SortType,
   Table,
-  BasicOperator,
-  RangeOperator,
-  LogicalOperator,
-  isLogicalSearchOperator,
-  UISearchFilter,
   UILogicalOperator,
-  isBasicSearchOperator,
+  UISearchFilter,
-  isArraySearchOperator,
-  isRangeSearchOperator,
-  SearchFilter,
 } from "@budibase/types"
 import dayjs from "dayjs"
 import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
@@ -444,6 +444,7 @@ export function buildQuery(
     return {}
   }
 
+  // Migrate legacy filters if required
   if (Array.isArray(filter)) {
     filter = processSearchFilters(filter)
     if (!filter) {
@@ -451,10 +452,7 @@ export function buildQuery(
     }
   }
 
-  const operator = logicalOperatorFromUI(
+  // Determine top level empty filter behaviour
-    filter.logicalOperator || UILogicalOperator.ALL
-  )
-
   const query: SearchFilters = {}
   if (filter.onEmptyFilter) {
     query.onEmptyFilter = filter.onEmptyFilter
@@ -462,8 +460,24 @@ export function buildQuery(
     query.onEmptyFilter = EmptyFilterOption.RETURN_ALL
   }
 
+  // Default to matching all groups/filters
+  const operator = logicalOperatorFromUI(
+    filter.logicalOperator || UILogicalOperator.ALL
+  )
+
   query[operator] = {
     conditions: (filter.groups || []).map(group => {
+      // Check if we contain more groups
+      if (group.groups) {
+        const searchFilter = buildQuery(group)
+
+        // We don't define this properly in the types, but certain fields should
+        // not be present in these nested search filters
+        delete searchFilter.onEmptyFilter
+        return searchFilter
+      }
+
+      // Otherwise handle filters
       const { allOr, onEmptyFilter, filters } = splitFiltersArray(
         group.filters || []
       )
@@ -471,7 +485,7 @@ export function buildQuery(
         query.onEmptyFilter = onEmptyFilter
       }
 
-      // logicalOperator takes precendence over allOr
+      // logicalOperator takes precedence over allOr
       let operator = allOr ? LogicalOperator.OR : LogicalOperator.AND
       if (group.logicalOperator) {
         operator = logicalOperatorFromUI(group.logicalOperator)

packages/shared-core/src/tests/filters.spec.ts

@@ -0,0 +1,156 @@
+import { buildQuery } from "../filters"
+import {
+  BasicOperator,
+  EmptyFilterOption,
+  FieldType,
+  UILogicalOperator,
+  UISearchFilter,
+} from "@budibase/types"
+
+describe("filter to query conversion", () => {
+  it("handles a filter with 1 group", () => {
+    const filter: UISearchFilter = {
+      logicalOperator: UILogicalOperator.ALL,
+      onEmptyFilter: EmptyFilterOption.RETURN_NONE,
+      groups: [
+        {
+          logicalOperator: UILogicalOperator.ALL,
+          filters: [
+            {
+              field: "city",
+              operator: BasicOperator.STRING,
+              value: "lon",
+            },
+          ],
+        },
+      ],
+    }
+    const query = buildQuery(filter)
+    expect(query).toEqual({
+      onEmptyFilter: "none",
+      $and: {
+        conditions: [
+          {
+            $and: {
+              conditions: [
+                {
+                  string: {
+                    city: "lon",
+                  },
+                },
+              ],
+            },
+          },
+        ],
+      },
+    })
+  })
+
+  it("handles an empty filter", () => {
+    const filter = undefined
+    const query = buildQuery(filter)
+    expect(query).toEqual({})
+  })
+
+  it("handles legacy filters", () => {
+    const filter = [
+      {
+        field: "city",
+        operator: BasicOperator.STRING,
+        value: "lon",
+      },
+    ]
+    const query = buildQuery(filter)
+    expect(query).toEqual({
+      onEmptyFilter: "all",
+      $and: {
+        conditions: [
+          {
+            $and: {
+              conditions: [
+                {
+                  string: {
+                    city: "lon",
+                  },
+                },
+              ],
+            },
+          },
+        ],
+      },
+    })
+  })
+
+  it("handles nested groups", () => {
+    const filter: UISearchFilter = {
+      logicalOperator: UILogicalOperator.ALL,
+      onEmptyFilter: EmptyFilterOption.RETURN_NONE,
+      groups: [
+        {
+          logicalOperator: UILogicalOperator.ALL,
+          filters: [
+            {
+              field: "city",
+              operator: BasicOperator.STRING,
+              value: "lon",
+            },
+          ],
+        },
+        {
+          logicalOperator: UILogicalOperator.ALL,
+          groups: [
+            {
+              logicalOperator: UILogicalOperator.ANY,
+              filters: [
+                {
+                  valueType: "Binding",
+                  field: "country.country_name",
+                  type: FieldType.STRING,
+                  operator: BasicOperator.EQUAL,
+                  noValue: false,
+                  value: "England",
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    }
+    const query = buildQuery(filter)
+    expect(query).toEqual({
+      onEmptyFilter: "none",
+      $and: {
+        conditions: [
+          {
+            $and: {
+              conditions: [
+                {
+                  string: {
+                    city: "lon",
+                  },
+                },
+              ],
+            },
+          },
+          {
+            $and: {
+              conditions: [
+                {
+                  $or: {
+                    conditions: [
+                      {
+                        equal: {
+                          "country.country_name": "England",
+                        },
+                      },
+                    ],
+                  },
+                },
+              ],
+            },
+          },
+        ],
+      },
+    })
+  })
+})

packages/types/src/api/web/searchFilter.ts

@@ -6,6 +6,7 @@ import {
   RangeOperator,
   ArrayOperator,
 } from "../../sdk"
+import { WithRequired } from "../../shared"
 
 type AllOr = {
   operator: "allOr"
@@ -38,11 +39,19 @@ export type SearchFilter = {
 // involved. We convert this to a SearchFilters before use with the search SDK.
 export type LegacyFilter = AllOr | OnEmptyFilter | SearchFilter
 
+// A search filter group should either contain groups or filters, but not both
 export type SearchFilterGroup = {
   logicalOperator?: UILogicalOperator
-  groups?: SearchFilterGroup[]
+} & (
-  filters?: LegacyFilter[]
+  | {
-}
+      groups?: (SearchFilterGroup | UISearchFilter)[]
+      filters?: never
+    }
+  | {
+      filters?: LegacyFilter[]
+      groups?: never
+    }
+)
 
 // As of v3, this is the format that the frontend always sends when search
 // filters are involved. We convert this to SearchFilters before use with the