Fix allOr for everything except SQS.

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -3756,6 +3756,26 @@ describe.each([
               }),
               expected: [],
             },
+            {
+              name: "allOr",
+              insert: [{ string: "foo" }, { string: "bar" }],
+              query: simpleQuery(
+                {
+                  operator: BasicOperator.EQUAL,
+                  field: "string",
+                  value: "foo",
+                },
+                {
+                  operator: BasicOperator.EQUAL,
+                  field: "string",
+                  value: "bar",
+                },
+                {
+                  operator: "allOr",
+                }
+              ),
+              expected: [{ string: "foo" }, { string: "bar" }],
+            },
           ]
 
           it.only.each(testCases)(

packages/server/src/sdk/app/rows/search.ts

@@ -114,11 +114,12 @@ export async function search(
           ? view.query
           : []
 
+        const { filters } = dataFilters.splitFiltersArray(queryFilters)
+
         // Extract existing fields
-        const existingFields =
+        const existingFields = filters.map(filter =>
-          queryFilters
+          db.removeKeyNumbering(filter.field)
-            ?.filter(filter => filter.field)
+        )
-            .map(filter => db.removeKeyNumbering(filter.field)) || []
 
         // Carry over filters for unused fields
         Object.keys(options.query).forEach(key => {

packages/shared-core/src/filters.ts

@@ -24,6 +24,7 @@ import {
   isBasicSearchOperator,
   isArraySearchOperator,
   isRangeSearchOperator,
+  SearchFilter,
 } from "@budibase/types"
 import dayjs from "dayjs"
 import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
@@ -310,24 +311,17 @@ export class ColumnSplitter {
  * Builds a JSON query from the filter a SearchFilter definition
  * @param filter the builder filter structure
  */
-const buildCondition = (expression: LegacyFilter) => {
+
+function buildCondition(filter: undefined): undefined
+function buildCondition(filter: SearchFilter): SearchFilters
+function buildCondition(filter?: SearchFilter): SearchFilters | undefined {
+  if (!filter) {
+    return
+  }
+
   const query: SearchFilters = {}
-  const { operator, field, type, externalType, onEmptyFilter } = expression
+  const { operator, field, type, externalType } = filter
-  let { value } = expression
+  let { value } = filter
-
-  if (!operator || !field) {
-    return
-  }
-
-  if (operator === "allOr") {
-    query.allOr = true
-    return
-  }
-
-  if (onEmptyFilter) {
-    query.onEmptyFilter = onEmptyFilter
-    return
-  }
 
   // Default the value for noValue fields to ensure they are correctly added
   // to the final query
@@ -432,16 +426,36 @@ const buildCondition = (expression: LegacyFilter) => {
   return query
 }
 
+export interface LegacyFilterSplit {
+  allOr?: boolean
+  onEmptyFilter?: EmptyFilterOption
+  filters: SearchFilter[]
+}
+
+export function splitFiltersArray(filters: LegacyFilter[]) {
+  const split: LegacyFilterSplit = {
+    filters: [],
+  }
+
+  for (const filter of filters) {
+    if ("operator" in filter && filter.operator === "allOr") {
+      split.allOr = true
+    } else if ("onEmptyFilter" in filter) {
+      split.onEmptyFilter = filter.onEmptyFilter
+    } else {
+      split.filters.push(filter)
+    }
+  }
+
+  return split
+}
+
 /**
  * Converts a **UISearchFilter** filter definition into a grouped
  * search query of type **SearchFilters**
  *
  * Legacy support remains for the old **SearchFilter[]** format.
  * These will be migrated to an appropriate **SearchFilters** object, if encountered
- *
- * @param filter
- *
- * @returns {SearchFilters}
  */
 export function buildQuery(filter: undefined): undefined
 export function buildQuery(
@@ -454,26 +468,33 @@ export function buildQuery(
     return
   }
 
-  let parsedFilter: UISearchFilter
   if (Array.isArray(filter)) {
-    parsedFilter = processSearchFilters(filter)
+    filter = processSearchFilters(filter)
-  } else {
-    parsedFilter = filter
   }
 
   const operator = logicalOperatorFromUI(
-    parsedFilter.logicalOperator || UILogicalOperator.ALL
+    filter.logicalOperator || UILogicalOperator.ALL
   )
-  const groups = parsedFilter.groups || []
-  const conditions: SearchFilters[] = groups.map(group => {
-    const filters = group.filters || []
-    return { [operator]: { conditions: filters.map(x => buildCondition(x)) } }
-  })
 
-  return {
+  const query: SearchFilters = {}
-    onEmptyFilter: parsedFilter.onEmptyFilter,
+  if (filter.onEmptyFilter) {
-    [operator]: { conditions },
+    query.onEmptyFilter = filter.onEmptyFilter
   }
+
+  query[operator] = {
+    conditions: (filter.groups || []).map(group => {
+      const { allOr, onEmptyFilter, filters } = splitFiltersArray(
+        group.filters || []
+      )
+      if (onEmptyFilter) {
+        query.onEmptyFilter = onEmptyFilter
+      }
+      const operator = allOr ? LogicalOperator.OR : LogicalOperator.AND
+      return { [operator]: { conditions: filters.map(buildCondition) } }
+    }),
+  }
+
+  return query
 }
 
 function logicalOperatorFromUI(operator: UILogicalOperator): LogicalOperator {

packages/shared-core/src/utils.ts

@@ -6,15 +6,22 @@ import {
   BasicOperator,
   ArrayOperator,
   isLogicalSearchOperator,
-  EmptyFilterOption,
+  SearchFilter,
-  SearchFilterGroup,
 } from "@budibase/types"
 import * as Constants from "./constants"
-import { removeKeyNumbering } from "./filters"
+import { removeKeyNumbering, splitFiltersArray } from "./filters"
+import _ from "lodash"
 
-// an array of keys from filter type to properties that are in the type
+const FILTER_ALLOWED_KEYS = [
-// this can then be converted using .fromEntries to an object
+  "field",
-type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][]
+  "operator",
+  "value",
+  "type",
+  "externalType",
+  "valueType",
+  "noValue",
+  "formulaType",
+]
 
 export function unreachable(
   value: never,
@@ -130,88 +137,20 @@ export function isSupportedUserSearch(query: SearchFilters) {
   return true
 }
 
-/**
- * Processes the filter config. Filters are migrated from
- * SearchFilter[] to UISearchFilter
- *
- * If filters is not an array, the migration is skipped
- *
- * @param {LegacyFilter[] | UISearchFilter} filters
- */
 export const processSearchFilters = (
-  filters: LegacyFilter[]
+  filterArray: LegacyFilter[]
 ): UISearchFilter => {
-  // Base search config.
+  const { allOr, onEmptyFilter, filters } = splitFiltersArray(filterArray)
-  const defaultCfg: UISearchFilter = {
+  return {
-    logicalOperator: UILogicalOperator.ALL,
+    onEmptyFilter,
-    onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+    groups: [
-    groups: [],
+      {
-  }
+        logicalOperator: allOr ? UILogicalOperator.ANY : UILogicalOperator.ALL,
-
+        filters: filters.map(filter => {
-  const filterAllowedKeys = [
+          filter.field = removeKeyNumbering(filter.field)
-    "field",
+          return _.pick(filter, FILTER_ALLOWED_KEYS) as SearchFilter
-    "operator",
+        }),
-    "value",
-    "type",
-    "externalType",
-    "valueType",
-    "noValue",
-    "formulaType",
-  ]
-
-  let baseGroup: SearchFilterGroup = {
-    logicalOperator: UILogicalOperator.ALL,
-  }
-
-  return filters.reduce((acc: UISearchFilter, filter: LegacyFilter) => {
-    // Sort the properties for easier debugging
-    const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[])
-      .sort((a, b) => {
-        return a.localeCompare(b)
-      })
-      .filter(key => filter[key])
-
-    if (filterPropertyKeys.length == 1) {
-      const key = filterPropertyKeys[0],
-        value = filter[key]
-      // Global
-      if (key === "onEmptyFilter") {
-        // unset otherwise
-        acc.onEmptyFilter = value
-      } else if (key === "operator" && value === "allOr") {
-        // Group 1 logical operator
-        baseGroup.logicalOperator = UILogicalOperator.ANY
-      }
-
-      return acc
-    }
-
-    const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce(
-      (acc: AllowedFilters, key) => {
-        const value = filter[key]
-        if (filterAllowedKeys.includes(key)) {
-          if (key === "field") {
-            acc.push([key, removeKeyNumbering(value)])
-          } else {
-            acc.push([key, value])
-          }
-        }
-        return acc
       },
-      []
+    ],
-    )
-
-    const migratedFilter: LegacyFilter = Object.fromEntries(
-      allowedFilterSettings
-    ) as LegacyFilter
-
-    baseGroup.filters!.push(migratedFilter)
-
-    if (!acc.groups || !acc.groups.length) {
-      // init the base group
-      acc.groups = [baseGroup]
   }
-
-    return acc
-  }, defaultCfg)
 }

packages/types/src/api/web/searchFilter.ts

@@ -1,16 +1,40 @@
 import { FieldType } from "../../documents"
-import { EmptyFilterOption, UILogicalOperator, SearchFilters } from "../../sdk"
+import {
+  EmptyFilterOption,
+  UILogicalOperator,
+  BasicOperator,
+  RangeOperator,
+  ArrayOperator,
+} from "../../sdk"
+
+type AllOr = {
+  operator: "allOr"
+}
+
+type OnEmptyFilter = {
+  onEmptyFilter: EmptyFilterOption
+}
+
+// TODO(samwho): this could be broken down further
+export type SearchFilter = {
+  operator:
+    | BasicOperator
+    | RangeOperator
+    | ArrayOperator
+    | "rangeLow"
+    | "rangeHigh"
+  // Field name will often have a numerical prefix when coming from the frontend,
+  // use the ColumnSplitter class to remove it.
+  field: string
+  value: any
+  type?: FieldType
+  externalType?: string
+  noValue?: boolean
+}
 
 // Prior to v2, this is the type the frontend sent us when filters were
 // involved. We convert this to a SearchFilters before use with the search SDK.
-export type LegacyFilter = {
+export type LegacyFilter = AllOr | OnEmptyFilter | SearchFilter
-  operator: keyof SearchFilters | "rangeLow" | "rangeHigh"
-  onEmptyFilter?: EmptyFilterOption
-  field: string
-  type?: FieldType
-  value: any
-  externalType?: string
-}
 
 export type SearchFilterGroup = {
   logicalOperator?: UILogicalOperator