From 0fce17c3c06198505f8cddcf4c8f94d1511cacbb Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 14:50:06 +0100 Subject: [PATCH] Fix allOr for everything except SQS. --- .../src/api/routes/tests/viewV2.spec.ts | 20 ++++ packages/server/src/sdk/app/rows/search.ts | 9 +- packages/shared-core/src/filters.ts | 89 ++++++++------ packages/shared-core/src/utils.ts | 113 ++++-------------- packages/types/src/api/web/searchFilter.ts | 42 +++++-- 5 files changed, 139 insertions(+), 134 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index fa1f1690f6..6ba6fb8cb8 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -3756,6 +3756,26 @@ describe.each([ }), expected: [], }, + { + name: "allOr", + insert: [{ string: "foo" }, { string: "bar" }], + query: simpleQuery( + { + operator: BasicOperator.EQUAL, + field: "string", + value: "foo", + }, + { + operator: BasicOperator.EQUAL, + field: "string", + value: "bar", + }, + { + operator: "allOr", + } + ), + expected: [{ string: "foo" }, { string: "bar" }], + }, ] it.only.each(testCases)( diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index caf85a7858..ee16058416 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -114,11 +114,12 @@ export async function search( ? view.query : [] + const { filters } = dataFilters.splitFiltersArray(queryFilters) + // Extract existing fields - const existingFields = - queryFilters - ?.filter(filter => filter.field) - .map(filter => db.removeKeyNumbering(filter.field)) || [] + const existingFields = filters.map(filter => + db.removeKeyNumbering(filter.field) + ) // Carry over filters for unused fields Object.keys(options.query).forEach(key => { diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index eda1139c5e..2d01c24363 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -24,6 +24,7 @@ import { isBasicSearchOperator, isArraySearchOperator, isRangeSearchOperator, + SearchFilter, } from "@budibase/types" import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" @@ -310,24 +311,17 @@ export class ColumnSplitter { * Builds a JSON query from the filter a SearchFilter definition * @param filter the builder filter structure */ -const buildCondition = (expression: LegacyFilter) => { + +function buildCondition(filter: undefined): undefined +function buildCondition(filter: SearchFilter): SearchFilters +function buildCondition(filter?: SearchFilter): SearchFilters | undefined { + if (!filter) { + return + } + const query: SearchFilters = {} - const { operator, field, type, externalType, onEmptyFilter } = expression - let { value } = expression - - if (!operator || !field) { - return - } - - if (operator === "allOr") { - query.allOr = true - return - } - - if (onEmptyFilter) { - query.onEmptyFilter = onEmptyFilter - return - } + const { operator, field, type, externalType } = filter + let { value } = filter // Default the value for noValue fields to ensure they are correctly added // to the final query @@ -432,16 +426,36 @@ const buildCondition = (expression: LegacyFilter) => { return query } +export interface LegacyFilterSplit { + allOr?: boolean + onEmptyFilter?: EmptyFilterOption + filters: SearchFilter[] +} + +export function splitFiltersArray(filters: LegacyFilter[]) { + const split: LegacyFilterSplit = { + filters: [], + } + + for (const filter of filters) { + if ("operator" in filter && filter.operator === "allOr") { + split.allOr = true + } else if ("onEmptyFilter" in filter) { + split.onEmptyFilter = filter.onEmptyFilter + } else { + split.filters.push(filter) + } + } + + return split +} + /** * Converts a **UISearchFilter** filter definition into a grouped * search query of type **SearchFilters** * * Legacy support remains for the old **SearchFilter[]** format. * These will be migrated to an appropriate **SearchFilters** object, if encountered - * - * @param filter - * - * @returns {SearchFilters} */ export function buildQuery(filter: undefined): undefined export function buildQuery( @@ -454,26 +468,33 @@ export function buildQuery( return } - let parsedFilter: UISearchFilter if (Array.isArray(filter)) { - parsedFilter = processSearchFilters(filter) - } else { - parsedFilter = filter + filter = processSearchFilters(filter) } const operator = logicalOperatorFromUI( - parsedFilter.logicalOperator || UILogicalOperator.ALL + filter.logicalOperator || UILogicalOperator.ALL ) - const groups = parsedFilter.groups || [] - const conditions: SearchFilters[] = groups.map(group => { - const filters = group.filters || [] - return { [operator]: { conditions: filters.map(x => buildCondition(x)) } } - }) - return { - onEmptyFilter: parsedFilter.onEmptyFilter, - [operator]: { conditions }, + const query: SearchFilters = {} + if (filter.onEmptyFilter) { + query.onEmptyFilter = filter.onEmptyFilter } + + query[operator] = { + conditions: (filter.groups || []).map(group => { + const { allOr, onEmptyFilter, filters } = splitFiltersArray( + group.filters || [] + ) + if (onEmptyFilter) { + query.onEmptyFilter = onEmptyFilter + } + const operator = allOr ? LogicalOperator.OR : LogicalOperator.AND + return { [operator]: { conditions: filters.map(buildCondition) } } + }), + } + + return query } function logicalOperatorFromUI(operator: UILogicalOperator): LogicalOperator { diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index 4f3c4f3deb..dce1b8b960 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -6,15 +6,22 @@ import { BasicOperator, ArrayOperator, isLogicalSearchOperator, - EmptyFilterOption, - SearchFilterGroup, + SearchFilter, } from "@budibase/types" import * as Constants from "./constants" -import { removeKeyNumbering } from "./filters" +import { removeKeyNumbering, splitFiltersArray } from "./filters" +import _ from "lodash" -// an array of keys from filter type to properties that are in the type -// this can then be converted using .fromEntries to an object -type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][] +const FILTER_ALLOWED_KEYS = [ + "field", + "operator", + "value", + "type", + "externalType", + "valueType", + "noValue", + "formulaType", +] export function unreachable( value: never, @@ -130,88 +137,20 @@ export function isSupportedUserSearch(query: SearchFilters) { return true } -/** - * Processes the filter config. Filters are migrated from - * SearchFilter[] to UISearchFilter - * - * If filters is not an array, the migration is skipped - * - * @param {LegacyFilter[] | UISearchFilter} filters - */ export const processSearchFilters = ( - filters: LegacyFilter[] + filterArray: LegacyFilter[] ): UISearchFilter => { - // Base search config. - const defaultCfg: UISearchFilter = { - logicalOperator: UILogicalOperator.ALL, - onEmptyFilter: EmptyFilterOption.RETURN_ALL, - groups: [], - } - - const filterAllowedKeys = [ - "field", - "operator", - "value", - "type", - "externalType", - "valueType", - "noValue", - "formulaType", - ] - - let baseGroup: SearchFilterGroup = { - logicalOperator: UILogicalOperator.ALL, - } - - return filters.reduce((acc: UISearchFilter, filter: LegacyFilter) => { - // Sort the properties for easier debugging - const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[]) - .sort((a, b) => { - return a.localeCompare(b) - }) - .filter(key => filter[key]) - - if (filterPropertyKeys.length == 1) { - const key = filterPropertyKeys[0], - value = filter[key] - // Global - if (key === "onEmptyFilter") { - // unset otherwise - acc.onEmptyFilter = value - } else if (key === "operator" && value === "allOr") { - // Group 1 logical operator - baseGroup.logicalOperator = UILogicalOperator.ANY - } - - return acc - } - - const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce( - (acc: AllowedFilters, key) => { - const value = filter[key] - if (filterAllowedKeys.includes(key)) { - if (key === "field") { - acc.push([key, removeKeyNumbering(value)]) - } else { - acc.push([key, value]) - } - } - return acc + const { allOr, onEmptyFilter, filters } = splitFiltersArray(filterArray) + return { + onEmptyFilter, + groups: [ + { + logicalOperator: allOr ? UILogicalOperator.ANY : UILogicalOperator.ALL, + filters: filters.map(filter => { + filter.field = removeKeyNumbering(filter.field) + return _.pick(filter, FILTER_ALLOWED_KEYS) as SearchFilter + }), }, - [] - ) - - const migratedFilter: LegacyFilter = Object.fromEntries( - allowedFilterSettings - ) as LegacyFilter - - baseGroup.filters!.push(migratedFilter) - - if (!acc.groups || !acc.groups.length) { - // init the base group - acc.groups = [baseGroup] - } - - return acc - }, defaultCfg) + ], + } } diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts index 428d5d364a..b21484f6ed 100644 --- a/packages/types/src/api/web/searchFilter.ts +++ b/packages/types/src/api/web/searchFilter.ts @@ -1,16 +1,40 @@ import { FieldType } from "../../documents" -import { EmptyFilterOption, UILogicalOperator, SearchFilters } from "../../sdk" +import { + EmptyFilterOption, + UILogicalOperator, + BasicOperator, + RangeOperator, + ArrayOperator, +} from "../../sdk" + +type AllOr = { + operator: "allOr" +} + +type OnEmptyFilter = { + onEmptyFilter: EmptyFilterOption +} + +// TODO(samwho): this could be broken down further +export type SearchFilter = { + operator: + | BasicOperator + | RangeOperator + | ArrayOperator + | "rangeLow" + | "rangeHigh" + // Field name will often have a numerical prefix when coming from the frontend, + // use the ColumnSplitter class to remove it. + field: string + value: any + type?: FieldType + externalType?: string + noValue?: boolean +} // Prior to v2, this is the type the frontend sent us when filters were // involved. We convert this to a SearchFilters before use with the search SDK. -export type LegacyFilter = { - operator: keyof SearchFilters | "rangeLow" | "rangeHigh" - onEmptyFilter?: EmptyFilterOption - field: string - type?: FieldType - value: any - externalType?: string -} +export type LegacyFilter = AllOr | OnEmptyFilter | SearchFilter export type SearchFilterGroup = { logicalOperator?: UILogicalOperator