From 49628e6235f528d2cac03ecf412c300cb54e0ea1 Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 22 Aug 2024 09:26:08 +0100 Subject: [PATCH 01/19] Merge commit --- .../builder/src/stores/builder/components.js | 35 +++- packages/frontend-core/src/utils/utils.js | 97 +++++++++++ packages/shared-core/src/filters.ts | 154 +++++++++++++++++- 3 files changed, 282 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js index c281c73dfe..dae2301068 100644 --- a/packages/builder/src/stores/builder/components.js +++ b/packages/builder/src/stores/builder/components.js @@ -29,7 +29,7 @@ import { DB_TYPE_EXTERNAL, } from "constants/backend" import BudiStore from "../BudiStore" -import { Utils } from "@budibase/frontend-core" +import { Utils, Constants } from "@budibase/frontend-core" import { FieldType } from "@budibase/types" export const INITIAL_COMPONENTS_STATE = { @@ -196,6 +196,26 @@ export class ComponentStore extends BudiStore { } } + if (!enrichedComponent?._component) { + return migrated + } + + const def = this.getDefinition(enrichedComponent?._component) + const filterableTypes = def.settings.filter(setting => + setting?.type?.startsWith("filter") + ) + + // Map this? + for (let setting of filterableTypes) { + const updateFilter = Utils.migrateSearchFilters( + enrichedComponent[setting.key] + ) + // DEAN - switch to real value when complete + if (updateFilter) { + enrichedComponent[setting.key + "_x"] = updateFilter + } + } + return migrated } @@ -405,7 +425,13 @@ export class ComponentStore extends BudiStore { screen: get(selectedScreen), useDefaultValues: true, }) - this.migrateSettings(instance) + + try { + this.migrateSettings(instance) + } catch (e) { + console.error(e) + throw e + } // Custom post processing for creation only let extras = {} @@ -555,7 +581,10 @@ export class ComponentStore extends BudiStore { const patchResult = patchFn(component, screen) // Post processing - const migrated = this.migrateSettings(component) + // DEAN - SKIP ON SAVE FOR THE MOMENT + const migrated = null + + this.migrateSettings(component) // Returning an explicit false signifies that we should skip this // update. If we migrated something, ensure we never skip. diff --git a/packages/frontend-core/src/utils/utils.js b/packages/frontend-core/src/utils/utils.js index b8cfa5ad37..60752a35fe 100644 --- a/packages/frontend-core/src/utils/utils.js +++ b/packages/frontend-core/src/utils/utils.js @@ -1,5 +1,6 @@ import { makePropSafe as safe } from "@budibase/string-templates" import { Helpers } from "@budibase/bbui" +import * as Constants from "../constants" export const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)) @@ -350,3 +351,99 @@ export const buildMultiStepFormBlockDefaultProps = props => { title, } } + +/** + * Processes the filter config. Filters are migrated + * SearchFilter[] to SearchFilterGroup + * + * This is supposed to be in shared-core + * @param {Object[]} filter + */ +export const migrateSearchFilters = filters => { + const defaultCfg = { + logicalOperator: Constants.FilterOperator.ALL, + groups: [], + } + + const filterWhitelistKeys = [ + "field", + "operator", + "valueType", // bb + "value", + "type", + "noValue", // bb + ] + + /** + * Review these + * externalType, formulaType, subtype + */ + + if (Array.isArray(filters)) { + const baseGroup = { + filters: [], + logicalOperator: Constants.FilterOperator.ALL, + } + + const migratedSetting = filters.reduce((acc, filter) => { + // Sort the properties for easier debugging + // Remove unset values + const filterEntries = Object.entries(filter) + .sort((a, b) => { + return a[0].localeCompare(b[0]) + }) + .filter(x => x[1] ?? false) + + // Scrub invalid filters + const { operator, onEmptyFilter, field, valueType } = filter + if (!field || !valueType) { + // THIS SCRUBS THE 2 GLOBALS + // return acc + } + + if (filterEntries.length == 1) { + console.log("### one entry ") + const [key, value] = filterEntries[0] + // Global + if (key === "onEmptyFilter") { + // unset otherwise, seems to be the default + acc.onEmptyFilter = value + } else if (key === "operator" && value === "allOr") { + // Group 1 logical operator + baseGroup.logicalOperator = Constants.FilterOperator.ANY + } + + return acc + } + + // Process settings?? + const whiteListedFilterSettings = filterEntries.reduce((acc, entry) => { + const [key, value] = entry + + if (filterWhitelistKeys.includes(key)) { + if (key === "field") { + acc.push([key, value.replace(/^[0-9]+:/, "")]) + } else { + acc.push([key, value]) + } + } + return acc + }, []) + + const migratedFilter = Object.fromEntries(whiteListedFilterSettings) + + baseGroup.filters.push(migratedFilter) + + if (!acc.groups.length) { + // init the base group + acc.groups.push(baseGroup) + } + + return acc + }, defaultCfg) + + console.log("MIGRATED ", migratedSetting) + return migratedSetting + } + return false +} diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 2dd485a8a0..d695859979 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -19,12 +19,15 @@ import { RangeOperator, LogicalOperator, isLogicalSearchOperator, + SearchFilterGroup, + FilterGroupLogicalOperator, } from "@budibase/types" import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" import { deepGet, schema } from "./helpers" import { isPlainObject, isEmpty } from "lodash" import { decodeNonAscii } from "./helpers/schema" +// import { Constants } from "@budibase/frontend-core" const HBS_REGEX = /{{([^{].*?)}}/g @@ -263,11 +266,134 @@ export class ColumnSplitter { } } +/* + separate into buildQuery and build filter? +*/ + /** * Builds a JSON query from the filter structure generated in the builder * @param filter the builder filter structure */ + +const builderFilter = (expression: SearchFilter) => { + // Filter body + let query: SearchFilters = { + // string: {}, + // fuzzy: {}, + // range: {}, + // equal: {}, + // notEqual: {}, + // empty: {}, + // notEmpty: {}, + // contains: {}, + // notContains: {}, + // oneOf: {}, + // containsAny: {}, + } + + // DEAN - + // This is the chattiest service we have, pruning the requests + // of bloat should have meaningful impact + // Further validation in this area is a must + + let { operator, field, type, value, externalType, onEmptyFilter } = expression + const queryOperator = operator as SearchFilterOperator + const isHbs = + typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0 + // Parse all values into correct types + if (operator === "allOr") { + query.allOr = true + return + } + if (onEmptyFilter) { + query.onEmptyFilter = onEmptyFilter + return + } + if ( + type === "datetime" && + !isHbs && + queryOperator !== "empty" && + queryOperator !== "notEmpty" + ) { + // Ensure date value is a valid date and parse into correct format + if (!value) { + return + } + try { + value = new Date(value).toISOString() + } catch (error) { + return + } + } + if (type === "number" && typeof value === "string" && !isHbs) { + if (queryOperator === "oneOf") { + value = value.split(",").map(item => parseFloat(item)) + } else { + value = parseFloat(value) + } + } + if (type === "boolean") { + value = `${value}`?.toLowerCase() === "true" + } + if ( + ["contains", "notContains", "containsAny"].includes(operator) && + type === "array" && + typeof value === "string" + ) { + value = value.split(",") + } + if (operator.startsWith("range") && query.range) { + const minint = + SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap] + ?.min || Number.MIN_SAFE_INTEGER + const maxint = + SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap] + ?.max || Number.MAX_SAFE_INTEGER + if (!query.range[field]) { + query.range[field] = { + low: type === "number" ? minint : "0000-00-00T00:00:00.000Z", + high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z", + } + } + if (operator === "rangeLow" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + low: value, + } + } else if (operator === "rangeHigh" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + high: value, + } + } + } else if (isLogicalSearchOperator(queryOperator)) { + // TODO + } else if (query[queryOperator] && operator !== "onEmptyFilter") { + if (type === "boolean") { + // Transform boolean filters to cope with null. + // "equals false" needs to be "not equals true" + // "not equals false" needs to be "equals true" + if (queryOperator === "equal" && value === false) { + query.notEqual = query.notEqual || {} + query.notEqual[field] = true + } else if (queryOperator === "notEqual" && value === false) { + query.equal = query.equal || {} + query.equal[field] = true + } else { + query[queryOperator] ??= {} + query[queryOperator]![field] = value + } + } else { + query[queryOperator] ??= {} + query[queryOperator]![field] = value + } + } + + return query +} + export const buildQuery = (filter: SearchFilter[]) => { + // let query: SearchFilters = { string: {}, fuzzy: {}, @@ -383,10 +509,36 @@ export const buildQuery = (filter: SearchFilter[]) => { } } }) - return query } +/** + * FOR TESTING - this still isnt type safe + */ +export const buildQueryX = (filter: SearchFilterGroup) => { + const operatorMap = { + [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND, + [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR, + } + + const globalOnEmpty = filter.onEmptyFilter ? filter.onEmptyFilter : null + const globalOperator = operatorMap[filter.logicalOperator] + + const coreRequest: SearchFilters = { + ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}), + [globalOperator]: { + conditions: filter.groups?.map((group: SearchFilterGroup) => { + return { + [operatorMap[group.logicalOperator]]: { + conditions: group.filters?.map(x => builderFilter(x)), //buildFilters + }, + } + }), + }, + } + return coreRequest +} + // The frontend can send single values for array fields sometimes, so to handle // this we convert them to arrays at the controller level so that nothing below // this has to worry about the non-array values. From df038417b54c9368eb346efdd801ffe70619a3a9 Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 2 Sep 2024 09:02:28 +0100 Subject: [PATCH 02/19] Merge commit --- packages/frontend-core/src/constants.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js index 22e5e8583a..05f501cfba 100644 --- a/packages/frontend-core/src/constants.js +++ b/packages/frontend-core/src/constants.js @@ -161,3 +161,13 @@ export const TypeIconMap = { export const OptionColours = [...new Array(12).keys()].map(idx => { return `hsla(${((idx + 1) * 222) % 360}, 90%, 75%, 0.3)` }) + +export const FilterOperator = { + ANY: "any", + ALL: "all", +} + +export const OnEmptyFilter = { + RETURN_ALL: "all", + RETURN_NONE: "none", +} From 7703da159d793c6a6d13047423c9e75289f6603a Mon Sep 17 00:00:00 2001 From: Dean Date: Tue, 3 Sep 2024 17:16:49 +0100 Subject: [PATCH 03/19] Merge commit --- packages/frontend-core/src/constants.js | 5 ++++ packages/shared-core/src/filters.ts | 35 +++++++++++++----------- packages/types/src/documents/app/view.ts | 4 +-- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js index bb6f4b4873..c004f72dc2 100644 --- a/packages/frontend-core/src/constants.js +++ b/packages/frontend-core/src/constants.js @@ -182,3 +182,8 @@ export const OnEmptyFilter = { RETURN_ALL: "all", RETURN_NONE: "none", } + +export const FilterValueType = { + BINDING: "Binding", + VALUE: "Value", +} diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 1cad32e790..82ef6759d2 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -27,7 +27,6 @@ import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" import { deepGet, schema } from "./helpers" import { isPlainObject, isEmpty } from "lodash" import { decodeNonAscii } from "./helpers/schema" -// import { Constants } from "@budibase/frontend-core" const HBS_REGEX = /{{([^{].*?)}}/g @@ -293,20 +292,21 @@ export class ColumnSplitter { const builderFilter = (expression: SearchFilter) => { // Filter body let query: SearchFilters = { - // string: {}, - // fuzzy: {}, - // range: {}, - // equal: {}, - // notEqual: {}, - // empty: {}, - // notEmpty: {}, - // contains: {}, - // notContains: {}, - // oneOf: {}, - // containsAny: {}, + string: {}, + fuzzy: {}, + range: {}, + equal: {}, + notEqual: {}, + empty: {}, + notEmpty: {}, + contains: {}, + notContains: {}, + oneOf: {}, + containsAny: {}, } // DEAN - + // This is the chattiest service we have, pruning the requests // of bloat should have meaningful impact // Further validation in this area is a must @@ -427,6 +427,7 @@ export const buildQuery = (filter: SearchFilter[]) => { return query } + // DEAN Replace with builderFilter filter.forEach(expression => { let { operator, field, type, value, externalType, onEmptyFilter } = expression @@ -527,10 +528,12 @@ export const buildQuery = (filter: SearchFilter[]) => { return query } -/** - * FOR TESTING - this still isnt type safe - */ +// Grouped query export const buildQueryX = (filter: SearchFilterGroup) => { + if (!filter) { + return {} + } + const operatorMap = { [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND, [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR, @@ -545,7 +548,7 @@ export const buildQueryX = (filter: SearchFilterGroup) => { conditions: filter.groups?.map((group: SearchFilterGroup) => { return { [operatorMap[group.logicalOperator]]: { - conditions: group.filters?.map(x => builderFilter(x)), //buildFilters + conditions: group.filters?.map(x => builderFilter(x)), }, } }), diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index 24dad0bcca..d02e5af1b1 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -1,4 +1,4 @@ -import { SearchFilter, SortOrder, SortType } from "../../api" +import { SearchFilter, SearchFilterGroup, SortOrder, SortType } from "../../api" import { UIFieldMetadata } from "./table" import { Document } from "../document" import { DBView } from "../../sdk" @@ -56,7 +56,7 @@ export interface ViewV2 { name: string primaryDisplay?: string tableId: string - query?: SearchFilter[] + query?: SearchFilter[] | SearchFilterGroup sort?: { field: string order?: SortOrder From 11b146fcbf0a39e344e859cb63ec6840f15de15d Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 9 Sep 2024 16:36:31 +0100 Subject: [PATCH 04/19] Updates to filter UI and API requests across budibase --- packages/backend-core/src/users/users.ts | 29 - .../SetupPanel/AutomationBlockSetup.svelte | 36 +- .../automation/SetupPanel/RowSelector.svelte | 10 +- .../buttons/TableFilterButton.svelte | 35 +- .../FilterEditor/FilterBuilder.svelte | 83 +-- .../controls/FilterEditor/FilterEditor.svelte | 40 +- .../GridColumnConfiguration.svelte | 30 +- .../controls/TableConditionEditor.svelte | 5 +- .../builder/src/stores/builder/components.js | 13 +- .../builder/src/stores/portal/featureFlags.js | 14 + packages/builder/src/stores/portal/index.js | 1 + packages/client/manifest.json | 3 +- .../src/components/app/DataProvider.svelte | 31 +- .../app/dynamic-filter/DynamicFilter.svelte | 40 +- .../app/dynamic-filter/FilterModal.svelte | 14 - .../src/components/CoreFilterBuilder.svelte | 505 ++++++++++++++++++ .../src/components/FilterBuilder.svelte | 379 ------------- .../src/components/FilterField.svelte | 318 +++++++++++ .../src/components/FilterUsers.svelte | 3 +- .../src/components/grid/stores/filter.js | 18 +- .../frontend-core/src/components/index.js | 2 +- .../frontend-core/src/fetch/CustomFetch.js | 8 + packages/frontend-core/src/fetch/DataFetch.js | 13 +- packages/frontend-core/src/fetch/UserFetch.js | 12 +- .../frontend-core/src/fetch/ViewV2Fetch.js | 6 +- packages/frontend-core/src/utils/utils.js | 137 ++--- .../server/src/api/controllers/row/views.ts | 13 +- packages/shared-core/src/filters.ts | 177 +----- packages/shared-core/src/utils.ts | 135 +++++ packages/types/src/api/web/searchFilter.ts | 13 +- packages/types/src/sdk/search.ts | 5 + .../src/api/controllers/global/users.ts | 4 +- 32 files changed, 1300 insertions(+), 832 deletions(-) create mode 100644 packages/builder/src/stores/portal/featureFlags.js delete mode 100644 packages/client/src/components/app/dynamic-filter/FilterModal.svelte create mode 100644 packages/frontend-core/src/components/CoreFilterBuilder.svelte delete mode 100644 packages/frontend-core/src/components/FilterBuilder.svelte create mode 100644 packages/frontend-core/src/components/FilterField.svelte diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 0c994d8287..d8546afa8b 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -17,11 +17,8 @@ import { ContextUser, CouchFindOptions, DatabaseQueryOpts, - SearchFilters, SearchUsersRequest, User, - BasicOperator, - ArrayOperator, } from "@budibase/types" import * as context from "../context" import { getGlobalDB } from "../context" @@ -45,32 +42,6 @@ function removeUserPassword(users: User | User[]) { return users } -export function isSupportedUserSearch(query: SearchFilters) { - const allowed = [ - { op: BasicOperator.STRING, key: "email" }, - { op: BasicOperator.EQUAL, key: "_id" }, - { op: ArrayOperator.ONE_OF, key: "_id" }, - ] - for (let [key, operation] of Object.entries(query)) { - if (typeof operation !== "object") { - return false - } - const fields = Object.keys(operation || {}) - // this filter doesn't contain options - ignore - if (fields.length === 0) { - continue - } - const allowedOperation = allowed.find( - allow => - allow.op === key && fields.length === 1 && fields[0] === allow.key - ) - if (!allowedOperation) { - return false - } - } - return true -} - export async function bulkGetGlobalUsersById( userIds: string[], opts?: GetOpts diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 6c4865b539..577f4a3f90 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -62,6 +62,7 @@ } from "@budibase/types" import { FIELDS } from "constants/backend" import PropField from "./PropField.svelte" + import { utils } from "@budibase/shared-core" export let block export let testData @@ -95,8 +96,14 @@ $: memoEnvVariables.set($environment.variables) $: memoBlock.set(block) - $: filters = lookForFilters(schemaProperties) || [] - $: tempFilters = filters + $: filters = lookForFilters(schemaProperties) + $: filterCount = + filters?.groups?.reduce((acc, group) => { + acc = acc += group?.filters?.length || 0 + return acc + }, 0) || 0 + + $: tempFilters = cloneDeep(filters) $: stepId = $memoBlock.stepId $: automationBindings = getAvailableBindings( @@ -768,14 +775,13 @@ break } } - return filters || [] + return utils.processSearchFilters(filters) } function saveFilters(key) { - const filters = QueryUtils.buildQuery(tempFilters) - + const query = QueryUtils.buildQuery(tempFilters) onChange({ - [key]: filters, + [key]: query, [`${key}-def`]: tempFilters, // need to store the builder definition in the automation }) @@ -1004,18 +1010,24 @@ {:else if value.customType === AutomationCustomIOType.FILTERS || value.customType === AutomationCustomIOType.TRIGGER_FILTER} - {filters.length > 0 - ? "Update Filter" - : "No Filter set"} + {filterCount > 0 ? "Update Filter" : "No Filter set"} + + { + tempFilters = filters + }} > - + { + return Array.isArray(fieldValue) ? fieldValue.join(",") : fieldValue + } {#each schemaFields || [] as [field, schema]} @@ -257,7 +265,7 @@ panel={AutomationBindingPanel} type={schema.type} {schema} - value={editableRow[field]} + value={drawerValue(editableRow[field])} on:change={e => onChange({ row: { diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte index 91ec9a9b01..6c6ce8c56a 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte @@ -5,6 +5,7 @@ import { getUserBindings } from "dataBinding" import { makePropSafe } from "@budibase/string-templates" import { search } from "@budibase/frontend-core" + import { utils } from "@budibase/shared-core" import { tables } from "stores/builder" export let schema @@ -16,15 +17,19 @@ let drawer - $: tempValue = filters || [] + $: localFilters = utils.processSearchFilters(filters) + $: schemaFields = search.getFields( $tables.list, Object.values(schema || {}), { allowLinks: true } ) - $: text = getText(filters) - $: selected = tempValue.filter(x => !x.onEmptyFilter)?.length > 0 + $: filterCount = + localFilters?.groups?.reduce((acc, group) => { + return (acc += group.filters.filter(filter => filter.field).length) + }, 0) || 0 + $: bindings = [ { type: "context", @@ -38,28 +43,32 @@ }, ...getUserBindings(), ] - const getText = filters => { - const count = filters?.filter(filter => filter.field)?.length - return count ? `Filter (${count})` : "Filter" - } - - {text} + 0} +> + {filterCount ? `Filter (${filterCount})` : "Filter"} { + localFilters = utils.processSearchFilters(filters) + }} forceModal > (tempValue = e.detail)} + on:change={e => (localFilters = e.detail)} {bindings} /> diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte index 64e93675d9..1e79f61bae 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte @@ -1,87 +1,32 @@ - -
- - { - const indexToUpdate = rawFilters.findIndex(f => f.id === filter.id) - rawFilters[indexToUpdate] = { - ...rawFilters[indexToUpdate], - value: event.detail, - } - }} - /> - + {bindings} + on:change +/> diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte index ed5e36cd65..d7b0c07b26 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte @@ -5,9 +5,14 @@ Button, Drawer, DrawerContent, + Helpers, } from "@budibase/bbui" import { createEventDispatcher } from "svelte" - import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding" + import { + getDatasourceForProvider, + getSchemaForDatasource, + readableToRuntimeBinding, + } from "dataBinding" import FilterBuilder from "./FilterBuilder.svelte" import { tables, selectedScreen } from "stores/builder" import { search } from "@budibase/frontend-core" @@ -21,7 +26,7 @@ let drawer - $: tempValue = value + $: localFilters = Helpers.cloneDeep(value) $: datasource = getDatasourceForProvider($selectedScreen, componentInstance) $: dsSchema = getSchemaForDatasource($selectedScreen, datasource)?.schema $: schemaFields = search.getFields( @@ -29,19 +34,24 @@ Object.values(schema || dsSchema || {}), { allowLinks: true } ) - $: text = getText(value?.filter(filter => filter.field)) + + $: text = getText(value?.groups) async function saveFilter() { - dispatch("change", tempValue) + dispatch("change", localFilters) notifications.success("Filters saved") drawer.hide() } - const getText = filters => { - if (!filters?.length) { + const getText = (filterGroups = []) => { + const allFilters = filterGroups.reduce((acc, group) => { + return (acc += group.filters.filter(filter => filter.field).length) + }, 0) + + if (allFilters === 0) { return "No filters set" } else { - return `${filters.length} filter${filters.length === 1 ? "" : "s"} set` + return `${allFilters} filter${allFilters === 1 ? "" : "s"} set` } } @@ -49,15 +59,25 @@
{text}
- + { + // Reset to the currently available value. + localFilters = Helpers.cloneDeep(value) + }} +> (tempValue = e.detail)} + on:change={e => { + localFilters = e.detail + }} /> diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte index 17cb171da5..eae26348fd 100644 --- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte +++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte @@ -6,6 +6,7 @@ import FieldSetting from "./FieldSetting.svelte" import PrimaryColumnFieldSetting from "./PrimaryColumnFieldSetting.svelte" import getColumns from "./getColumns.js" + import InfoDisplay from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte" export let value export let componentInstance @@ -58,16 +59,25 @@
{/if} - columns.updateSortable(e.detail)} - on:itemChange={e => columns.update(e.detail)} - items={columns.sortable} - listItemKey={"_id"} - listType={FieldSetting} - listTypeProps={{ - bindings, - }} -/> + +{#if columns?.sortable?.length} + columns.updateSortable(e.detail)} + on:itemChange={e => columns.update(e.detail)} + items={columns.sortable} + listItemKey={"_id"} + listType={FieldSetting} + listTypeProps={{ + bindings, + }} + /> +{:else} + +{/if} diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte deleted file mode 100644 index 3a0c789b9e..0000000000 --- a/packages/frontend-core/src/components/FilterBuilder.svelte +++ /dev/null @@ -1,379 +0,0 @@ - - -
- - {#if fieldOptions?.length} - - {#if !fieldFilters?.length} - Add your first filter expression. - {:else} - - {#if behaviourFilters} -
- opt.label} - getOptionValue={opt => opt.value} - on:change={e => handleOnEmptyFilter(e.detail)} - placeholder={null} - /> - {/if} -
- {/if} - {/if} - - {#if fieldFilters?.length} -
- {#if filtersLabel} -
- -
- {/if} -
- {#each fieldFilters as filter} - onOperatorChange(filter)} - placeholder={null} - /> - {#if allowBindings} - - {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === ArrayOperator.ONE_OF)} - - {:else if filter.type === FieldType.OPTIONS} - - {:else if filter.type === FieldType.BOOLEAN} - - {:else if filter.type === FieldType.DATETIME} - - {:else if [FieldType.BB_REFERENCE, FieldType.BB_REFERENCE_SINGLE].includes(filter.type)} - - {:else} - - {/if} -
- duplicateFilter(filter.id)} - /> - removeFilter(filter.id)} - /> -
- {/each} -
-
- {/if} -
- -
- {:else} - None of the table column can be used for filtering. - {/if} - -
- - diff --git a/packages/frontend-core/src/components/FilterField.svelte b/packages/frontend-core/src/components/FilterField.svelte new file mode 100644 index 0000000000..03a09c9db0 --- /dev/null +++ b/packages/frontend-core/src/components/FilterField.svelte @@ -0,0 +1,318 @@ + + +
+ + + + + + +
+
+ {#if filter.valueType === FilterValueType.BINDING} + + {:else} +
+ {#if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)} + + {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === ArrayOperator.ONE_OF)} + + {:else if filter.type === FieldType.OPTIONS} + + {:else if filter.type === FieldType.BOOLEAN} + + {:else if filter.type === FieldType.DATETIME} + + {:else if [FieldType.BB_REFERENCE, FieldType.BB_REFERENCE_SINGLE].includes(filter.type)} + + {:else} + + {/if} +
+ {/if} +
+ +
+ + {#if !disabled && allowBindings && filter.field && !filter.noValue} + + +
{ + bindingDrawer.show() + }} + > + +
+ {/if} +
+
+
+ + diff --git a/packages/frontend-core/src/components/FilterUsers.svelte b/packages/frontend-core/src/components/FilterUsers.svelte index 489426df1e..4640561afd 100644 --- a/packages/frontend-core/src/components/FilterUsers.svelte +++ b/packages/frontend-core/src/components/FilterUsers.svelte @@ -27,7 +27,8 @@
option.email} diff --git a/packages/frontend-core/src/components/grid/stores/filter.js b/packages/frontend-core/src/components/grid/stores/filter.js index a16b101bbb..f5feee9c35 100644 --- a/packages/frontend-core/src/components/grid/stores/filter.js +++ b/packages/frontend-core/src/components/grid/stores/filter.js @@ -1,5 +1,5 @@ import { writable, get, derived } from "svelte/store" -import { FieldType } from "@budibase/types" +import { FieldType, FilterGroupLogicalOperator } from "@budibase/types" export const createStores = context => { const { props } = context @@ -16,11 +16,22 @@ export const createStores = context => { export const deriveStores = context => { const { filter, inlineFilters } = context - const allFilters = derived( [filter, inlineFilters], ([$filter, $inlineFilters]) => { - return [...($filter || []), ...$inlineFilters] + const inlineFilterGroup = $inlineFilters?.length + ? { + logicalOperator: FilterGroupLogicalOperator.ALL, + filters: [...($inlineFilters || [])], + } + : null + + return inlineFilterGroup + ? { + logicalOperator: FilterGroupLogicalOperator.ALL, + groups: [...($filter?.groups || []), inlineFilterGroup], + } + : $filter } ) @@ -54,7 +65,6 @@ export const createActions = context => { inlineFilter.operator = "contains" } - // Add this filter inlineFilters.update($inlineFilters => { // Remove any existing inline filter for this column $inlineFilters = $inlineFilters?.filter(x => x.id !== filterId) diff --git a/packages/frontend-core/src/components/index.js b/packages/frontend-core/src/components/index.js index d494abb82d..0557ec080e 100644 --- a/packages/frontend-core/src/components/index.js +++ b/packages/frontend-core/src/components/index.js @@ -7,5 +7,5 @@ export { default as UserAvatars } from "./UserAvatars.svelte" export { default as Updating } from "./Updating.svelte" export { Grid } from "./grid" export { default as ClientAppSkeleton } from "./ClientAppSkeleton.svelte" -export { default as FilterBuilder } from "./FilterBuilder.svelte" +export { default as CoreFilterBuilder } from "./CoreFilterBuilder.svelte" export { default as FilterUsers } from "./FilterUsers.svelte" diff --git a/packages/frontend-core/src/fetch/CustomFetch.js b/packages/frontend-core/src/fetch/CustomFetch.js index fc62d790e2..a8b14a0809 100644 --- a/packages/frontend-core/src/fetch/CustomFetch.js +++ b/packages/frontend-core/src/fetch/CustomFetch.js @@ -1,6 +1,14 @@ import DataFetch from "./DataFetch.js" export default class CustomFetch extends DataFetch { + determineFeatureFlags() { + return { + supportsSearch: false, + supportsSort: false, + supportsPagination: false, + } + } + // Gets the correct Budibase type for a JS value getType(value) { if (value == null) { diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js index dedd06264c..1743eef71e 100644 --- a/packages/frontend-core/src/fetch/DataFetch.js +++ b/packages/frontend-core/src/fetch/DataFetch.js @@ -2,6 +2,7 @@ import { writable, derived, get } from "svelte/store" import { cloneDeep } from "lodash/fp" import { QueryUtils } from "../utils" import { convertJSONSchemaToTableSchema } from "../utils/json" +import { FilterGroupLogicalOperator, EmptyFilterOption } from "@budibase/types" const { buildQuery, limit: queryLimit, runQuery, sort } = QueryUtils @@ -176,10 +177,16 @@ export default class DataFetch { } } + let defaultQuery = { + logicalOperator: FilterGroupLogicalOperator.ALL, + groups: [], + } + // Build the query - let query = this.options.query - if (!query) { - query = buildQuery(filter) + let query = this.features.supportsSearch ? this.options.query : null + + if (!query && this.features.supportsSearch) { + query = buildQuery(filter || defaultQuery) } // Update store diff --git a/packages/frontend-core/src/fetch/UserFetch.js b/packages/frontend-core/src/fetch/UserFetch.js index cb2c045cc6..39a8d79214 100644 --- a/packages/frontend-core/src/fetch/UserFetch.js +++ b/packages/frontend-core/src/fetch/UserFetch.js @@ -1,7 +1,7 @@ import { get } from "svelte/store" import DataFetch from "./DataFetch.js" import { TableNames } from "../constants" -import { QueryUtils } from "../utils" +import { utils } from "@budibase/shared-core" export default class UserFetch extends DataFetch { constructor(opts) { @@ -33,11 +33,11 @@ export default class UserFetch extends DataFetch { let finalQuery // convert old format to new one - we now allow use of the lucene format const { appId, paginated, ...rest } = query - if (!QueryUtils.hasFilters(query) && rest.email != null) { - finalQuery = { string: { email: rest.email } } - } else { - finalQuery = rest - } + + finalQuery = utils.isSupportedUserSearch(rest) + ? query + : { string: { email: null } } + try { const opts = { bookmark: cursor, diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.js b/packages/frontend-core/src/fetch/ViewV2Fetch.js index 9d2f8c103a..303414b2dc 100644 --- a/packages/frontend-core/src/fetch/ViewV2Fetch.js +++ b/packages/frontend-core/src/fetch/ViewV2Fetch.js @@ -1,5 +1,6 @@ import DataFetch from "./DataFetch.js" import { get } from "svelte/store" +import { utils } from "@budibase/shared-core" export default class ViewV2Fetch extends DataFetch { determineFeatureFlags() { @@ -53,7 +54,10 @@ export default class ViewV2Fetch extends DataFetch { this.options.sortColumn = definition.sort.field this.options.sortOrder = definition.sort.order } - if (!filter?.length && definition.query?.length) { + + const parsed = utils.processSearchFilters(filter) + + if (!parsed?.groups?.length && definition.query?.groups?.length) { this.options.filter = definition.query } diff --git a/packages/frontend-core/src/utils/utils.js b/packages/frontend-core/src/utils/utils.js index 6c9acb0f89..7194aab07b 100644 --- a/packages/frontend-core/src/utils/utils.js +++ b/packages/frontend-core/src/utils/utils.js @@ -353,98 +353,57 @@ export const buildMultiStepFormBlockDefaultProps = props => { } } -/** - * Processes the filter config. Filters are migrated - * SearchFilter[] to SearchFilterGroup - * - * This is supposed to be in shared-core - * @param {Object[]} filter - */ -export const migrateSearchFilters = filters => { - const defaultCfg = { - logicalOperator: Constants.FilterOperator.ALL, - groups: [], - } +export const handleFilterChange = (req, filters) => { + const { + groupIdx, + filterIdx, + filter, + group, + addFilter, + addGroup, + deleteGroup, + deleteFilter, + logicalOperator, + onEmptyFilter, + } = req - const filterWhitelistKeys = [ - "field", - "operator", - "valueType", // bb - "value", - "type", - "noValue", // bb - ] + let editable = Helpers.cloneDeep(filters) + let targetGroup = editable.groups?.[groupIdx] + let targetFilter = targetGroup?.filters?.[filterIdx] - /** - * Review these - * externalType, formulaType, subtype - */ - - if (Array.isArray(filters)) { - const baseGroup = { - filters: [], - logicalOperator: Constants.FilterOperator.ALL, + if (targetFilter) { + if (deleteFilter) { + targetGroup.filters.splice(filterIdx, 1) + } else if (filter) { + targetGroup.filters[filterIdx] = filter } - - const migratedSetting = filters.reduce((acc, filter) => { - // Sort the properties for easier debugging - // Remove unset values - const filterEntries = Object.entries(filter) - .sort((a, b) => { - return a[0].localeCompare(b[0]) - }) - .filter(x => x[1] ?? false) - - // Scrub invalid filters - const { operator, onEmptyFilter, field, valueType } = filter - if (!field || !valueType) { - // THIS SCRUBS THE 2 GLOBALS - // return acc + } else if (targetGroup) { + if (deleteGroup) { + editable.groups.splice(groupIdx, 1) + } else if (addFilter) { + targetGroup.filters.push({}) + } else if (group) { + editable.groups[groupIdx] = { + ...targetGroup, + ...group, } - - if (filterEntries.length == 1) { - console.log("### one entry ") - const [key, value] = filterEntries[0] - // Global - if (key === "onEmptyFilter") { - // unset otherwise, seems to be the default - acc.onEmptyFilter = value - } else if (key === "operator" && value === "allOr") { - // Group 1 logical operator - baseGroup.logicalOperator = Constants.FilterOperator.ANY - } - - return acc - } - - // Process settings?? - const whiteListedFilterSettings = filterEntries.reduce((acc, entry) => { - const [key, value] = entry - - if (filterWhitelistKeys.includes(key)) { - if (key === "field") { - acc.push([key, value.replace(/^[0-9]+:/, "")]) - } else { - acc.push([key, value]) - } - } - return acc - }, []) - - const migratedFilter = Object.fromEntries(whiteListedFilterSettings) - - baseGroup.filters.push(migratedFilter) - - if (!acc.groups.length) { - // init the base group - acc.groups.push(baseGroup) - } - - return acc - }, defaultCfg) - - console.log("MIGRATED ", migratedSetting) - return migratedSetting + } + } else if (addGroup) { + editable.groups.push({ + logicalOperator: Constants.FilterOperator.ANY, + filters: [{}], + }) + } else if (logicalOperator) { + editable = { + ...editable, + logicalOperator, + } + } else if (onEmptyFilter) { + editable = { + ...editable, + onEmptyFilter, + } } - return false + + return editable } diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 06a01646a7..220e2098f7 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -7,6 +7,7 @@ import { RowSearchParams, SearchFilterKey, LogicalOperator, + SearchFilter, } from "@budibase/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../../sdk" @@ -19,7 +20,7 @@ export async function searchView( ) { const { viewId } = ctx.params - const view = await sdk.views.get(viewId) + const view: ViewV2 = await sdk.views.get(viewId) if (!view) { ctx.throw(404, `View ${viewId} not found`) } @@ -35,7 +36,7 @@ export async function searchView( // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as // that could let users find rows they should not be allowed to access. - let query = dataFilters.buildQuery(view.query || []) + let query: any = dataFilters.buildQuery(view.query ?? []) if (body.query) { // Delete extraneous search params that cannot be overridden delete body.query.onEmptyFilter @@ -44,9 +45,15 @@ export async function searchView( !isExternalTableID(view.tableId) && !(await features.flags.isEnabled("SQS")) ) { + // In the unlikely event that a Grouped Filter is in a non-SQS environment + // It needs to be ignored. Entirely + let queryFilters: SearchFilter[] = Array.isArray(view.query) + ? view.query + : [] + // Extract existing fields const existingFields = - view.query + queryFilters ?.filter(filter => filter.field) .map(filter => db.removeKeyNumbering(filter.field)) || [] diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index c71dfe9dba..6017b41d13 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -24,6 +24,7 @@ import { } from "@budibase/types" import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" +import { processSearchFilters } from "./utils" import { deepGet, schema } from "./helpers" import { isPlainObject, isEmpty } from "lodash" import { decodeNonAscii } from "./helpers/schema" @@ -299,10 +300,6 @@ export class ColumnSplitter { } } -/* - separate into buildQuery and build filter? -*/ - /** * Builds a JSON query from the filter structure generated in the builder * @param filter the builder filter structure @@ -323,13 +320,6 @@ const builderFilter = (expression: SearchFilter) => { oneOf: {}, containsAny: {}, } - - // DEAN - - - // This is the chattiest service we have, pruning the requests - // of bloat should have meaningful impact - // Further validation in this area is a must - let { operator, field, type, value, externalType, onEmptyFilter } = expression const queryOperator = operator as SearchFilterOperator const isHbs = @@ -343,6 +333,13 @@ const builderFilter = (expression: SearchFilter) => { query.onEmptyFilter = onEmptyFilter return } + + // Default the value for noValue fields to ensure they are correctly added + // to the final query + if (queryOperator === "empty" || queryOperator === "notEmpty") { + value = null + } + if ( type === "datetime" && !isHbs && @@ -426,131 +423,11 @@ const builderFilter = (expression: SearchFilter) => { return query } -export const buildQuery = (filter: SearchFilter[]) => { - // - let query: SearchFilters = { - string: {}, - fuzzy: {}, - range: {}, - equal: {}, - notEqual: {}, - empty: {}, - notEmpty: {}, - contains: {}, - notContains: {}, - oneOf: {}, - containsAny: {}, - } +export const buildQuery = (filter: SearchFilterGroup | SearchFilter[]) => { + const parsedFilter = processSearchFilters(filter) - if (!Array.isArray(filter)) { - return query - } - - // DEAN Replace with builderFilter - filter.forEach(expression => { - let { operator, field, type, value, externalType, onEmptyFilter } = - expression - const queryOperator = operator as SearchFilterOperator - const isHbs = - typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0 - // Parse all values into correct types - if (operator === "allOr") { - query.allOr = true - return - } - if (onEmptyFilter) { - query.onEmptyFilter = onEmptyFilter - return - } - if ( - type === "datetime" && - !isHbs && - queryOperator !== "empty" && - queryOperator !== "notEmpty" - ) { - // Ensure date value is a valid date and parse into correct format - if (!value) { - return - } - try { - value = new Date(value).toISOString() - } catch (error) { - return - } - } - if (type === "number" && typeof value === "string" && !isHbs) { - if (queryOperator === "oneOf") { - value = value.split(",").map(item => parseFloat(item)) - } else { - value = parseFloat(value) - } - } - if (type === "boolean") { - value = `${value}`?.toLowerCase() === "true" - } - if ( - ["contains", "notContains", "containsAny"].includes(operator) && - type === "array" && - typeof value === "string" - ) { - value = value.split(",") - } - if (operator.startsWith("range") && query.range) { - const minint = - SqlNumberTypeRangeMap[ - externalType as keyof typeof SqlNumberTypeRangeMap - ]?.min || Number.MIN_SAFE_INTEGER - const maxint = - SqlNumberTypeRangeMap[ - externalType as keyof typeof SqlNumberTypeRangeMap - ]?.max || Number.MAX_SAFE_INTEGER - if (!query.range[field]) { - query.range[field] = { - low: type === "number" ? minint : "0000-00-00T00:00:00.000Z", - high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z", - } - } - if (operator === "rangeLow" && value != null && value !== "") { - query.range[field] = { - ...query.range[field], - low: value, - } - } else if (operator === "rangeHigh" && value != null && value !== "") { - query.range[field] = { - ...query.range[field], - high: value, - } - } - } else if (isLogicalSearchOperator(queryOperator)) { - // TODO - } else if (query[queryOperator] && operator !== "onEmptyFilter") { - if (type === "boolean") { - // Transform boolean filters to cope with null. - // "equals false" needs to be "not equals true" - // "not equals false" needs to be "equals true" - if (queryOperator === "equal" && value === false) { - query.notEqual = query.notEqual || {} - query.notEqual[field] = true - } else if (queryOperator === "notEqual" && value === false) { - query.equal = query.equal || {} - query.equal[field] = true - } else { - query[queryOperator] ??= {} - query[queryOperator]![field] = value - } - } else { - query[queryOperator] ??= {} - query[queryOperator]![field] = value - } - } - }) - return query -} - -// Grouped query -export const buildQueryX = (filter: SearchFilterGroup) => { - if (!filter) { - return {} + if (!parsedFilter) { + return } const operatorMap = { @@ -558,13 +435,15 @@ export const buildQueryX = (filter: SearchFilterGroup) => { [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR, } - const globalOnEmpty = filter.onEmptyFilter ? filter.onEmptyFilter : null - const globalOperator = operatorMap[filter.logicalOperator] + const globalOnEmpty = parsedFilter.onEmptyFilter + ? parsedFilter.onEmptyFilter + : null + const globalOperator = operatorMap[parsedFilter.logicalOperator] const coreRequest: SearchFilters = { ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}), [globalOperator]: { - conditions: filter.groups?.map((group: SearchFilterGroup) => { + conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => { return { [operatorMap[group.logicalOperator]]: { conditions: group.filters?.map(x => builderFilter(x)), @@ -580,6 +459,9 @@ export const buildQueryX = (filter: SearchFilterGroup) => { // this we convert them to arrays at the controller level so that nothing below // this has to worry about the non-array values. export function fixupFilterArrays(filters: SearchFilters) { + if (!filters) { + return filters + } for (const searchField of Object.values(ArrayOperator)) { const field = filters[searchField] if (field == null || !isPlainObject(field)) { @@ -1005,14 +887,13 @@ export const hasFilters = (query?: SearchFilters) => { if (!query) { return false } - const skipped = ["allOr", "onEmptyFilter"] - for (let [key, value] of Object.entries(query)) { - if (skipped.includes(key) || typeof value !== "object") { - continue - } - if (Object.keys(value || {}).length !== 0) { - return true - } - } - return false + const queryRoot = query[LogicalOperator.AND] ?? query[LogicalOperator.OR] + + return ( + (queryRoot?.conditions || []).reduce((acc, group) => { + const groupRoot = group[LogicalOperator.AND] ?? group[LogicalOperator.OR] + acc += groupRoot?.conditions?.length || 0 + return acc + }, 0) > 0 + ) } diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index b69a059745..0824c7e8d5 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -1,4 +1,14 @@ +import { + SearchFilter, + SearchFilterGroup, + FilterGroupLogicalOperator, + EmptyFilterOption, + SearchFilters, + BasicOperator, + ArrayOperator, +} from "@budibase/types" import * as Constants from "./constants" +import { removeKeyNumbering } from "./filters" export function unreachable( value: never, @@ -77,3 +87,128 @@ export function trimOtherProps(object: any, allowedProps: string[]) { ) return result } + +/** + * Processes the filter config. Filters are migrated from + * SearchFilter[] to SearchFilterGroup + * + * If filters is not an array, the migration is skipped + * + * @param {SearchFilter[] | SearchFilterGroup} filters + */ +export const processSearchFilters = ( + filters: SearchFilter[] | SearchFilterGroup +) => { + if (!filters) { + return + } + const defaultCfg: SearchFilterGroup = { + logicalOperator: FilterGroupLogicalOperator.ALL, + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + groups: [], + } + + const filterWhitelistKeys = [ + "field", + "operator", + "value", + "type", + "externalType", + "valueType", + "noValue", + "formulaType", + ] + + if (Array.isArray(filters)) { + let baseGroup: SearchFilterGroup = { + filters: [], + logicalOperator: FilterGroupLogicalOperator.ALL, + } + + const migratedSetting: SearchFilterGroup = filters.reduce( + (acc: SearchFilterGroup, filter: SearchFilter) => { + // Sort the properties for easier debugging + const filterEntries = Object.entries(filter) + .sort((a, b) => { + return a[0].localeCompare(b[0]) + }) + .filter(x => x[1] ?? false) + + if (filterEntries.length == 1) { + const [key, value] = filterEntries[0] + // Global + if (key === "onEmptyFilter") { + // unset otherwise + acc.onEmptyFilter = value + } else if (key === "operator" && value === "allOr") { + // Group 1 logical operator + baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY + } + + return acc + } + + const whiteListedFilterSettings: [string, any][] = filterEntries.reduce( + (acc: [string, any][], entry: [string, any]) => { + const [key, value] = entry + + if (filterWhitelistKeys.includes(key)) { + if (key === "field") { + acc.push([key, removeKeyNumbering(value)]) + } else { + acc.push([key, value]) + } + } + return acc + }, + [] + ) + + const migratedFilter: SearchFilter = Object.fromEntries( + whiteListedFilterSettings + ) as SearchFilter + + baseGroup.filters!.push(migratedFilter) + + if (!acc.groups || !acc.groups.length) { + // init the base group + acc.groups = [baseGroup] + } + + return acc + }, + defaultCfg + ) + + return migratedSetting + } else if (!filters?.groups) { + return defaultCfg + } + return filters +} + +export function isSupportedUserSearch(query: SearchFilters) { + const allowed = [ + { op: BasicOperator.STRING, key: "email" }, + { op: BasicOperator.EQUAL, key: "_id" }, + { op: ArrayOperator.ONE_OF, key: "_id" }, + ] + for (let [key, operation] of Object.entries(query)) { + if (typeof operation !== "object") { + return false + } + const fields = Object.keys(operation || {}) + // this filter doesn't contain options - ignore + if (fields.length === 0) { + continue + } + const allowedOperation = allowed.find( + allow => + allow.op === key && fields.length === 1 && fields[0] === allow.key + ) + if (!allowedOperation) { + return false + } + } + return true +} diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts index 5223204a7f..b3d577f0c8 100644 --- a/packages/types/src/api/web/searchFilter.ts +++ b/packages/types/src/api/web/searchFilter.ts @@ -1,5 +1,9 @@ import { FieldType } from "../../documents" -import { EmptyFilterOption, SearchFilters } from "../../sdk" +import { + EmptyFilterOption, + FilterGroupLogicalOperator, + SearchFilters, +} from "../../sdk" export type SearchFilter = { operator: keyof SearchFilters | "rangeLow" | "rangeHigh" @@ -9,3 +13,10 @@ export type SearchFilter = { value: any externalType?: string } + +export type SearchFilterGroup = { + logicalOperator: FilterGroupLogicalOperator + onEmptyFilter?: EmptyFilterOption + groups?: SearchFilterGroup[] + filters?: SearchFilter[] +} diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 6feea40766..4a98fe5bae 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -180,6 +180,11 @@ export enum EmptyFilterOption { RETURN_NONE = "none", } +export enum FilterGroupLogicalOperator { + ALL = "all", + ANY = "any", +} + export enum SqlClient { MS_SQL = "mssql", POSTGRES = "pg", diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 6ce0eef5a0..921e0324d1 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -37,7 +37,7 @@ import { } from "@budibase/backend-core" import { checkAnyUserExists } from "../../../utilities/users" import { isEmailConfigured } from "../../../utilities/email" -import { BpmStatusKey, BpmStatusValue } from "@budibase/shared-core" +import { BpmStatusKey, BpmStatusValue, utils } from "@budibase/shared-core" const MAX_USERS_UPLOAD_LIMIT = 1000 @@ -256,7 +256,7 @@ export const search = async (ctx: Ctx) => { } } // Validate we aren't trying to search on any illegal fields - if (!userSdk.core.isSupportedUserSearch(body.query)) { + if (!utils.isSupportedUserSearch(body.query)) { ctx.throw(400, "Can only search by string.email, equal._id or oneOf._id") } } From ca5891b88bd4ff535c5c527cfef72f4b0bb3ad45 Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 9 Sep 2024 16:48:32 +0100 Subject: [PATCH 05/19] Lint --- .../settings/controls/FilterEditor/FilterEditor.svelte | 6 +----- packages/frontend-core/src/fetch/DataFetch.js | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte index d7b0c07b26..d6f1732e64 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte @@ -8,11 +8,7 @@ Helpers, } from "@budibase/bbui" import { createEventDispatcher } from "svelte" - import { - getDatasourceForProvider, - getSchemaForDatasource, - readableToRuntimeBinding, - } from "dataBinding" + import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding" import FilterBuilder from "./FilterBuilder.svelte" import { tables, selectedScreen } from "stores/builder" import { search } from "@budibase/frontend-core" diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js index 1743eef71e..545fc03c67 100644 --- a/packages/frontend-core/src/fetch/DataFetch.js +++ b/packages/frontend-core/src/fetch/DataFetch.js @@ -2,7 +2,7 @@ import { writable, derived, get } from "svelte/store" import { cloneDeep } from "lodash/fp" import { QueryUtils } from "../utils" import { convertJSONSchemaToTableSchema } from "../utils/json" -import { FilterGroupLogicalOperator, EmptyFilterOption } from "@budibase/types" +import { FilterGroupLogicalOperator } from "@budibase/types" const { buildQuery, limit: queryLimit, runQuery, sort } = QueryUtils From 162d0d0c7ee9ea4965659e27ba8aab94d74f63a7 Mon Sep 17 00:00:00 2001 From: Dean Date: Tue, 10 Sep 2024 09:08:43 +0100 Subject: [PATCH 06/19] Clean up dead code and improve behaviour when emptying filter config --- .../src/components/CoreFilterBuilder.svelte | 27 +++++---- packages/frontend-core/src/fetch/DataFetch.js | 2 +- packages/frontend-core/src/utils/utils.js | 56 ------------------- packages/shared-core/src/filters.ts | 9 ++- packages/shared-core/src/utils.ts | 2 +- 5 files changed, 27 insertions(+), 69 deletions(-) diff --git a/packages/frontend-core/src/components/CoreFilterBuilder.svelte b/packages/frontend-core/src/components/CoreFilterBuilder.svelte index c078901880..839b370eca 100644 --- a/packages/frontend-core/src/components/CoreFilterBuilder.svelte +++ b/packages/frontend-core/src/components/CoreFilterBuilder.svelte @@ -39,13 +39,13 @@ export let toReadable export let toRuntime - $: editableFilters = filters - ? Helpers.cloneDeep(filters) - : { - logicalOperator: FilterGroupLogicalOperator.ALL, - onEmptyFilter: EmptyFilterOption.RETURN_NONE, - groups: [], - } + const defaultConf = { + logicalOperator: FilterGroupLogicalOperator.ALL, + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + groups: [], + } + + $: editableFilters = filters ? Helpers.cloneDeep(filters) : null $: { if ( @@ -191,7 +191,7 @@ } = req let editable = Helpers.cloneDeep(editableFilters) - let targetGroup = editable.groups?.[groupIdx] + let targetGroup = editable?.groups?.[groupIdx] let targetFilter = targetGroup?.filters?.[filterIdx] if (targetFilter) { @@ -202,7 +202,11 @@ } } else if (targetGroup) { if (deleteGroup) { - editable.groups.splice(groupIdx, 1) + if (editable.groups.length > 1) { + editable.groups.splice(groupIdx, 1) + } else { + editable = {} + } } else if (addFilter) { targetGroup.filters.push({ valueType: FilterValueType.VALUE, @@ -214,6 +218,9 @@ } } } else if (addGroup) { + if (!editable) { + editable = defaultConf + } editable.groups.push({ logicalOperator: Constants.FilterOperator.ANY, filters: [ @@ -381,7 +388,7 @@ {/if}
- {#if behaviourFilters && editableFilters?.groups.length} + {#if behaviourFilters && editableFilters?.groups?.length}
Return diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js index 545fc03c67..18dc72bf24 100644 --- a/packages/frontend-core/src/fetch/DataFetch.js +++ b/packages/frontend-core/src/fetch/DataFetch.js @@ -183,7 +183,7 @@ export default class DataFetch { } // Build the query - let query = this.features.supportsSearch ? this.options.query : null + let query = this.options.query || null if (!query && this.features.supportsSearch) { query = buildQuery(filter || defaultQuery) diff --git a/packages/frontend-core/src/utils/utils.js b/packages/frontend-core/src/utils/utils.js index 7194aab07b..3e8cdcc781 100644 --- a/packages/frontend-core/src/utils/utils.js +++ b/packages/frontend-core/src/utils/utils.js @@ -1,6 +1,5 @@ import { makePropSafe as safe } from "@budibase/string-templates" import { Helpers } from "@budibase/bbui" -import * as Constants from "../constants" export const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)) @@ -352,58 +351,3 @@ export const buildMultiStepFormBlockDefaultProps = props => { title, } } - -export const handleFilterChange = (req, filters) => { - const { - groupIdx, - filterIdx, - filter, - group, - addFilter, - addGroup, - deleteGroup, - deleteFilter, - logicalOperator, - onEmptyFilter, - } = req - - let editable = Helpers.cloneDeep(filters) - let targetGroup = editable.groups?.[groupIdx] - let targetFilter = targetGroup?.filters?.[filterIdx] - - if (targetFilter) { - if (deleteFilter) { - targetGroup.filters.splice(filterIdx, 1) - } else if (filter) { - targetGroup.filters[filterIdx] = filter - } - } else if (targetGroup) { - if (deleteGroup) { - editable.groups.splice(groupIdx, 1) - } else if (addFilter) { - targetGroup.filters.push({}) - } else if (group) { - editable.groups[groupIdx] = { - ...targetGroup, - ...group, - } - } - } else if (addGroup) { - editable.groups.push({ - logicalOperator: Constants.FilterOperator.ANY, - filters: [{}], - }) - } else if (logicalOperator) { - editable = { - ...editable, - logicalOperator, - } - } else if (onEmptyFilter) { - editable = { - ...editable, - onEmptyFilter, - } - } - - return editable -} diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index ded45746a9..4bb3a6a7df 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -321,6 +321,11 @@ const builderFilter = (expression: SearchFilter) => { containsAny: {}, } let { operator, field, type, value, externalType, onEmptyFilter } = expression + + if (!operator || !field) { + return + } + const queryOperator = operator as SearchFilterOperator const isHbs = typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0 @@ -446,7 +451,9 @@ export const buildQuery = (filter: SearchFilterGroup | SearchFilter[]) => { conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => { return { [operatorMap[group.logicalOperator]]: { - conditions: group.filters?.map(x => builderFilter(x)), + conditions: group.filters + ?.map(x => builderFilter(x)) + .filter(filter => filter), }, } }), diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index 0824c7e8d5..027db8e914 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -182,7 +182,7 @@ export const processSearchFilters = ( return migratedSetting } else if (!filters?.groups) { - return defaultCfg + return null } return filters } From e389bd8121bdb147c38901e1b5f9b969cfab960e Mon Sep 17 00:00:00 2001 From: Dean Date: Tue, 10 Sep 2024 09:51:34 +0100 Subject: [PATCH 07/19] Fix for resetting the default config when the groups are cleared --- .../src/components/CoreFilterBuilder.svelte | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/frontend-core/src/components/CoreFilterBuilder.svelte b/packages/frontend-core/src/components/CoreFilterBuilder.svelte index 839b370eca..405373e0a7 100644 --- a/packages/frontend-core/src/components/CoreFilterBuilder.svelte +++ b/packages/frontend-core/src/components/CoreFilterBuilder.svelte @@ -39,12 +39,6 @@ export let toReadable export let toRuntime - const defaultConf = { - logicalOperator: FilterGroupLogicalOperator.ALL, - onEmptyFilter: EmptyFilterOption.RETURN_NONE, - groups: [], - } - $: editableFilters = filters ? Helpers.cloneDeep(filters) : null $: { @@ -218,8 +212,12 @@ } } } else if (addGroup) { - if (!editable) { - editable = defaultConf + if (!editable?.groups?.length) { + editable = { + logicalOperator: FilterGroupLogicalOperator.ALL, + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + groups: [], + } } editable.groups.push({ logicalOperator: Constants.FilterOperator.ANY, From 5cc8433773a2512f57ad76a6854fae9564042477 Mon Sep 17 00:00:00 2001 From: Dean Date: Tue, 10 Sep 2024 11:05:49 +0100 Subject: [PATCH 08/19] Builder test fixes --- packages/builder/src/stores/builder/components.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js index 9cc5bf6c40..0250ed6556 100644 --- a/packages/builder/src/stores/builder/components.js +++ b/packages/builder/src/stores/builder/components.js @@ -202,10 +202,10 @@ export class ComponentStore extends BudiStore { } const def = this.getDefinition(enrichedComponent?._component) - const filterableTypes = def.settings.filter(setting => + const filterableTypes = def?.settings?.filter(setting => setting?.type?.startsWith("filter") ) - for (let setting of filterableTypes) { + for (let setting of filterableTypes || []) { enrichedComponent[setting.key] = utils.processSearchFilters( enrichedComponent[setting.key] ) From f9e8689ce9cddb14fca5ad18ef142a9f5970ca0e Mon Sep 17 00:00:00 2001 From: Dean Date: Tue, 10 Sep 2024 12:19:41 +0100 Subject: [PATCH 09/19] Test fixes for view search --- packages/server/src/api/controllers/row/views.ts | 2 +- packages/shared-core/src/filters.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 220e2098f7..8a1e5e08ce 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -36,7 +36,7 @@ export async function searchView( // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as // that could let users find rows they should not be allowed to access. - let query: any = dataFilters.buildQuery(view.query ?? []) + let query: any = dataFilters.buildQuery(view.query) if (body.query) { // Delete extraneous search params that cannot be overridden delete body.query.onEmptyFilter diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 4bb3a6a7df..79c95380e2 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -428,7 +428,11 @@ const builderFilter = (expression: SearchFilter) => { return query } -export const buildQuery = (filter: SearchFilterGroup | SearchFilter[]) => { +export const buildQuery = (filter?: SearchFilterGroup | SearchFilter[]) => { + if (!filter) { + return + } + const parsedFilter = processSearchFilters(filter) if (!parsedFilter) { From 9ea74dcb453d348925d6763a0df9ed90f6c5cb48 Mon Sep 17 00:00:00 2001 From: Dean Date: Wed, 11 Sep 2024 11:56:09 +0100 Subject: [PATCH 10/19] Updates for view tests --- .../server/src/api/controllers/row/views.ts | 19 +-- .../server/src/sdk/app/rows/queryUtils.ts | 6 +- packages/shared-core/src/filters.ts | 136 +++++++++++++++++- 3 files changed, 150 insertions(+), 11 deletions(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 8a1e5e08ce..5957531857 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -33,23 +33,26 @@ export async function searchView( .map(([key]) => key) const { body } = ctx.request + const sqsEnabled = await features.flags.isEnabled("SQS") + const supportsLogicalOperators = isExternalTableID(view.tableId) || sqsEnabled + // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as // that could let users find rows they should not be allowed to access. - let query: any = dataFilters.buildQuery(view.query) + let query: any = supportsLogicalOperators + ? dataFilters.buildQuery(view.query) + : dataFilters.buildQueryLegacy(view.query as SearchFilter[]) + + delete query?.onEmptyFilter + if (body.query) { // Delete extraneous search params that cannot be overridden delete body.query.onEmptyFilter - if ( - !isExternalTableID(view.tableId) && - !(await features.flags.isEnabled("SQS")) - ) { + if (!supportsLogicalOperators) { // In the unlikely event that a Grouped Filter is in a non-SQS environment // It needs to be ignored. Entirely - let queryFilters: SearchFilter[] = Array.isArray(view.query) - ? view.query - : [] + let queryFilters: SearchFilter[] = Array.isArray(query) ? query : [] // Extract existing fields const existingFields = diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index 65f400a1d9..a73992bcee 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -15,7 +15,9 @@ export const removeInvalidFilters = ( const result = cloneDeep(filters) validFields = validFields.map(f => f.toLowerCase()) - for (const filterKey of Object.keys(result) as (keyof SearchFilters)[]) { + for (const filterKey of Object.keys( + result || {} + ) as (keyof SearchFilters)[]) { const filter = result[filterKey] if (!filter || typeof filter !== "object") { continue @@ -24,7 +26,7 @@ export const removeInvalidFilters = ( const resultingConditions: SearchFilters[] = [] for (const condition of filter.conditions) { const resultingCondition = removeInvalidFilters(condition, validFields) - if (Object.keys(resultingCondition).length) { + if (Object.keys(resultingCondition || {}).length) { resultingConditions.push(resultingCondition) } } diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 97e1212401..63d3044da9 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -428,6 +428,126 @@ const builderFilter = (expression: SearchFilter) => { return query } +export const buildQueryLegacy = (filter: SearchFilter[]) => { + // + let query: SearchFilters = { + string: {}, + fuzzy: {}, + range: {}, + equal: {}, + notEqual: {}, + empty: {}, + notEmpty: {}, + contains: {}, + notContains: {}, + oneOf: {}, + containsAny: {}, + } + + if (!Array.isArray(filter)) { + return query + } + + filter.forEach(expression => { + let { operator, field, type, value, externalType, onEmptyFilter } = + expression + const queryOperator = operator as SearchFilterOperator + const isHbs = + typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0 + // Parse all values into correct types + if (operator === "allOr") { + query.allOr = true + return + } + if (onEmptyFilter) { + query.onEmptyFilter = onEmptyFilter + return + } + if ( + type === "datetime" && + !isHbs && + queryOperator !== "empty" && + queryOperator !== "notEmpty" + ) { + // Ensure date value is a valid date and parse into correct format + if (!value) { + return + } + try { + value = new Date(value).toISOString() + } catch (error) { + return + } + } + if (type === "number" && typeof value === "string" && !isHbs) { + if (queryOperator === "oneOf") { + value = value.split(",").map(item => parseFloat(item)) + } else { + value = parseFloat(value) + } + } + if (type === "boolean") { + value = `${value}`?.toLowerCase() === "true" + } + if ( + ["contains", "notContains", "containsAny"].includes(operator) && + type === "array" && + typeof value === "string" + ) { + value = value.split(",") + } + if (operator.startsWith("range") && query.range) { + const minint = + SqlNumberTypeRangeMap[ + externalType as keyof typeof SqlNumberTypeRangeMap + ]?.min || Number.MIN_SAFE_INTEGER + const maxint = + SqlNumberTypeRangeMap[ + externalType as keyof typeof SqlNumberTypeRangeMap + ]?.max || Number.MAX_SAFE_INTEGER + if (!query.range[field]) { + query.range[field] = { + low: type === "number" ? minint : "0000-00-00T00:00:00.000Z", + high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z", + } + } + if (operator === "rangeLow" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + low: value, + } + } else if (operator === "rangeHigh" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + high: value, + } + } + } else if (isLogicalSearchOperator(queryOperator)) { + // TODO + } else if (query[queryOperator] && operator !== "onEmptyFilter") { + if (type === "boolean") { + // Transform boolean filters to cope with null. + // "equals false" needs to be "not equals true" + // "not equals false" needs to be "equals true" + if (queryOperator === "equal" && value === false) { + query.notEqual = query.notEqual || {} + query.notEqual[field] = true + } else if (queryOperator === "notEqual" && value === false) { + query.equal = query.equal || {} + query.equal[field] = true + } else { + query[queryOperator] ??= {} + query[queryOperator]![field] = value + } + } else { + query[queryOperator] ??= {} + query[queryOperator]![field] = value + } + } + }) + return query +} + export const buildQuery = (filter?: SearchFilterGroup | SearchFilter[]) => { if (!filter) { return @@ -891,9 +1011,23 @@ export const hasFilters = (query?: SearchFilters) => { } const queryRoot = query[LogicalOperator.AND] ?? query[LogicalOperator.OR] + if (!queryRoot) { + const skipped = ["allOr", "onEmptyFilter"] + for (let [key, value] of Object.entries(query)) { + if (skipped.includes(key) || typeof value !== "object") { + continue + } + if (Object.keys(value || {}).length !== 0) { + return true + } + } + return false + } + return ( (queryRoot?.conditions || []).reduce((acc, group) => { - const groupRoot = group[LogicalOperator.AND] ?? group[LogicalOperator.OR] + const groupRoot = + group?.[LogicalOperator.AND] ?? group?.[LogicalOperator.OR] acc += groupRoot?.conditions?.length || 0 return acc }, 0) > 0 From 9ea3aca5e4776599aa5924917b7d73a2ebbeba4b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 11 Sep 2024 12:21:19 +0100 Subject: [PATCH 11/19] Typing improvements and fixing issue detected by test cases (old lucene syntax). --- .../server/src/api/controllers/row/views.ts | 17 ++++++++++------- packages/shared-core/src/filters.ts | 10 ++++++---- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 5957531857..de01386f6e 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -39,9 +39,9 @@ export async function searchView( // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as // that could let users find rows they should not be allowed to access. - let query: any = supportsLogicalOperators + let query = supportsLogicalOperators ? dataFilters.buildQuery(view.query) - : dataFilters.buildQueryLegacy(view.query as SearchFilter[]) + : dataFilters.buildQueryLegacy(view.query) delete query?.onEmptyFilter @@ -51,8 +51,10 @@ export async function searchView( if (!supportsLogicalOperators) { // In the unlikely event that a Grouped Filter is in a non-SQS environment - // It needs to be ignored. Entirely - let queryFilters: SearchFilter[] = Array.isArray(query) ? query : [] + // It needs to be ignored entirely + let queryFilters: SearchFilter[] = Array.isArray(view.query) + ? view.query + : [] // Extract existing fields const existingFields = @@ -64,15 +66,16 @@ export async function searchView( Object.keys(body.query).forEach(key => { const operator = key as Exclude Object.keys(body.query[operator] || {}).forEach(field => { - if (!existingFields.includes(db.removeKeyNumbering(field))) { + if (query && !existingFields.includes(db.removeKeyNumbering(field))) { query[operator]![field] = body.query[operator]![field] } }) }) } else { + const conditions = query ? [query] : [] query = { $and: { - conditions: [query, body.query], + conditions: [...conditions, body.query], }, } } @@ -80,7 +83,7 @@ export async function searchView( await context.ensureSnippetContext(true) - const enrichedQuery = await enrichSearchContext(query, { + const enrichedQuery = await enrichSearchContext(query || {}, { user: sdk.users.getUserContextBindings(ctx.user), }) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 63d3044da9..941c1846ca 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -428,8 +428,9 @@ const builderFilter = (expression: SearchFilter) => { return query } -export const buildQueryLegacy = (filter: SearchFilter[]) => { - // +export const buildQueryLegacy = ( + filter?: SearchFilterGroup | SearchFilter[] +): SearchFilters | undefined => { let query: SearchFilters = { string: {}, fuzzy: {}, @@ -523,7 +524,6 @@ export const buildQueryLegacy = (filter: SearchFilter[]) => { } } } else if (isLogicalSearchOperator(queryOperator)) { - // TODO } else if (query[queryOperator] && operator !== "onEmptyFilter") { if (type === "boolean") { // Transform boolean filters to cope with null. @@ -548,7 +548,9 @@ export const buildQueryLegacy = (filter: SearchFilter[]) => { return query } -export const buildQuery = (filter?: SearchFilterGroup | SearchFilter[]) => { +export const buildQuery = ( + filter?: SearchFilterGroup | SearchFilter[] +): SearchFilters | undefined => { if (!filter) { return } From ce73307f0885e0ff5a146dfbeee96d3de9b845c3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 11 Sep 2024 12:31:11 +0100 Subject: [PATCH 12/19] Updating yarn.lock. --- yarn.lock | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 146fa74d5b..e3a0f72555 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17798,11 +17798,21 @@ periscopic@^3.1.0: estree-walker "^3.0.0" is-reference "^3.0.0" +pg-cloudflare@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98" + integrity sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q== + pg-connection-string@2.5.0, pg-connection-string@^2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34" integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ== +pg-connection-string@^2.6.4: + version "2.6.4" + resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.4.tgz#f543862adfa49fa4e14bc8a8892d2a84d754246d" + integrity sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA== + pg-int8@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" @@ -17813,11 +17823,21 @@ pg-pool@^3.6.0: resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.0.tgz#3190df3e4747a0d23e5e9e8045bcd99bda0a712e" integrity sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ== +pg-pool@^3.6.2: + version "3.6.2" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.2.tgz#3a592370b8ae3f02a7c8130d245bc02fa2c5f3f2" + integrity sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg== + pg-protocol@*, pg-protocol@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.0.tgz#4c91613c0315349363af2084608db843502f8833" integrity sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q== +pg-protocol@^1.6.1: + version "1.6.1" + resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.1.tgz#21333e6d83b01faaebfe7a33a7ad6bfd9ed38cb3" + integrity sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg== + pg-types@^2.1.0, pg-types@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" @@ -17842,6 +17862,19 @@ pg@8.10.0: pg-types "^2.1.0" pgpass "1.x" +pg@^8.12.0: + version "8.12.0" + resolved "https://registry.yarnpkg.com/pg/-/pg-8.12.0.tgz#9341724db571022490b657908f65aee8db91df79" + integrity sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ== + dependencies: + pg-connection-string "^2.6.4" + pg-pool "^3.6.2" + pg-protocol "^1.6.1" + pg-types "^2.1.0" + pgpass "1.x" + optionalDependencies: + pg-cloudflare "^1.1.1" + pgpass@1.x: version "1.0.5" resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d" @@ -20755,7 +20788,16 @@ string-similarity@^4.0.4: resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b" integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ== -"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -20846,7 +20888,7 @@ stringify-object@^3.2.1: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -20860,6 +20902,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -22820,7 +22869,7 @@ worker-farm@1.7.0: dependencies: errno "~0.1.7" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -22838,6 +22887,15 @@ wrap-ansi@^5.1.0: string-width "^3.0.0" strip-ansi "^5.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From 956a10e1b44f1bffabea1555c2e63dc3d2828b02 Mon Sep 17 00:00:00 2001 From: Dean Date: Wed, 11 Sep 2024 13:05:14 +0100 Subject: [PATCH 13/19] Lint --- packages/shared-core/src/filters.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 1eb77efde0..431f6836da 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -530,6 +530,7 @@ export const buildQueryLegacy = ( } } } else if (isLogicalSearchOperator(queryOperator)) { + // ignore } else if (query[queryOperator] && operator !== "onEmptyFilter") { if (type === "boolean") { // Transform boolean filters to cope with null. From e0d2c706119d198b8b4cdcd4480a41ac38a1a696 Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 16 Sep 2024 12:46:21 +0100 Subject: [PATCH 14/19] PR feedback --- .../builder/src/stores/builder/components.js | 16 ++- .../src/components/app/DataProvider.svelte | 11 +- .../src/components/CoreFilterBuilder.svelte | 116 ++++++++++-------- .../src/components/FilterField.svelte | 19 +-- .../frontend-core/src/fetch/CustomFetch.js | 8 -- packages/frontend-core/src/fetch/DataFetch.js | 2 +- packages/shared-core/src/filters.ts | 50 +++++--- packages/shared-core/src/utils.ts | 6 +- 8 files changed, 127 insertions(+), 101 deletions(-) diff --git a/packages/builder/src/stores/builder/components.js b/packages/builder/src/stores/builder/components.js index 0250ed6556..6c2c438f0c 100644 --- a/packages/builder/src/stores/builder/components.js +++ b/packages/builder/src/stores/builder/components.js @@ -206,9 +206,15 @@ export class ComponentStore extends BudiStore { setting?.type?.startsWith("filter") ) for (let setting of filterableTypes || []) { - enrichedComponent[setting.key] = utils.processSearchFilters( - enrichedComponent[setting.key] - ) + const isLegacy = Array.isArray(enrichedComponent[setting.key]) + + if (isLegacy) { + const processedSetting = utils.processSearchFilters( + enrichedComponent[setting.key] + ) + enrichedComponent[setting.key] = processedSetting + migrated = true + } } return migrated } @@ -575,9 +581,7 @@ export class ComponentStore extends BudiStore { const patchResult = patchFn(component, screen) // Post processing - const migrated = null - - this.migrateSettings(component) + const migrated = this.migrateSettings(component) // Returning an explicit false signifies that we should skip this // update. If we migrated something, ensure we never skip. diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte index 7b90432bbf..a2ba674beb 100644 --- a/packages/client/src/components/app/DataProvider.svelte +++ b/packages/client/src/components/app/DataProvider.svelte @@ -19,9 +19,7 @@ let queryExtensions = {} let localFilters - $: if (filter) { - localFilters = Helpers.cloneDeep(filter) - } + $: localFilters = filter ? Helpers.cloneDeep(filter) : null $: defaultQuery = QueryUtils.buildQuery(localFilters) @@ -131,7 +129,7 @@ } const extendQuery = (defaultQuery, extensions) => { - return { + const extended = { [LogicalOperator.AND]: { conditions: [ ...(defaultQuery ? [defaultQuery] : []), @@ -140,6 +138,11 @@ }, onEmptyFilter: EmptyFilterOption.RETURN_NONE, } + + // If there are no conditions applied at all, clear the request. + return extended[LogicalOperator.AND]?.conditions?.length > 0 + ? extended + : null } const setUpAutoRefresh = autoRefresh => { diff --git a/packages/frontend-core/src/components/CoreFilterBuilder.svelte b/packages/frontend-core/src/components/CoreFilterBuilder.svelte index 405373e0a7..c711a57e1c 100644 --- a/packages/frontend-core/src/components/CoreFilterBuilder.svelte +++ b/packages/frontend-core/src/components/CoreFilterBuilder.svelte @@ -58,8 +58,13 @@ return { value: entry, label: Helpers.capitalise(entry) } }) + const onEmptyLabelling = { + [OnEmptyFilter.RETURN_ALL]: "All rows", + [OnEmptyFilter.RETURN_NONE]: "No rows", + } + const onEmptyOptions = Object.values(OnEmptyFilter).map(entry => { - return { value: entry, label: Helpers.capitalise(entry) } + return { value: entry, label: onEmptyLabelling[entry] } }) const context = getContext("context") @@ -151,7 +156,7 @@ const getGroupPrefix = groupIdx => { if (groupIdx == 0) { - return "Where" + return "When" } const operatorMapping = { [FilterOperator.ANY]: "or", @@ -191,16 +196,17 @@ if (targetFilter) { if (deleteFilter) { targetGroup.filters.splice(filterIdx, 1) + + // Clear the group entirely if no valid filters remain + if (targetGroup.filters.length === 0) { + editable.groups.splice(groupIdx, 1) + } } else if (filter) { targetGroup.filters[filterIdx] = filter } } else if (targetGroup) { if (deleteGroup) { - if (editable.groups.length > 1) { - editable.groups.splice(groupIdx, 1) - } else { - editable = {} - } + editable.groups.splice(groupIdx, 1) } else if (addFilter) { targetGroup.filters.push({ valueType: FilterValueType.VALUE, @@ -239,6 +245,9 @@ } } + // Set the request to null if the groups are emptied + editable = editable.groups.length ? editable : null + dispatch("change", editable) } @@ -294,7 +303,7 @@ placeholder={false} /> - of the following filters are met: + of the following filters are matched:
- {#if behaviourFilters && editableFilters?.groups?.length} -
- Return - - opt.label} + getOptionValue={opt => opt.value} + on:change={e => { + handleFilterChange({ + onEmptyFilter: e.detail, + }) + }} + placeholder={false} + /> + + when all filters are empty +
+ {/if} +
+ + + - - when all filters are empty +
- {/if} -
- - - - -
+
{:else} None of the table column can be used for filtering. @@ -446,7 +457,7 @@ .empty-filter, .group-options { display: flex; - gap: var(--spacing-xl); + gap: var(--spacing-m); align-items: center; } @@ -461,14 +472,13 @@ } .empty-filter-picker { - width: 80px; + width: 92px; } .filter-groups { display: flex; flex-direction: column; gap: var(--spacing-xl); - /* overflow-x: scroll; */ } .group { @@ -489,7 +499,7 @@ .filter { display: grid; gap: var(--spacing-l); - grid-template-columns: minmax(150px, 1fr) 170px 200px 40px 40px; + grid-template-columns: minmax(150px, 1fr) 170px minmax(200px, 1fr) 40px 40px; } .filters-footer { diff --git a/packages/frontend-core/src/components/FilterField.svelte b/packages/frontend-core/src/components/FilterField.svelte index 03a09c9db0..c763194d69 100644 --- a/packages/frontend-core/src/components/FilterField.svelte +++ b/packages/frontend-core/src/components/FilterField.svelte @@ -20,12 +20,9 @@ export let bindings = [] export let allowBindings = false export let schemaFields - - // Export these to another field type export let panel export let toReadable export let toRuntime - // Only required if you're in the builder, which we aint. const dispatch = createEventDispatcher() const { OperatorOptions, FilterValueType } = Constants @@ -48,17 +45,21 @@ return schemaFields.find(field => field.name === filter.field) } + const drawerOnChange = e => { + drawerValue = e.detail + } + const onChange = e => { fieldValue = e.detail dispatch("change", { - value: toRuntime(bindings, fieldValue), + value: toRuntime ? toRuntime(bindings, fieldValue) : fieldValue, }) } - const onConfirm = () => { + const onConfirmBinding = () => { dispatch("change", { - value: fieldValue, - valueType: fieldValue ? FilterValueType.BINDING : FilterValueType.VALUE, + value: toRuntime ? toRuntime(bindings, drawerValue) : drawerValue, + valueType: drawerValue ? FilterValueType.BINDING : FilterValueType.VALUE, }) } @@ -139,7 +140,7 @@ cta slot="buttons" on:click={() => { - onConfirm() + onConfirmBinding() bindingDrawer.hide() }} > @@ -153,7 +154,7 @@ allowJS allowHelpers allowHBS - on:change={onChange} + on:change={drawerOnChange} {bindings} /> diff --git a/packages/frontend-core/src/fetch/CustomFetch.js b/packages/frontend-core/src/fetch/CustomFetch.js index a8b14a0809..fc62d790e2 100644 --- a/packages/frontend-core/src/fetch/CustomFetch.js +++ b/packages/frontend-core/src/fetch/CustomFetch.js @@ -1,14 +1,6 @@ import DataFetch from "./DataFetch.js" export default class CustomFetch extends DataFetch { - determineFeatureFlags() { - return { - supportsSearch: false, - supportsSort: false, - supportsPagination: false, - } - } - // Gets the correct Budibase type for a JS value getType(value) { if (value == null) { diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js index 18dc72bf24..4f51cdad71 100644 --- a/packages/frontend-core/src/fetch/DataFetch.js +++ b/packages/frontend-core/src/fetch/DataFetch.js @@ -183,7 +183,7 @@ export default class DataFetch { } // Build the query - let query = this.options.query || null + let query = this.options.query if (!query && this.features.supportsSearch) { query = buildQuery(filter || defaultQuery) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 431f6836da..360d3ae512 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -307,11 +307,11 @@ export class ColumnSplitter { } /** - * Builds a JSON query from the filter structure generated in the builder + * Builds a JSON query from the filter a SearchFilter definition * @param filter the builder filter structure */ -const builderFilter = (expression: SearchFilter) => { +const buildCondition = (expression: SearchFilter) => { // Filter body let query: SearchFilters = { string: {}, @@ -378,13 +378,15 @@ const builderFilter = (expression: SearchFilter) => { value = `${value}`?.toLowerCase() === "true" } if ( - ["contains", "notContains", "containsAny"].includes(operator) && + ["contains", "notContains", "containsAny"].includes( + operator.toLocaleString() + ) && type === "array" && typeof value === "string" ) { value = value.split(",") } - if (operator.startsWith("range") && query.range) { + if (operator.toLocaleString().startsWith("range") && query.range) { const minint = SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap] ?.min || Number.MIN_SAFE_INTEGER @@ -497,13 +499,15 @@ export const buildQueryLegacy = ( value = `${value}`?.toLowerCase() === "true" } if ( - ["contains", "notContains", "containsAny"].includes(operator) && + ["contains", "notContains", "containsAny"].includes( + operator.toLocaleString() + ) && type === "array" && typeof value === "string" ) { value = value.split(",") } - if (operator.startsWith("range") && query.range) { + if (operator.toLocaleString().startsWith("range") && query.range) { const minint = SqlNumberTypeRangeMap[ externalType as keyof typeof SqlNumberTypeRangeMap @@ -555,28 +559,40 @@ export const buildQueryLegacy = ( return query } +/** + * Converts a **SearchFilterGroup** filter definition into a grouped + * search query of type **SearchFilters** + * + * Legacy support remains for the old **SearchFilter[]** format. + * These will be migrated to an appropriate **SearchFilters** object, if encountered + * + * @param filter + * + * @returns {SearchFilters} + */ + export const buildQuery = ( filter?: SearchFilterGroup | SearchFilter[] ): SearchFilters | undefined => { - if (!filter) { - return - } - - const parsedFilter = processSearchFilters(filter) + const parsedFilter: SearchFilterGroup | undefined = + processSearchFilters(filter) if (!parsedFilter) { return } - const operatorMap = { - [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND, - [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR, - } + const operatorMap: { [key in FilterGroupLogicalOperator]: LogicalOperator } = + { + [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND, + [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR, + } const globalOnEmpty = parsedFilter.onEmptyFilter ? parsedFilter.onEmptyFilter : null - const globalOperator = operatorMap[parsedFilter.logicalOperator] + + const globalOperator: LogicalOperator = + operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator] const coreRequest: SearchFilters = { ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}), @@ -585,7 +601,7 @@ export const buildQuery = ( return { [operatorMap[group.logicalOperator]]: { conditions: group.filters - ?.map(x => builderFilter(x)) + ?.map(x => buildCondition(x)) .filter(filter => filter), }, } diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index 027db8e914..f37d50d172 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -97,8 +97,8 @@ export function trimOtherProps(object: any, allowedProps: string[]) { * @param {SearchFilter[] | SearchFilterGroup} filters */ export const processSearchFilters = ( - filters: SearchFilter[] | SearchFilterGroup -) => { + filters: SearchFilter[] | SearchFilterGroup | undefined +): SearchFilterGroup | undefined => { if (!filters) { return } @@ -182,7 +182,7 @@ export const processSearchFilters = ( return migratedSetting } else if (!filters?.groups) { - return null + return } return filters } From 22eafa892c9a8f11512058f54f19ba9780f8f1a0 Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 16 Sep 2024 13:02:35 +0100 Subject: [PATCH 15/19] PR feedback --- packages/client/src/components/app/DataProvider.svelte | 5 +---- .../src/components/app/dynamic-filter/DynamicFilter.svelte | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte index a2ba674beb..c05d4de74e 100644 --- a/packages/client/src/components/app/DataProvider.svelte +++ b/packages/client/src/components/app/DataProvider.svelte @@ -17,11 +17,8 @@ let interval let queryExtensions = {} - let localFilters - $: localFilters = filter ? Helpers.cloneDeep(filter) : null - - $: defaultQuery = QueryUtils.buildQuery(localFilters) + $: defaultQuery = QueryUtils.buildQuery(filter) // We need to manage our lucene query manually as we want to allow components // to extend it diff --git a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte index a1b64b181a..4ab01de17d 100644 --- a/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte +++ b/packages/client/src/components/app/dynamic-filter/DynamicFilter.svelte @@ -95,7 +95,7 @@ } const updateQuery = () => { - filters = Helpers.cloneDeep(editableFilters) + filters = editableFilters } onDestroy(() => { From e8f71f6aced5fcb5a6a9123177f03c1cfaa694e4 Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 16 Sep 2024 14:24:50 +0100 Subject: [PATCH 16/19] Lint --- packages/client/src/components/app/DataProvider.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/components/app/DataProvider.svelte b/packages/client/src/components/app/DataProvider.svelte index c05d4de74e..7b5187ffb6 100644 --- a/packages/client/src/components/app/DataProvider.svelte +++ b/packages/client/src/components/app/DataProvider.svelte @@ -1,6 +1,6 @@