From 11b146fcbf0a39e344e859cb63ec6840f15de15d Mon Sep 17 00:00:00 2001 From: Dean Date: Mon, 9 Sep 2024 16:36:31 +0100 Subject: [PATCH] Updates to filter UI and API requests across budibase --- packages/backend-core/src/users/users.ts | 29 - .../SetupPanel/AutomationBlockSetup.svelte | 36 +- .../automation/SetupPanel/RowSelector.svelte | 10 +- .../buttons/TableFilterButton.svelte | 35 +- .../FilterEditor/FilterBuilder.svelte | 83 +-- .../controls/FilterEditor/FilterEditor.svelte | 40 +- .../GridColumnConfiguration.svelte | 30 +- .../controls/TableConditionEditor.svelte | 5 +- .../builder/src/stores/builder/components.js | 13 +- .../builder/src/stores/portal/featureFlags.js | 14 + packages/builder/src/stores/portal/index.js | 1 + packages/client/manifest.json | 3 +- .../src/components/app/DataProvider.svelte | 31 +- .../app/dynamic-filter/DynamicFilter.svelte | 40 +- .../app/dynamic-filter/FilterModal.svelte | 14 - .../src/components/CoreFilterBuilder.svelte | 505 ++++++++++++++++++ .../src/components/FilterBuilder.svelte | 379 ------------- .../src/components/FilterField.svelte | 318 +++++++++++ .../src/components/FilterUsers.svelte | 3 +- .../src/components/grid/stores/filter.js | 18 +- .../frontend-core/src/components/index.js | 2 +- .../frontend-core/src/fetch/CustomFetch.js | 8 + packages/frontend-core/src/fetch/DataFetch.js | 13 +- packages/frontend-core/src/fetch/UserFetch.js | 12 +- .../frontend-core/src/fetch/ViewV2Fetch.js | 6 +- packages/frontend-core/src/utils/utils.js | 137 ++--- .../server/src/api/controllers/row/views.ts | 13 +- packages/shared-core/src/filters.ts | 177 +----- packages/shared-core/src/utils.ts | 135 +++++ packages/types/src/api/web/searchFilter.ts | 13 +- packages/types/src/sdk/search.ts | 5 + .../src/api/controllers/global/users.ts | 4 +- 32 files changed, 1300 insertions(+), 832 deletions(-) create mode 100644 packages/builder/src/stores/portal/featureFlags.js delete mode 100644 packages/client/src/components/app/dynamic-filter/FilterModal.svelte create mode 100644 packages/frontend-core/src/components/CoreFilterBuilder.svelte delete mode 100644 packages/frontend-core/src/components/FilterBuilder.svelte create mode 100644 packages/frontend-core/src/components/FilterField.svelte diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 0c994d8287..d8546afa8b 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -17,11 +17,8 @@ import { ContextUser, CouchFindOptions, DatabaseQueryOpts, - SearchFilters, SearchUsersRequest, User, - BasicOperator, - ArrayOperator, } from "@budibase/types" import * as context from "../context" import { getGlobalDB } from "../context" @@ -45,32 +42,6 @@ function removeUserPassword(users: User | User[]) { return users } -export function isSupportedUserSearch(query: SearchFilters) { - const allowed = [ - { op: BasicOperator.STRING, key: "email" }, - { op: BasicOperator.EQUAL, key: "_id" }, - { op: ArrayOperator.ONE_OF, key: "_id" }, - ] - for (let [key, operation] of Object.entries(query)) { - if (typeof operation !== "object") { - return false - } - const fields = Object.keys(operation || {}) - // this filter doesn't contain options - ignore - if (fields.length === 0) { - continue - } - const allowedOperation = allowed.find( - allow => - allow.op === key && fields.length === 1 && fields[0] === allow.key - ) - if (!allowedOperation) { - return false - } - } - return true -} - export async function bulkGetGlobalUsersById( userIds: string[], opts?: GetOpts diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 6c4865b539..577f4a3f90 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -62,6 +62,7 @@ } from "@budibase/types" import { FIELDS } from "constants/backend" import PropField from "./PropField.svelte" + import { utils } from "@budibase/shared-core" export let block export let testData @@ -95,8 +96,14 @@ $: memoEnvVariables.set($environment.variables) $: memoBlock.set(block) - $: filters = lookForFilters(schemaProperties) || [] - $: tempFilters = filters + $: filters = lookForFilters(schemaProperties) + $: filterCount = + filters?.groups?.reduce((acc, group) => { + acc = acc += group?.filters?.length || 0 + return acc + }, 0) || 0 + + $: tempFilters = cloneDeep(filters) $: stepId = $memoBlock.stepId $: automationBindings = getAvailableBindings( @@ -768,14 +775,13 @@ break } } - return filters || [] + return utils.processSearchFilters(filters) } function saveFilters(key) { - const filters = QueryUtils.buildQuery(tempFilters) - + const query = QueryUtils.buildQuery(tempFilters) onChange({ - [key]: filters, + [key]: query, [`${key}-def`]: tempFilters, // need to store the builder definition in the automation }) @@ -1004,18 +1010,24 @@ {:else if value.customType === AutomationCustomIOType.FILTERS || value.customType === AutomationCustomIOType.TRIGGER_FILTER} - {filters.length > 0 - ? "Update Filter" - : "No Filter set"} + {filterCount > 0 ? "Update Filter" : "No Filter set"} + + { + tempFilters = filters + }} > - + { + return Array.isArray(fieldValue) ? fieldValue.join(",") : fieldValue + } {#each schemaFields || [] as [field, schema]} @@ -257,7 +265,7 @@ panel={AutomationBindingPanel} type={schema.type} {schema} - value={editableRow[field]} + value={drawerValue(editableRow[field])} on:change={e => onChange({ row: { diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte index 91ec9a9b01..6c6ce8c56a 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte @@ -5,6 +5,7 @@ import { getUserBindings } from "dataBinding" import { makePropSafe } from "@budibase/string-templates" import { search } from "@budibase/frontend-core" + import { utils } from "@budibase/shared-core" import { tables } from "stores/builder" export let schema @@ -16,15 +17,19 @@ let drawer - $: tempValue = filters || [] + $: localFilters = utils.processSearchFilters(filters) + $: schemaFields = search.getFields( $tables.list, Object.values(schema || {}), { allowLinks: true } ) - $: text = getText(filters) - $: selected = tempValue.filter(x => !x.onEmptyFilter)?.length > 0 + $: filterCount = + localFilters?.groups?.reduce((acc, group) => { + return (acc += group.filters.filter(filter => filter.field).length) + }, 0) || 0 + $: bindings = [ { type: "context", @@ -38,28 +43,32 @@ }, ...getUserBindings(), ] - const getText = filters => { - const count = filters?.filter(filter => filter.field)?.length - return count ? `Filter (${count})` : "Filter" - } - - {text} + 0} +> + {filterCount ? `Filter (${filterCount})` : "Filter"} { + localFilters = utils.processSearchFilters(filters) + }} forceModal > (tempValue = e.detail)} + on:change={e => (localFilters = e.detail)} {bindings} /> diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte index 64e93675d9..1e79f61bae 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte @@ -1,87 +1,32 @@ - -
- - { - const indexToUpdate = rawFilters.findIndex(f => f.id === filter.id) - rawFilters[indexToUpdate] = { - ...rawFilters[indexToUpdate], - value: event.detail, - } - }} - /> - + {bindings} + on:change +/> diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte index ed5e36cd65..d7b0c07b26 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte @@ -5,9 +5,14 @@ Button, Drawer, DrawerContent, + Helpers, } from "@budibase/bbui" import { createEventDispatcher } from "svelte" - import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding" + import { + getDatasourceForProvider, + getSchemaForDatasource, + readableToRuntimeBinding, + } from "dataBinding" import FilterBuilder from "./FilterBuilder.svelte" import { tables, selectedScreen } from "stores/builder" import { search } from "@budibase/frontend-core" @@ -21,7 +26,7 @@ let drawer - $: tempValue = value + $: localFilters = Helpers.cloneDeep(value) $: datasource = getDatasourceForProvider($selectedScreen, componentInstance) $: dsSchema = getSchemaForDatasource($selectedScreen, datasource)?.schema $: schemaFields = search.getFields( @@ -29,19 +34,24 @@ Object.values(schema || dsSchema || {}), { allowLinks: true } ) - $: text = getText(value?.filter(filter => filter.field)) + + $: text = getText(value?.groups) async function saveFilter() { - dispatch("change", tempValue) + dispatch("change", localFilters) notifications.success("Filters saved") drawer.hide() } - const getText = filters => { - if (!filters?.length) { + const getText = (filterGroups = []) => { + const allFilters = filterGroups.reduce((acc, group) => { + return (acc += group.filters.filter(filter => filter.field).length) + }, 0) + + if (allFilters === 0) { return "No filters set" } else { - return `${filters.length} filter${filters.length === 1 ? "" : "s"} set` + return `${allFilters} filter${allFilters === 1 ? "" : "s"} set` } } @@ -49,15 +59,25 @@
{text}
- + { + // Reset to the currently available value. + localFilters = Helpers.cloneDeep(value) + }} +> (tempValue = e.detail)} + on:change={e => { + localFilters = e.detail + }} /> diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte index 17cb171da5..eae26348fd 100644 --- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte +++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte @@ -6,6 +6,7 @@ import FieldSetting from "./FieldSetting.svelte" import PrimaryColumnFieldSetting from "./PrimaryColumnFieldSetting.svelte" import getColumns from "./getColumns.js" + import InfoDisplay from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte" export let value export let componentInstance @@ -58,16 +59,25 @@
{/if} - columns.updateSortable(e.detail)} - on:itemChange={e => columns.update(e.detail)} - items={columns.sortable} - listItemKey={"_id"} - listType={FieldSetting} - listTypeProps={{ - bindings, - }} -/> + +{#if columns?.sortable?.length} + columns.updateSortable(e.detail)} + on:itemChange={e => columns.update(e.detail)} + items={columns.sortable} + listItemKey={"_id"} + listType={FieldSetting} + listTypeProps={{ + bindings, + }} + /> +{:else} + +{/if} diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte deleted file mode 100644 index 3a0c789b9e..0000000000 --- a/packages/frontend-core/src/components/FilterBuilder.svelte +++ /dev/null @@ -1,379 +0,0 @@ - - -
- - {#if fieldOptions?.length} - - {#if !fieldFilters?.length} - Add your first filter expression. - {:else} - - {#if behaviourFilters} -
- opt.label} - getOptionValue={opt => opt.value} - on:change={e => handleOnEmptyFilter(e.detail)} - placeholder={null} - /> - {/if} -
- {/if} - {/if} - - {#if fieldFilters?.length} -
- {#if filtersLabel} -
- -
- {/if} -
- {#each fieldFilters as filter} - onOperatorChange(filter)} - placeholder={null} - /> - {#if allowBindings} - - {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === ArrayOperator.ONE_OF)} - - {:else if filter.type === FieldType.OPTIONS} - - {:else if filter.type === FieldType.BOOLEAN} - - {:else if filter.type === FieldType.DATETIME} - - {:else if [FieldType.BB_REFERENCE, FieldType.BB_REFERENCE_SINGLE].includes(filter.type)} - - {:else} - - {/if} -
- duplicateFilter(filter.id)} - /> - removeFilter(filter.id)} - /> -
- {/each} -
-
- {/if} -
- -
- {:else} - None of the table column can be used for filtering. - {/if} - -
- - diff --git a/packages/frontend-core/src/components/FilterField.svelte b/packages/frontend-core/src/components/FilterField.svelte new file mode 100644 index 0000000000..03a09c9db0 --- /dev/null +++ b/packages/frontend-core/src/components/FilterField.svelte @@ -0,0 +1,318 @@ + + +
+ + + + + + +
+
+ {#if filter.valueType === FilterValueType.BINDING} + + {:else} +
+ {#if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)} + + {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === ArrayOperator.ONE_OF)} + + {:else if filter.type === FieldType.OPTIONS} + + {:else if filter.type === FieldType.BOOLEAN} + + {:else if filter.type === FieldType.DATETIME} + + {:else if [FieldType.BB_REFERENCE, FieldType.BB_REFERENCE_SINGLE].includes(filter.type)} + + {:else} + + {/if} +
+ {/if} +
+ +
+ + {#if !disabled && allowBindings && filter.field && !filter.noValue} + + +
{ + bindingDrawer.show() + }} + > + +
+ {/if} +
+
+
+ + diff --git a/packages/frontend-core/src/components/FilterUsers.svelte b/packages/frontend-core/src/components/FilterUsers.svelte index 489426df1e..4640561afd 100644 --- a/packages/frontend-core/src/components/FilterUsers.svelte +++ b/packages/frontend-core/src/components/FilterUsers.svelte @@ -27,7 +27,8 @@
option.email} diff --git a/packages/frontend-core/src/components/grid/stores/filter.js b/packages/frontend-core/src/components/grid/stores/filter.js index a16b101bbb..f5feee9c35 100644 --- a/packages/frontend-core/src/components/grid/stores/filter.js +++ b/packages/frontend-core/src/components/grid/stores/filter.js @@ -1,5 +1,5 @@ import { writable, get, derived } from "svelte/store" -import { FieldType } from "@budibase/types" +import { FieldType, FilterGroupLogicalOperator } from "@budibase/types" export const createStores = context => { const { props } = context @@ -16,11 +16,22 @@ export const createStores = context => { export const deriveStores = context => { const { filter, inlineFilters } = context - const allFilters = derived( [filter, inlineFilters], ([$filter, $inlineFilters]) => { - return [...($filter || []), ...$inlineFilters] + const inlineFilterGroup = $inlineFilters?.length + ? { + logicalOperator: FilterGroupLogicalOperator.ALL, + filters: [...($inlineFilters || [])], + } + : null + + return inlineFilterGroup + ? { + logicalOperator: FilterGroupLogicalOperator.ALL, + groups: [...($filter?.groups || []), inlineFilterGroup], + } + : $filter } ) @@ -54,7 +65,6 @@ export const createActions = context => { inlineFilter.operator = "contains" } - // Add this filter inlineFilters.update($inlineFilters => { // Remove any existing inline filter for this column $inlineFilters = $inlineFilters?.filter(x => x.id !== filterId) diff --git a/packages/frontend-core/src/components/index.js b/packages/frontend-core/src/components/index.js index d494abb82d..0557ec080e 100644 --- a/packages/frontend-core/src/components/index.js +++ b/packages/frontend-core/src/components/index.js @@ -7,5 +7,5 @@ export { default as UserAvatars } from "./UserAvatars.svelte" export { default as Updating } from "./Updating.svelte" export { Grid } from "./grid" export { default as ClientAppSkeleton } from "./ClientAppSkeleton.svelte" -export { default as FilterBuilder } from "./FilterBuilder.svelte" +export { default as CoreFilterBuilder } from "./CoreFilterBuilder.svelte" export { default as FilterUsers } from "./FilterUsers.svelte" diff --git a/packages/frontend-core/src/fetch/CustomFetch.js b/packages/frontend-core/src/fetch/CustomFetch.js index fc62d790e2..a8b14a0809 100644 --- a/packages/frontend-core/src/fetch/CustomFetch.js +++ b/packages/frontend-core/src/fetch/CustomFetch.js @@ -1,6 +1,14 @@ import DataFetch from "./DataFetch.js" export default class CustomFetch extends DataFetch { + determineFeatureFlags() { + return { + supportsSearch: false, + supportsSort: false, + supportsPagination: false, + } + } + // Gets the correct Budibase type for a JS value getType(value) { if (value == null) { diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js index dedd06264c..1743eef71e 100644 --- a/packages/frontend-core/src/fetch/DataFetch.js +++ b/packages/frontend-core/src/fetch/DataFetch.js @@ -2,6 +2,7 @@ import { writable, derived, get } from "svelte/store" import { cloneDeep } from "lodash/fp" import { QueryUtils } from "../utils" import { convertJSONSchemaToTableSchema } from "../utils/json" +import { FilterGroupLogicalOperator, EmptyFilterOption } from "@budibase/types" const { buildQuery, limit: queryLimit, runQuery, sort } = QueryUtils @@ -176,10 +177,16 @@ export default class DataFetch { } } + let defaultQuery = { + logicalOperator: FilterGroupLogicalOperator.ALL, + groups: [], + } + // Build the query - let query = this.options.query - if (!query) { - query = buildQuery(filter) + let query = this.features.supportsSearch ? this.options.query : null + + if (!query && this.features.supportsSearch) { + query = buildQuery(filter || defaultQuery) } // Update store diff --git a/packages/frontend-core/src/fetch/UserFetch.js b/packages/frontend-core/src/fetch/UserFetch.js index cb2c045cc6..39a8d79214 100644 --- a/packages/frontend-core/src/fetch/UserFetch.js +++ b/packages/frontend-core/src/fetch/UserFetch.js @@ -1,7 +1,7 @@ import { get } from "svelte/store" import DataFetch from "./DataFetch.js" import { TableNames } from "../constants" -import { QueryUtils } from "../utils" +import { utils } from "@budibase/shared-core" export default class UserFetch extends DataFetch { constructor(opts) { @@ -33,11 +33,11 @@ export default class UserFetch extends DataFetch { let finalQuery // convert old format to new one - we now allow use of the lucene format const { appId, paginated, ...rest } = query - if (!QueryUtils.hasFilters(query) && rest.email != null) { - finalQuery = { string: { email: rest.email } } - } else { - finalQuery = rest - } + + finalQuery = utils.isSupportedUserSearch(rest) + ? query + : { string: { email: null } } + try { const opts = { bookmark: cursor, diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.js b/packages/frontend-core/src/fetch/ViewV2Fetch.js index 9d2f8c103a..303414b2dc 100644 --- a/packages/frontend-core/src/fetch/ViewV2Fetch.js +++ b/packages/frontend-core/src/fetch/ViewV2Fetch.js @@ -1,5 +1,6 @@ import DataFetch from "./DataFetch.js" import { get } from "svelte/store" +import { utils } from "@budibase/shared-core" export default class ViewV2Fetch extends DataFetch { determineFeatureFlags() { @@ -53,7 +54,10 @@ export default class ViewV2Fetch extends DataFetch { this.options.sortColumn = definition.sort.field this.options.sortOrder = definition.sort.order } - if (!filter?.length && definition.query?.length) { + + const parsed = utils.processSearchFilters(filter) + + if (!parsed?.groups?.length && definition.query?.groups?.length) { this.options.filter = definition.query } diff --git a/packages/frontend-core/src/utils/utils.js b/packages/frontend-core/src/utils/utils.js index 6c9acb0f89..7194aab07b 100644 --- a/packages/frontend-core/src/utils/utils.js +++ b/packages/frontend-core/src/utils/utils.js @@ -353,98 +353,57 @@ export const buildMultiStepFormBlockDefaultProps = props => { } } -/** - * Processes the filter config. Filters are migrated - * SearchFilter[] to SearchFilterGroup - * - * This is supposed to be in shared-core - * @param {Object[]} filter - */ -export const migrateSearchFilters = filters => { - const defaultCfg = { - logicalOperator: Constants.FilterOperator.ALL, - groups: [], - } +export const handleFilterChange = (req, filters) => { + const { + groupIdx, + filterIdx, + filter, + group, + addFilter, + addGroup, + deleteGroup, + deleteFilter, + logicalOperator, + onEmptyFilter, + } = req - const filterWhitelistKeys = [ - "field", - "operator", - "valueType", // bb - "value", - "type", - "noValue", // bb - ] + let editable = Helpers.cloneDeep(filters) + let targetGroup = editable.groups?.[groupIdx] + let targetFilter = targetGroup?.filters?.[filterIdx] - /** - * Review these - * externalType, formulaType, subtype - */ - - if (Array.isArray(filters)) { - const baseGroup = { - filters: [], - logicalOperator: Constants.FilterOperator.ALL, + if (targetFilter) { + if (deleteFilter) { + targetGroup.filters.splice(filterIdx, 1) + } else if (filter) { + targetGroup.filters[filterIdx] = filter } - - const migratedSetting = filters.reduce((acc, filter) => { - // Sort the properties for easier debugging - // Remove unset values - const filterEntries = Object.entries(filter) - .sort((a, b) => { - return a[0].localeCompare(b[0]) - }) - .filter(x => x[1] ?? false) - - // Scrub invalid filters - const { operator, onEmptyFilter, field, valueType } = filter - if (!field || !valueType) { - // THIS SCRUBS THE 2 GLOBALS - // return acc + } else if (targetGroup) { + if (deleteGroup) { + editable.groups.splice(groupIdx, 1) + } else if (addFilter) { + targetGroup.filters.push({}) + } else if (group) { + editable.groups[groupIdx] = { + ...targetGroup, + ...group, } - - if (filterEntries.length == 1) { - console.log("### one entry ") - const [key, value] = filterEntries[0] - // Global - if (key === "onEmptyFilter") { - // unset otherwise, seems to be the default - acc.onEmptyFilter = value - } else if (key === "operator" && value === "allOr") { - // Group 1 logical operator - baseGroup.logicalOperator = Constants.FilterOperator.ANY - } - - return acc - } - - // Process settings?? - const whiteListedFilterSettings = filterEntries.reduce((acc, entry) => { - const [key, value] = entry - - if (filterWhitelistKeys.includes(key)) { - if (key === "field") { - acc.push([key, value.replace(/^[0-9]+:/, "")]) - } else { - acc.push([key, value]) - } - } - return acc - }, []) - - const migratedFilter = Object.fromEntries(whiteListedFilterSettings) - - baseGroup.filters.push(migratedFilter) - - if (!acc.groups.length) { - // init the base group - acc.groups.push(baseGroup) - } - - return acc - }, defaultCfg) - - console.log("MIGRATED ", migratedSetting) - return migratedSetting + } + } else if (addGroup) { + editable.groups.push({ + logicalOperator: Constants.FilterOperator.ANY, + filters: [{}], + }) + } else if (logicalOperator) { + editable = { + ...editable, + logicalOperator, + } + } else if (onEmptyFilter) { + editable = { + ...editable, + onEmptyFilter, + } } - return false + + return editable } diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 06a01646a7..220e2098f7 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -7,6 +7,7 @@ import { RowSearchParams, SearchFilterKey, LogicalOperator, + SearchFilter, } from "@budibase/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../../sdk" @@ -19,7 +20,7 @@ export async function searchView( ) { const { viewId } = ctx.params - const view = await sdk.views.get(viewId) + const view: ViewV2 = await sdk.views.get(viewId) if (!view) { ctx.throw(404, `View ${viewId} not found`) } @@ -35,7 +36,7 @@ export async function searchView( // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as // that could let users find rows they should not be allowed to access. - let query = dataFilters.buildQuery(view.query || []) + let query: any = dataFilters.buildQuery(view.query ?? []) if (body.query) { // Delete extraneous search params that cannot be overridden delete body.query.onEmptyFilter @@ -44,9 +45,15 @@ export async function searchView( !isExternalTableID(view.tableId) && !(await features.flags.isEnabled("SQS")) ) { + // In the unlikely event that a Grouped Filter is in a non-SQS environment + // It needs to be ignored. Entirely + let queryFilters: SearchFilter[] = Array.isArray(view.query) + ? view.query + : [] + // Extract existing fields const existingFields = - view.query + queryFilters ?.filter(filter => filter.field) .map(filter => db.removeKeyNumbering(filter.field)) || [] diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index c71dfe9dba..6017b41d13 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -24,6 +24,7 @@ import { } from "@budibase/types" import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" +import { processSearchFilters } from "./utils" import { deepGet, schema } from "./helpers" import { isPlainObject, isEmpty } from "lodash" import { decodeNonAscii } from "./helpers/schema" @@ -299,10 +300,6 @@ export class ColumnSplitter { } } -/* - separate into buildQuery and build filter? -*/ - /** * Builds a JSON query from the filter structure generated in the builder * @param filter the builder filter structure @@ -323,13 +320,6 @@ const builderFilter = (expression: SearchFilter) => { oneOf: {}, containsAny: {}, } - - // DEAN - - - // This is the chattiest service we have, pruning the requests - // of bloat should have meaningful impact - // Further validation in this area is a must - let { operator, field, type, value, externalType, onEmptyFilter } = expression const queryOperator = operator as SearchFilterOperator const isHbs = @@ -343,6 +333,13 @@ const builderFilter = (expression: SearchFilter) => { query.onEmptyFilter = onEmptyFilter return } + + // Default the value for noValue fields to ensure they are correctly added + // to the final query + if (queryOperator === "empty" || queryOperator === "notEmpty") { + value = null + } + if ( type === "datetime" && !isHbs && @@ -426,131 +423,11 @@ const builderFilter = (expression: SearchFilter) => { return query } -export const buildQuery = (filter: SearchFilter[]) => { - // - let query: SearchFilters = { - string: {}, - fuzzy: {}, - range: {}, - equal: {}, - notEqual: {}, - empty: {}, - notEmpty: {}, - contains: {}, - notContains: {}, - oneOf: {}, - containsAny: {}, - } +export const buildQuery = (filter: SearchFilterGroup | SearchFilter[]) => { + const parsedFilter = processSearchFilters(filter) - if (!Array.isArray(filter)) { - return query - } - - // DEAN Replace with builderFilter - filter.forEach(expression => { - let { operator, field, type, value, externalType, onEmptyFilter } = - expression - const queryOperator = operator as SearchFilterOperator - const isHbs = - typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0 - // Parse all values into correct types - if (operator === "allOr") { - query.allOr = true - return - } - if (onEmptyFilter) { - query.onEmptyFilter = onEmptyFilter - return - } - if ( - type === "datetime" && - !isHbs && - queryOperator !== "empty" && - queryOperator !== "notEmpty" - ) { - // Ensure date value is a valid date and parse into correct format - if (!value) { - return - } - try { - value = new Date(value).toISOString() - } catch (error) { - return - } - } - if (type === "number" && typeof value === "string" && !isHbs) { - if (queryOperator === "oneOf") { - value = value.split(",").map(item => parseFloat(item)) - } else { - value = parseFloat(value) - } - } - if (type === "boolean") { - value = `${value}`?.toLowerCase() === "true" - } - if ( - ["contains", "notContains", "containsAny"].includes(operator) && - type === "array" && - typeof value === "string" - ) { - value = value.split(",") - } - if (operator.startsWith("range") && query.range) { - const minint = - SqlNumberTypeRangeMap[ - externalType as keyof typeof SqlNumberTypeRangeMap - ]?.min || Number.MIN_SAFE_INTEGER - const maxint = - SqlNumberTypeRangeMap[ - externalType as keyof typeof SqlNumberTypeRangeMap - ]?.max || Number.MAX_SAFE_INTEGER - if (!query.range[field]) { - query.range[field] = { - low: type === "number" ? minint : "0000-00-00T00:00:00.000Z", - high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z", - } - } - if (operator === "rangeLow" && value != null && value !== "") { - query.range[field] = { - ...query.range[field], - low: value, - } - } else if (operator === "rangeHigh" && value != null && value !== "") { - query.range[field] = { - ...query.range[field], - high: value, - } - } - } else if (isLogicalSearchOperator(queryOperator)) { - // TODO - } else if (query[queryOperator] && operator !== "onEmptyFilter") { - if (type === "boolean") { - // Transform boolean filters to cope with null. - // "equals false" needs to be "not equals true" - // "not equals false" needs to be "equals true" - if (queryOperator === "equal" && value === false) { - query.notEqual = query.notEqual || {} - query.notEqual[field] = true - } else if (queryOperator === "notEqual" && value === false) { - query.equal = query.equal || {} - query.equal[field] = true - } else { - query[queryOperator] ??= {} - query[queryOperator]![field] = value - } - } else { - query[queryOperator] ??= {} - query[queryOperator]![field] = value - } - } - }) - return query -} - -// Grouped query -export const buildQueryX = (filter: SearchFilterGroup) => { - if (!filter) { - return {} + if (!parsedFilter) { + return } const operatorMap = { @@ -558,13 +435,15 @@ export const buildQueryX = (filter: SearchFilterGroup) => { [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR, } - const globalOnEmpty = filter.onEmptyFilter ? filter.onEmptyFilter : null - const globalOperator = operatorMap[filter.logicalOperator] + const globalOnEmpty = parsedFilter.onEmptyFilter + ? parsedFilter.onEmptyFilter + : null + const globalOperator = operatorMap[parsedFilter.logicalOperator] const coreRequest: SearchFilters = { ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}), [globalOperator]: { - conditions: filter.groups?.map((group: SearchFilterGroup) => { + conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => { return { [operatorMap[group.logicalOperator]]: { conditions: group.filters?.map(x => builderFilter(x)), @@ -580,6 +459,9 @@ export const buildQueryX = (filter: SearchFilterGroup) => { // this we convert them to arrays at the controller level so that nothing below // this has to worry about the non-array values. export function fixupFilterArrays(filters: SearchFilters) { + if (!filters) { + return filters + } for (const searchField of Object.values(ArrayOperator)) { const field = filters[searchField] if (field == null || !isPlainObject(field)) { @@ -1005,14 +887,13 @@ export const hasFilters = (query?: SearchFilters) => { if (!query) { return false } - const skipped = ["allOr", "onEmptyFilter"] - for (let [key, value] of Object.entries(query)) { - if (skipped.includes(key) || typeof value !== "object") { - continue - } - if (Object.keys(value || {}).length !== 0) { - return true - } - } - return false + const queryRoot = query[LogicalOperator.AND] ?? query[LogicalOperator.OR] + + return ( + (queryRoot?.conditions || []).reduce((acc, group) => { + const groupRoot = group[LogicalOperator.AND] ?? group[LogicalOperator.OR] + acc += groupRoot?.conditions?.length || 0 + return acc + }, 0) > 0 + ) } diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index b69a059745..0824c7e8d5 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -1,4 +1,14 @@ +import { + SearchFilter, + SearchFilterGroup, + FilterGroupLogicalOperator, + EmptyFilterOption, + SearchFilters, + BasicOperator, + ArrayOperator, +} from "@budibase/types" import * as Constants from "./constants" +import { removeKeyNumbering } from "./filters" export function unreachable( value: never, @@ -77,3 +87,128 @@ export function trimOtherProps(object: any, allowedProps: string[]) { ) return result } + +/** + * Processes the filter config. Filters are migrated from + * SearchFilter[] to SearchFilterGroup + * + * If filters is not an array, the migration is skipped + * + * @param {SearchFilter[] | SearchFilterGroup} filters + */ +export const processSearchFilters = ( + filters: SearchFilter[] | SearchFilterGroup +) => { + if (!filters) { + return + } + const defaultCfg: SearchFilterGroup = { + logicalOperator: FilterGroupLogicalOperator.ALL, + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + groups: [], + } + + const filterWhitelistKeys = [ + "field", + "operator", + "value", + "type", + "externalType", + "valueType", + "noValue", + "formulaType", + ] + + if (Array.isArray(filters)) { + let baseGroup: SearchFilterGroup = { + filters: [], + logicalOperator: FilterGroupLogicalOperator.ALL, + } + + const migratedSetting: SearchFilterGroup = filters.reduce( + (acc: SearchFilterGroup, filter: SearchFilter) => { + // Sort the properties for easier debugging + const filterEntries = Object.entries(filter) + .sort((a, b) => { + return a[0].localeCompare(b[0]) + }) + .filter(x => x[1] ?? false) + + if (filterEntries.length == 1) { + const [key, value] = filterEntries[0] + // Global + if (key === "onEmptyFilter") { + // unset otherwise + acc.onEmptyFilter = value + } else if (key === "operator" && value === "allOr") { + // Group 1 logical operator + baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY + } + + return acc + } + + const whiteListedFilterSettings: [string, any][] = filterEntries.reduce( + (acc: [string, any][], entry: [string, any]) => { + const [key, value] = entry + + if (filterWhitelistKeys.includes(key)) { + if (key === "field") { + acc.push([key, removeKeyNumbering(value)]) + } else { + acc.push([key, value]) + } + } + return acc + }, + [] + ) + + const migratedFilter: SearchFilter = Object.fromEntries( + whiteListedFilterSettings + ) as SearchFilter + + baseGroup.filters!.push(migratedFilter) + + if (!acc.groups || !acc.groups.length) { + // init the base group + acc.groups = [baseGroup] + } + + return acc + }, + defaultCfg + ) + + return migratedSetting + } else if (!filters?.groups) { + return defaultCfg + } + return filters +} + +export function isSupportedUserSearch(query: SearchFilters) { + const allowed = [ + { op: BasicOperator.STRING, key: "email" }, + { op: BasicOperator.EQUAL, key: "_id" }, + { op: ArrayOperator.ONE_OF, key: "_id" }, + ] + for (let [key, operation] of Object.entries(query)) { + if (typeof operation !== "object") { + return false + } + const fields = Object.keys(operation || {}) + // this filter doesn't contain options - ignore + if (fields.length === 0) { + continue + } + const allowedOperation = allowed.find( + allow => + allow.op === key && fields.length === 1 && fields[0] === allow.key + ) + if (!allowedOperation) { + return false + } + } + return true +} diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts index 5223204a7f..b3d577f0c8 100644 --- a/packages/types/src/api/web/searchFilter.ts +++ b/packages/types/src/api/web/searchFilter.ts @@ -1,5 +1,9 @@ import { FieldType } from "../../documents" -import { EmptyFilterOption, SearchFilters } from "../../sdk" +import { + EmptyFilterOption, + FilterGroupLogicalOperator, + SearchFilters, +} from "../../sdk" export type SearchFilter = { operator: keyof SearchFilters | "rangeLow" | "rangeHigh" @@ -9,3 +13,10 @@ export type SearchFilter = { value: any externalType?: string } + +export type SearchFilterGroup = { + logicalOperator: FilterGroupLogicalOperator + onEmptyFilter?: EmptyFilterOption + groups?: SearchFilterGroup[] + filters?: SearchFilter[] +} diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 6feea40766..4a98fe5bae 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -180,6 +180,11 @@ export enum EmptyFilterOption { RETURN_NONE = "none", } +export enum FilterGroupLogicalOperator { + ALL = "all", + ANY = "any", +} + export enum SqlClient { MS_SQL = "mssql", POSTGRES = "pg", diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 6ce0eef5a0..921e0324d1 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -37,7 +37,7 @@ import { } from "@budibase/backend-core" import { checkAnyUserExists } from "../../../utilities/users" import { isEmailConfigured } from "../../../utilities/email" -import { BpmStatusKey, BpmStatusValue } from "@budibase/shared-core" +import { BpmStatusKey, BpmStatusValue, utils } from "@budibase/shared-core" const MAX_USERS_UPLOAD_LIMIT = 1000 @@ -256,7 +256,7 @@ export const search = async (ctx: Ctx) => { } } // Validate we aren't trying to search on any illegal fields - if (!userSdk.core.isSupportedUserSearch(body.query)) { + if (!utils.isSupportedUserSearch(body.query)) { ctx.throw(400, "Can only search by string.email, equal._id or oneOf._id") } }