diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 0c994d8287..d8546afa8b 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -17,11 +17,8 @@ import { ContextUser, CouchFindOptions, DatabaseQueryOpts, - SearchFilters, SearchUsersRequest, User, - BasicOperator, - ArrayOperator, } from "@budibase/types" import * as context from "../context" import { getGlobalDB } from "../context" @@ -45,32 +42,6 @@ function removeUserPassword(users: User | User[]) { return users } -export function isSupportedUserSearch(query: SearchFilters) { - const allowed = [ - { op: BasicOperator.STRING, key: "email" }, - { op: BasicOperator.EQUAL, key: "_id" }, - { op: ArrayOperator.ONE_OF, key: "_id" }, - ] - for (let [key, operation] of Object.entries(query)) { - if (typeof operation !== "object") { - return false - } - const fields = Object.keys(operation || {}) - // this filter doesn't contain options - ignore - if (fields.length === 0) { - continue - } - const allowedOperation = allowed.find( - allow => - allow.op === key && fields.length === 1 && fields[0] === allow.key - ) - if (!allowedOperation) { - return false - } - } - return true -} - export async function bulkGetGlobalUsersById( userIds: string[], opts?: GetOpts diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index af67ae8d22..251d4a04e6 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -62,6 +62,7 @@ } from "@budibase/types" import { FIELDS } from "constants/backend" import PropField from "./PropField.svelte" + import { utils } from "@budibase/shared-core" export let block export let testData @@ -95,8 +96,14 @@ $: memoEnvVariables.set($environment.variables) $: memoBlock.set(block) - $: filters = lookForFilters(schemaProperties) || [] - $: tempFilters = filters + $: filters = lookForFilters(schemaProperties) + $: filterCount = + filters?.groups?.reduce((acc, group) => { + acc = acc += group?.filters?.length || 0 + return acc + }, 0) || 0 + + $: tempFilters = cloneDeep(filters) $: stepId = $memoBlock.stepId $: automationBindings = getAvailableBindings( @@ -780,14 +787,13 @@ break } } - return filters || [] + return utils.processSearchFilters(filters) } function saveFilters(key) { - const filters = QueryUtils.buildQuery(tempFilters) - + const query = QueryUtils.buildQuery(tempFilters) onChange({ - [key]: filters, + [key]: query, [`${key}-def`]: tempFilters, // need to store the builder definition in the automation }) @@ -1016,18 +1022,24 @@ {:else if value.customType === AutomationCustomIOType.FILTERS || value.customType === AutomationCustomIOType.TRIGGER_FILTER} - {filters.length > 0 - ? "Update Filter" - : "No Filter set"} + {filterCount > 0 ? "Update Filter" : "No Filter set"} + + { + tempFilters = filters + }} > - + { + return Array.isArray(fieldValue) ? fieldValue.join(",") : fieldValue + } {#each schemaFields || [] as [field, schema]} @@ -257,7 +265,7 @@ panel={AutomationBindingPanel} type={schema.type} {schema} - value={editableRow[field]} + value={drawerValue(editableRow[field])} on:change={e => onChange({ row: { diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte index c4bdc653c3..a467801214 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte @@ -5,6 +5,7 @@ import { getUserBindings } from "dataBinding" import { makePropSafe } from "@budibase/string-templates" import { search } from "@budibase/frontend-core" + import { utils } from "@budibase/shared-core" import { tables } from "stores/builder" export let schema @@ -16,15 +17,19 @@ let drawer - $: tempValue = filters || [] + $: localFilters = utils.processSearchFilters(filters) + $: schemaFields = search.getFields( $tables.list, Object.values(schema || {}), { allowLinks: true } ) - $: text = getText(filters) - $: selected = tempValue.filter(x => !x.onEmptyFilter)?.length > 0 + $: filterCount = + localFilters?.groups?.reduce((acc, group) => { + return (acc += group.filters.filter(filter => filter.field).length) + }, 0) || 0 + $: bindings = [ { type: "context", @@ -38,10 +43,6 @@ }, ...getUserBindings(), ] - const getText = filters => { - const count = filters?.filter(filter => filter.field)?.length - return count ? `Filter (${count})` : "Filter" - } 0} accentColor="#004EA6" > - {text} + {filterCount ? `Filter (${filterCount})` : "Filter"} { + localFilters = utils.processSearchFilters(filters) + }} forceModal > (tempValue = e.detail)} + on:change={e => (localFilters = e.detail)} {bindings} /> diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte index 64e93675d9..1e79f61bae 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte @@ -1,87 +1,32 @@ - -
- - { - const indexToUpdate = rawFilters.findIndex(f => f.id === filter.id) - rawFilters[indexToUpdate] = { - ...rawFilters[indexToUpdate], - value: event.detail, - } - }} - /> - + {bindings} + on:change +/> diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte index ed5e36cd65..d6f1732e64 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte @@ -5,6 +5,7 @@ Button, Drawer, DrawerContent, + Helpers, } from "@budibase/bbui" import { createEventDispatcher } from "svelte" import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding" @@ -21,7 +22,7 @@ let drawer - $: tempValue = value + $: localFilters = Helpers.cloneDeep(value) $: datasource = getDatasourceForProvider($selectedScreen, componentInstance) $: dsSchema = getSchemaForDatasource($selectedScreen, datasource)?.schema $: schemaFields = search.getFields( @@ -29,19 +30,24 @@ Object.values(schema || dsSchema || {}), { allowLinks: true } ) - $: text = getText(value?.filter(filter => filter.field)) + + $: text = getText(value?.groups) async function saveFilter() { - dispatch("change", tempValue) + dispatch("change", localFilters) notifications.success("Filters saved") drawer.hide() } - const getText = filters => { - if (!filters?.length) { + const getText = (filterGroups = []) => { + const allFilters = filterGroups.reduce((acc, group) => { + return (acc += group.filters.filter(filter => filter.field).length) + }, 0) + + if (allFilters === 0) { return "No filters set" } else { - return `${filters.length} filter${filters.length === 1 ? "" : "s"} set` + return `${allFilters} filter${allFilters === 1 ? "" : "s"} set` } } @@ -49,15 +55,25 @@
{text}
- + { + // Reset to the currently available value. + localFilters = Helpers.cloneDeep(value) + }} +> (tempValue = e.detail)} + on:change={e => { + localFilters = e.detail + }} /> diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte index 17cb171da5..eae26348fd 100644 --- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte +++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte @@ -6,6 +6,7 @@ import FieldSetting from "./FieldSetting.svelte" import PrimaryColumnFieldSetting from "./PrimaryColumnFieldSetting.svelte" import getColumns from "./getColumns.js" + import InfoDisplay from "pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/InfoDisplay.svelte" export let value export let componentInstance @@ -58,16 +59,25 @@
{/if} - columns.updateSortable(e.detail)} - on:itemChange={e => columns.update(e.detail)} - items={columns.sortable} - listItemKey={"_id"} - listType={FieldSetting} - listTypeProps={{ - bindings, - }} -/> + +{#if columns?.sortable?.length} + columns.updateSortable(e.detail)} + on:itemChange={e => columns.update(e.detail)} + items={columns.sortable} + listItemKey={"_id"} + listType={FieldSetting} + listTypeProps={{ + bindings, + }} + /> +{:else} + +{/if} diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte deleted file mode 100644 index 3a0c789b9e..0000000000 --- a/packages/frontend-core/src/components/FilterBuilder.svelte +++ /dev/null @@ -1,379 +0,0 @@ - - -
- - {#if fieldOptions?.length} - - {#if !fieldFilters?.length} - Add your first filter expression. - {:else} - - {#if behaviourFilters} -
- opt.label} - getOptionValue={opt => opt.value} - on:change={e => handleOnEmptyFilter(e.detail)} - placeholder={null} - /> - {/if} -
- {/if} - {/if} - - {#if fieldFilters?.length} -
- {#if filtersLabel} -
- -
- {/if} -
- {#each fieldFilters as filter} - onOperatorChange(filter)} - placeholder={null} - /> - {#if allowBindings} - - {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === ArrayOperator.ONE_OF)} - - {:else if filter.type === FieldType.OPTIONS} - - {:else if filter.type === FieldType.BOOLEAN} - - {:else if filter.type === FieldType.DATETIME} - - {:else if [FieldType.BB_REFERENCE, FieldType.BB_REFERENCE_SINGLE].includes(filter.type)} - - {:else} - - {/if} -
- duplicateFilter(filter.id)} - /> - removeFilter(filter.id)} - /> -
- {/each} -
-
- {/if} -
- -
- {:else} - None of the table column can be used for filtering. - {/if} - -
- - diff --git a/packages/frontend-core/src/components/FilterField.svelte b/packages/frontend-core/src/components/FilterField.svelte new file mode 100644 index 0000000000..c763194d69 --- /dev/null +++ b/packages/frontend-core/src/components/FilterField.svelte @@ -0,0 +1,319 @@ + + +
+ + + + + + +
+
+ {#if filter.valueType === FilterValueType.BINDING} + + {:else} +
+ {#if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)} + + {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === ArrayOperator.ONE_OF)} + + {:else if filter.type === FieldType.OPTIONS} + + {:else if filter.type === FieldType.BOOLEAN} + + {:else if filter.type === FieldType.DATETIME} + + {:else if [FieldType.BB_REFERENCE, FieldType.BB_REFERENCE_SINGLE].includes(filter.type)} + + {:else} + + {/if} +
+ {/if} +
+ +
+ + {#if !disabled && allowBindings && filter.field && !filter.noValue} + + +
{ + bindingDrawer.show() + }} + > + +
+ {/if} +
+
+
+ + diff --git a/packages/frontend-core/src/components/FilterUsers.svelte b/packages/frontend-core/src/components/FilterUsers.svelte index 489426df1e..4640561afd 100644 --- a/packages/frontend-core/src/components/FilterUsers.svelte +++ b/packages/frontend-core/src/components/FilterUsers.svelte @@ -27,7 +27,8 @@
option.email} diff --git a/packages/frontend-core/src/components/grid/stores/filter.js b/packages/frontend-core/src/components/grid/stores/filter.js index a16b101bbb..f5feee9c35 100644 --- a/packages/frontend-core/src/components/grid/stores/filter.js +++ b/packages/frontend-core/src/components/grid/stores/filter.js @@ -1,5 +1,5 @@ import { writable, get, derived } from "svelte/store" -import { FieldType } from "@budibase/types" +import { FieldType, FilterGroupLogicalOperator } from "@budibase/types" export const createStores = context => { const { props } = context @@ -16,11 +16,22 @@ export const createStores = context => { export const deriveStores = context => { const { filter, inlineFilters } = context - const allFilters = derived( [filter, inlineFilters], ([$filter, $inlineFilters]) => { - return [...($filter || []), ...$inlineFilters] + const inlineFilterGroup = $inlineFilters?.length + ? { + logicalOperator: FilterGroupLogicalOperator.ALL, + filters: [...($inlineFilters || [])], + } + : null + + return inlineFilterGroup + ? { + logicalOperator: FilterGroupLogicalOperator.ALL, + groups: [...($filter?.groups || []), inlineFilterGroup], + } + : $filter } ) @@ -54,7 +65,6 @@ export const createActions = context => { inlineFilter.operator = "contains" } - // Add this filter inlineFilters.update($inlineFilters => { // Remove any existing inline filter for this column $inlineFilters = $inlineFilters?.filter(x => x.id !== filterId) diff --git a/packages/frontend-core/src/components/index.js b/packages/frontend-core/src/components/index.js index d494abb82d..0557ec080e 100644 --- a/packages/frontend-core/src/components/index.js +++ b/packages/frontend-core/src/components/index.js @@ -7,5 +7,5 @@ export { default as UserAvatars } from "./UserAvatars.svelte" export { default as Updating } from "./Updating.svelte" export { Grid } from "./grid" export { default as ClientAppSkeleton } from "./ClientAppSkeleton.svelte" -export { default as FilterBuilder } from "./FilterBuilder.svelte" +export { default as CoreFilterBuilder } from "./CoreFilterBuilder.svelte" export { default as FilterUsers } from "./FilterUsers.svelte" diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js index 2a1e40c8ca..6e7b8e7c86 100644 --- a/packages/frontend-core/src/constants.js +++ b/packages/frontend-core/src/constants.js @@ -175,3 +175,24 @@ export const TypeIconMap = { export const OptionColours = [...new Array(12).keys()].map(idx => { return `hsla(${((idx + 1) * 222) % 360}, 90%, 75%, 0.3)` }) + +export const FilterOperator = { + ANY: "any", + ALL: "all", +} + +export const OnEmptyFilter = { + RETURN_ALL: "all", + RETURN_NONE: "none", +} + +export const FilterValueType = { + BINDING: "Binding", + VALUE: "Value", +} + +export const FieldPermissions = { + WRITABLE: "writable", + READONLY: "readonly", + HIDDEN: "hidden", +} diff --git a/packages/frontend-core/src/fetch/DataFetch.js b/packages/frontend-core/src/fetch/DataFetch.js index dedd06264c..246cc4f804 100644 --- a/packages/frontend-core/src/fetch/DataFetch.js +++ b/packages/frontend-core/src/fetch/DataFetch.js @@ -178,7 +178,8 @@ export default class DataFetch { // Build the query let query = this.options.query - if (!query) { + + if (!query && this.features.supportsSearch) { query = buildQuery(filter) } diff --git a/packages/frontend-core/src/fetch/UserFetch.js b/packages/frontend-core/src/fetch/UserFetch.js index cb2c045cc6..902aa7edad 100644 --- a/packages/frontend-core/src/fetch/UserFetch.js +++ b/packages/frontend-core/src/fetch/UserFetch.js @@ -1,7 +1,7 @@ import { get } from "svelte/store" import DataFetch from "./DataFetch.js" import { TableNames } from "../constants" -import { QueryUtils } from "../utils" +import { utils } from "@budibase/shared-core" export default class UserFetch extends DataFetch { constructor(opts) { @@ -32,12 +32,12 @@ export default class UserFetch extends DataFetch { const { cursor, query } = get(this.store) let finalQuery // convert old format to new one - we now allow use of the lucene format - const { appId, paginated, ...rest } = query - if (!QueryUtils.hasFilters(query) && rest.email != null) { - finalQuery = { string: { email: rest.email } } - } else { - finalQuery = rest - } + const { appId, paginated, ...rest } = query || {} + + finalQuery = utils.isSupportedUserSearch(rest) + ? query + : { string: { email: null } } + try { const opts = { bookmark: cursor, diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.js b/packages/frontend-core/src/fetch/ViewV2Fetch.js index 9d2f8c103a..46343492b4 100644 --- a/packages/frontend-core/src/fetch/ViewV2Fetch.js +++ b/packages/frontend-core/src/fetch/ViewV2Fetch.js @@ -1,5 +1,6 @@ import DataFetch from "./DataFetch.js" import { get } from "svelte/store" +import { utils } from "@budibase/shared-core" export default class ViewV2Fetch extends DataFetch { determineFeatureFlags() { @@ -53,14 +54,17 @@ export default class ViewV2Fetch extends DataFetch { this.options.sortColumn = definition.sort.field this.options.sortOrder = definition.sort.order } - if (!filter?.length && definition.query?.length) { + + const parsed = utils.processSearchFilters(filter) + + if (!parsed?.groups?.length && definition.query?.groups?.length) { this.options.filter = definition.query } try { const res = await this.API.viewV2.fetch({ viewId: datasource.id, - query, + ...(query ? { query } : {}), paginate, limit, bookmark: cursor, diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 06a01646a7..de01386f6e 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -7,6 +7,7 @@ import { RowSearchParams, SearchFilterKey, LogicalOperator, + SearchFilter, } from "@budibase/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../../sdk" @@ -19,7 +20,7 @@ export async function searchView( ) { const { viewId } = ctx.params - const view = await sdk.views.get(viewId) + const view: ViewV2 = await sdk.views.get(viewId) if (!view) { ctx.throw(404, `View ${viewId} not found`) } @@ -32,21 +33,32 @@ export async function searchView( .map(([key]) => key) const { body } = ctx.request + const sqsEnabled = await features.flags.isEnabled("SQS") + const supportsLogicalOperators = isExternalTableID(view.tableId) || sqsEnabled + // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as // that could let users find rows they should not be allowed to access. - let query = dataFilters.buildQuery(view.query || []) + let query = supportsLogicalOperators + ? dataFilters.buildQuery(view.query) + : dataFilters.buildQueryLegacy(view.query) + + delete query?.onEmptyFilter + if (body.query) { // Delete extraneous search params that cannot be overridden delete body.query.onEmptyFilter - if ( - !isExternalTableID(view.tableId) && - !(await features.flags.isEnabled("SQS")) - ) { + if (!supportsLogicalOperators) { + // In the unlikely event that a Grouped Filter is in a non-SQS environment + // It needs to be ignored entirely + let queryFilters: SearchFilter[] = Array.isArray(view.query) + ? view.query + : [] + // Extract existing fields const existingFields = - view.query + queryFilters ?.filter(filter => filter.field) .map(filter => db.removeKeyNumbering(filter.field)) || [] @@ -54,15 +66,16 @@ export async function searchView( Object.keys(body.query).forEach(key => { const operator = key as Exclude Object.keys(body.query[operator] || {}).forEach(field => { - if (!existingFields.includes(db.removeKeyNumbering(field))) { + if (query && !existingFields.includes(db.removeKeyNumbering(field))) { query[operator]![field] = body.query[operator]![field] } }) }) } else { + const conditions = query ? [query] : [] query = { $and: { - conditions: [query, body.query], + conditions: [...conditions, body.query], }, } } @@ -70,7 +83,7 @@ export async function searchView( await context.ensureSnippetContext(true) - const enrichedQuery = await enrichSearchContext(query, { + const enrichedQuery = await enrichSearchContext(query || {}, { user: sdk.users.getUserContextBindings(ctx.user), }) diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index 65f400a1d9..a73992bcee 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -15,7 +15,9 @@ export const removeInvalidFilters = ( const result = cloneDeep(filters) validFields = validFields.map(f => f.toLowerCase()) - for (const filterKey of Object.keys(result) as (keyof SearchFilters)[]) { + for (const filterKey of Object.keys( + result || {} + ) as (keyof SearchFilters)[]) { const filter = result[filterKey] if (!filter || typeof filter !== "object") { continue @@ -24,7 +26,7 @@ export const removeInvalidFilters = ( const resultingConditions: SearchFilters[] = [] for (const condition of filter.conditions) { const resultingCondition = removeInvalidFilters(condition, validFields) - if (Object.keys(resultingCondition).length) { + if (Object.keys(resultingCondition || {}).length) { resultingConditions.push(resultingCondition) } } diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 45e9a7c6d0..360d3ae512 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -19,9 +19,12 @@ import { RangeOperator, LogicalOperator, isLogicalSearchOperator, + SearchFilterGroup, + FilterGroupLogicalOperator, } from "@budibase/types" import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" +import { processSearchFilters } from "./utils" import { deepGet, schema } from "./helpers" import { isPlainObject, isEmpty } from "lodash" import { decodeNonAscii } from "./helpers/schema" @@ -304,10 +307,138 @@ export class ColumnSplitter { } /** - * Builds a JSON query from the filter structure generated in the builder + * Builds a JSON query from the filter a SearchFilter definition * @param filter the builder filter structure */ -export const buildQuery = (filter: SearchFilter[]) => { + +const buildCondition = (expression: SearchFilter) => { + // Filter body + let query: SearchFilters = { + string: {}, + fuzzy: {}, + range: {}, + equal: {}, + notEqual: {}, + empty: {}, + notEmpty: {}, + contains: {}, + notContains: {}, + oneOf: {}, + containsAny: {}, + } + let { operator, field, type, value, externalType, onEmptyFilter } = expression + + if (!operator || !field) { + return + } + + const queryOperator = operator as SearchFilterOperator + const isHbs = + typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0 + // Parse all values into correct types + if (operator === "allOr") { + query.allOr = true + return + } + if (onEmptyFilter) { + query.onEmptyFilter = onEmptyFilter + return + } + + // Default the value for noValue fields to ensure they are correctly added + // to the final query + if (queryOperator === "empty" || queryOperator === "notEmpty") { + value = null + } + + if ( + type === "datetime" && + !isHbs && + queryOperator !== "empty" && + queryOperator !== "notEmpty" + ) { + // Ensure date value is a valid date and parse into correct format + if (!value) { + return + } + try { + value = new Date(value).toISOString() + } catch (error) { + return + } + } + if (type === "number" && typeof value === "string" && !isHbs) { + if (queryOperator === "oneOf") { + value = value.split(",").map(item => parseFloat(item)) + } else { + value = parseFloat(value) + } + } + if (type === "boolean") { + value = `${value}`?.toLowerCase() === "true" + } + if ( + ["contains", "notContains", "containsAny"].includes( + operator.toLocaleString() + ) && + type === "array" && + typeof value === "string" + ) { + value = value.split(",") + } + if (operator.toLocaleString().startsWith("range") && query.range) { + const minint = + SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap] + ?.min || Number.MIN_SAFE_INTEGER + const maxint = + SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap] + ?.max || Number.MAX_SAFE_INTEGER + if (!query.range[field]) { + query.range[field] = { + low: type === "number" ? minint : "0000-00-00T00:00:00.000Z", + high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z", + } + } + if (operator === "rangeLow" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + low: value, + } + } else if (operator === "rangeHigh" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + high: value, + } + } + } else if (isLogicalSearchOperator(queryOperator)) { + // TODO + } else if (query[queryOperator] && operator !== "onEmptyFilter") { + if (type === "boolean") { + // Transform boolean filters to cope with null. + // "equals false" needs to be "not equals true" + // "not equals false" needs to be "equals true" + if (queryOperator === "equal" && value === false) { + query.notEqual = query.notEqual || {} + query.notEqual[field] = true + } else if (queryOperator === "notEqual" && value === false) { + query.equal = query.equal || {} + query.equal[field] = true + } else { + query[queryOperator] ??= {} + query[queryOperator]![field] = value + } + } else { + query[queryOperator] ??= {} + query[queryOperator]![field] = value + } + } + + return query +} + +export const buildQueryLegacy = ( + filter?: SearchFilterGroup | SearchFilter[] +): SearchFilters | undefined => { let query: SearchFilters = { string: {}, fuzzy: {}, @@ -368,13 +499,15 @@ export const buildQuery = (filter: SearchFilter[]) => { value = `${value}`?.toLowerCase() === "true" } if ( - ["contains", "notContains", "containsAny"].includes(operator) && + ["contains", "notContains", "containsAny"].includes( + operator.toLocaleString() + ) && type === "array" && typeof value === "string" ) { value = value.split(",") } - if (operator.startsWith("range") && query.range) { + if (operator.toLocaleString().startsWith("range") && query.range) { const minint = SqlNumberTypeRangeMap[ externalType as keyof typeof SqlNumberTypeRangeMap @@ -401,7 +534,7 @@ export const buildQuery = (filter: SearchFilter[]) => { } } } else if (isLogicalSearchOperator(queryOperator)) { - // TODO + // ignore } else if (query[queryOperator] && operator !== "onEmptyFilter") { if (type === "boolean") { // Transform boolean filters to cope with null. @@ -423,14 +556,68 @@ export const buildQuery = (filter: SearchFilter[]) => { } } }) - return query } +/** + * Converts a **SearchFilterGroup** filter definition into a grouped + * search query of type **SearchFilters** + * + * Legacy support remains for the old **SearchFilter[]** format. + * These will be migrated to an appropriate **SearchFilters** object, if encountered + * + * @param filter + * + * @returns {SearchFilters} + */ + +export const buildQuery = ( + filter?: SearchFilterGroup | SearchFilter[] +): SearchFilters | undefined => { + const parsedFilter: SearchFilterGroup | undefined = + processSearchFilters(filter) + + if (!parsedFilter) { + return + } + + const operatorMap: { [key in FilterGroupLogicalOperator]: LogicalOperator } = + { + [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND, + [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR, + } + + const globalOnEmpty = parsedFilter.onEmptyFilter + ? parsedFilter.onEmptyFilter + : null + + const globalOperator: LogicalOperator = + operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator] + + const coreRequest: SearchFilters = { + ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}), + [globalOperator]: { + conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => { + return { + [operatorMap[group.logicalOperator]]: { + conditions: group.filters + ?.map(x => buildCondition(x)) + .filter(filter => filter), + }, + } + }), + }, + } + return coreRequest +} + // The frontend can send single values for array fields sometimes, so to handle // this we convert them to arrays at the controller level so that nothing below // this has to worry about the non-array values. export function fixupFilterArrays(filters: SearchFilters) { + if (!filters) { + return filters + } for (const searchField of Object.values(ArrayOperator)) { const field = filters[searchField] if (field == null || !isPlainObject(field)) { diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index b69a059745..5b4d439984 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -1,4 +1,13 @@ +import { + SearchFilter, + SearchFilterGroup, + FilterGroupLogicalOperator, + SearchFilters, + BasicOperator, + ArrayOperator, +} from "@budibase/types" import * as Constants from "./constants" +import { removeKeyNumbering } from "./filters" export function unreachable( value: never, @@ -77,3 +86,129 @@ export function trimOtherProps(object: any, allowedProps: string[]) { ) return result } + +/** + * Processes the filter config. Filters are migrated from + * SearchFilter[] to SearchFilterGroup + * + * If filters is not an array, the migration is skipped + * + * @param {SearchFilter[] | SearchFilterGroup} filters + */ +export const processSearchFilters = ( + filters: SearchFilter[] | SearchFilterGroup | undefined +): SearchFilterGroup | undefined => { + if (!filters) { + return + } + + // Base search config. + const defaultCfg: SearchFilterGroup = { + logicalOperator: FilterGroupLogicalOperator.ALL, + groups: [], + } + + const filterWhitelistKeys = [ + "field", + "operator", + "value", + "type", + "externalType", + "valueType", + "noValue", + "formulaType", + ] + + if (Array.isArray(filters)) { + let baseGroup: SearchFilterGroup = { + filters: [], + logicalOperator: FilterGroupLogicalOperator.ALL, + } + + const migratedSetting: SearchFilterGroup = filters.reduce( + (acc: SearchFilterGroup, filter: SearchFilter) => { + // Sort the properties for easier debugging + const filterEntries = Object.entries(filter) + .sort((a, b) => { + return a[0].localeCompare(b[0]) + }) + .filter(x => x[1] ?? false) + + if (filterEntries.length == 1) { + const [key, value] = filterEntries[0] + // Global + if (key === "onEmptyFilter") { + // unset otherwise + acc.onEmptyFilter = value + } else if (key === "operator" && value === "allOr") { + // Group 1 logical operator + baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY + } + + return acc + } + + const whiteListedFilterSettings: [string, any][] = filterEntries.reduce( + (acc: [string, any][], entry: [string, any]) => { + const [key, value] = entry + + if (filterWhitelistKeys.includes(key)) { + if (key === "field") { + acc.push([key, removeKeyNumbering(value)]) + } else { + acc.push([key, value]) + } + } + return acc + }, + [] + ) + + const migratedFilter: SearchFilter = Object.fromEntries( + whiteListedFilterSettings + ) as SearchFilter + + baseGroup.filters!.push(migratedFilter) + + if (!acc.groups || !acc.groups.length) { + // init the base group + acc.groups = [baseGroup] + } + + return acc + }, + defaultCfg + ) + + return migratedSetting + } else if (!filters?.groups) { + return + } + return filters +} + +export function isSupportedUserSearch(query: SearchFilters) { + const allowed = [ + { op: BasicOperator.STRING, key: "email" }, + { op: BasicOperator.EQUAL, key: "_id" }, + { op: ArrayOperator.ONE_OF, key: "_id" }, + ] + for (let [key, operation] of Object.entries(query)) { + if (typeof operation !== "object") { + return false + } + const fields = Object.keys(operation || {}) + // this filter doesn't contain options - ignore + if (fields.length === 0) { + continue + } + const allowedOperation = allowed.find( + allow => + allow.op === key && fields.length === 1 && fields[0] === allow.key + ) + if (!allowedOperation) { + return false + } + } + return true +} diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts index 5223204a7f..b3d577f0c8 100644 --- a/packages/types/src/api/web/searchFilter.ts +++ b/packages/types/src/api/web/searchFilter.ts @@ -1,5 +1,9 @@ import { FieldType } from "../../documents" -import { EmptyFilterOption, SearchFilters } from "../../sdk" +import { + EmptyFilterOption, + FilterGroupLogicalOperator, + SearchFilters, +} from "../../sdk" export type SearchFilter = { operator: keyof SearchFilters | "rangeLow" | "rangeHigh" @@ -9,3 +13,10 @@ export type SearchFilter = { value: any externalType?: string } + +export type SearchFilterGroup = { + logicalOperator: FilterGroupLogicalOperator + onEmptyFilter?: EmptyFilterOption + groups?: SearchFilterGroup[] + filters?: SearchFilter[] +} diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index b847520526..f2d16b88b2 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -1,4 +1,4 @@ -import { SearchFilter, SortOrder, SortType } from "../../api" +import { SearchFilter, SearchFilterGroup, SortOrder, SortType } from "../../api" import { UIFieldMetadata } from "./table" import { Document } from "../document" import { DBView } from "../../sdk" @@ -61,7 +61,7 @@ export interface ViewV2 { name: string primaryDisplay?: string tableId: string - query?: SearchFilter[] + query?: SearchFilter[] | SearchFilterGroup sort?: { field: string order?: SortOrder diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 7d61aebdfb..bd67d1783a 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -191,6 +191,11 @@ export enum EmptyFilterOption { RETURN_NONE = "none", } +export enum FilterGroupLogicalOperator { + ALL = "all", + ANY = "any", +} + export enum SqlClient { MS_SQL = "mssql", POSTGRES = "pg", diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 6ce0eef5a0..921e0324d1 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -37,7 +37,7 @@ import { } from "@budibase/backend-core" import { checkAnyUserExists } from "../../../utilities/users" import { isEmailConfigured } from "../../../utilities/email" -import { BpmStatusKey, BpmStatusValue } from "@budibase/shared-core" +import { BpmStatusKey, BpmStatusValue, utils } from "@budibase/shared-core" const MAX_USERS_UPLOAD_LIMIT = 1000 @@ -256,7 +256,7 @@ export const search = async (ctx: Ctx) => { } } // Validate we aren't trying to search on any illegal fields - if (!userSdk.core.isSupportedUserSearch(body.query)) { + if (!utils.isSupportedUserSearch(body.query)) { ctx.throw(400, "Can only search by string.email, equal._id or oneOf._id") } } diff --git a/yarn.lock b/yarn.lock index 110cbd7a15..7f8d283276 100644 --- a/yarn.lock +++ b/yarn.lock @@ -20690,7 +20690,16 @@ string-similarity@^4.0.4: resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b" integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ== -"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -20781,7 +20790,7 @@ stringify-object@^3.2.1: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -20795,6 +20804,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -22755,7 +22771,7 @@ worker-farm@1.7.0: dependencies: errno "~0.1.7" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -22773,6 +22789,15 @@ wrap-ansi@^5.1.0: string-width "^3.0.0" strip-ansi "^5.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"