From 786bfdb0e2584c9e03ad54befa7202e5bd8354b5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 2 Oct 2024 18:09:53 +0100 Subject: [PATCH] Merging v3-backport branch --- .../server/src/api/controllers/row/views.ts | 38 ++-- .../src/api/controllers/view/viewsV2.ts | 4 +- packages/server/src/sdk/app/rows/search.ts | 45 ++-- .../server/src/sdk/app/rows/search/utils.ts | 4 +- packages/shared-core/src/filters.ts | 21 +- packages/shared-core/src/utils.ts | 201 +++++++++--------- packages/types/src/api/web/app/view.ts | 5 +- packages/types/src/api/web/searchFilter.ts | 5 +- packages/types/src/documents/app/view.ts | 8 +- 9 files changed, 176 insertions(+), 155 deletions(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index f804c650f9..b8d01424f2 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -3,6 +3,8 @@ import { ViewV2, SearchRowResponse, SearchViewRowRequest, + RequiredKeys, + RowSearchParams, } from "@budibase/types" import sdk from "../../../sdk" import { context } from "@budibase/backend-core" @@ -20,30 +22,34 @@ export async function searchView( ctx.throw(400, `This method only supports viewsV2`) } + const viewFields = Object.entries(view.schema || {}) + .filter(([_, value]) => value.visible) + .map(([key]) => key) const { body } = ctx.request await context.ensureSnippetContext(true) - const result = await sdk.rows.search( - { - viewId: view.id, - tableId: view.tableId, - query: body.query, - ...getSortOptions(body, view), - limit: body.limit, - bookmark: body.bookmark, - paginate: body.paginate, - countRows: body.countRows, - }, - { - user: sdk.users.getUserContextBindings(ctx.user), - } - ) + const searchOptions: RequiredKeys & + RequiredKeys< + Pick + > = { + tableId: view.tableId, + viewId: view.id, + query: body.query, + fields: viewFields, + ...getSortOptions(body, view), + limit: body.limit, + bookmark: body.bookmark, + paginate: body.paginate, + countRows: body.countRows, + } + const result = await sdk.rows.search(searchOptions, { + user: sdk.users.getUserContextBindings(ctx.user), + }) result.rows.forEach(r => (r._viewId = view.id)) ctx.body = result } - function getSortOptions(request: SearchViewRowRequest, view: ViewV2) { if (request.sort) { return { diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 7f6f638541..3df7172de2 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -99,7 +99,7 @@ export async function create(ctx: Ctx) { const schema = await parseSchema(view) - const parsedView: Omit, "id" | "version"> = { + const parsedView: Omit, "id" | "version" | "queryUI"> = { name: view.name, tableId: view.tableId, query: view.query, @@ -132,7 +132,7 @@ export async function update(ctx: Ctx) { const { tableId } = view const schema = await parseSchema(view) - const parsedView: RequiredKeys = { + const parsedView: RequiredKeys> = { id: view.id, name: view.name, version: view.version, diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index dae24c6bc0..7e73a51889 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -1,12 +1,10 @@ import { EmptyFilterOption, + LegacyFilter, LogicalOperator, Row, RowSearchParams, - SearchFilter, - SearchFilterGroup, SearchFilterKey, - SearchFilters, SearchResponse, SortOrder, Table, @@ -63,7 +61,6 @@ export async function search( if (options.viewId) { source = await sdk.views.get(options.viewId) table = await sdk.views.getTable(source) - options = searchInputMapping(table, options) } else if (options.tableId) { source = await sdk.tables.getTable(options.tableId) table = source @@ -76,7 +73,7 @@ export async function search( if (options.query) { const visibleFields = ( options.fields || Object.keys(table.schema) - ).filter(field => table.schema[field].visible !== false) + ).filter(field => table.schema[field]?.visible !== false) const queryableFields = await getQueryableFields(table, visibleFields) options.query = removeInvalidFilters(options.query, queryableFields) @@ -84,38 +81,52 @@ export async function search( options.query = {} } + // need to make sure filters in correct shape before checking for view + options = searchInputMapping(table, options) + if (options.viewId) { - const view = await sdk.views.get(options.viewId) + // Delete extraneous search params that cannot be overridden + delete options.query.onEmptyFilter + + const view = source as ViewV2 // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as // that could let users find rows they should not be allowed to access. - let viewQuery = dataFilters.buildQuery(view.query || []) + let viewQuery = dataFilters.buildQueryLegacy(view.query || []) + delete viewQuery?.onEmptyFilter - if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) { - // Lucene does not accept conditional filters, so we need to keep the old logic - const query: SearchFilters = viewQuery || {} - const viewFilters = view.query as SearchFilter[] + const sqsEnabled = await features.flags.isEnabled("SQS") + const supportsLogicalOperators = + isExternalTableID(view.tableId) || sqsEnabled + if (!supportsLogicalOperators) { + // In the unlikely event that a Grouped Filter is in a non-SQS environment + // It needs to be ignored entirely + let queryFilters: LegacyFilter[] = Array.isArray(view.query) + ? view.query + : [] // Extract existing fields const existingFields = - viewFilters + queryFilters ?.filter(filter => filter.field) .map(filter => db.removeKeyNumbering(filter.field)) || [] + viewQuery ??= {} // Carry over filters for unused fields - Object.keys(options.query || {}).forEach(key => { + Object.keys(options.query).forEach(key => { const operator = key as Exclude Object.keys(options.query[operator] || {}).forEach(field => { if (!existingFields.includes(db.removeKeyNumbering(field))) { - query[operator]![field] = options.query[operator]![field] + viewQuery![operator]![field] = options.query[operator]![field] } }) }) - options.query = query + options.query = viewQuery } else { + const conditions = viewQuery ? [viewQuery] : [] options.query = { $and: { - conditions: [viewQuery as SearchFilterGroup, options.query], + conditions: [...conditions, options.query], }, } } @@ -146,8 +157,6 @@ export async function search( options.sortOrder = options.sortOrder.toLowerCase() as SortOrder } - options = searchInputMapping(table, options) - let result: SearchResponse if (isExternalTable) { span?.addTags({ searchType: "external" }) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 1dba420a28..90303a6ca7 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -107,7 +107,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) { } return dataFilters.recurseLogicalOperators(filters, checkFilters) } - options.query = checkFilters(options.query) + if (options.query) { + options.query = checkFilters(options.query) + } return options } diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 23d7c7590c..fce355f7b9 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -3,7 +3,7 @@ import { BBReferenceFieldSubType, FieldType, FormulaType, - SearchFilter, + LegacyFilter, SearchFilters, SearchQueryFields, ArrayOperator, @@ -127,7 +127,7 @@ export function recurseLogicalOperators( fn: (f: SearchFilters) => SearchFilters ) { for (const logical of LOGICAL_OPERATORS) { - if (filters?.[logical]) { + if (filters[logical]) { filters[logical]!.conditions = filters[logical]!.conditions.map( condition => fn(condition) ) @@ -163,9 +163,6 @@ export function recurseSearchFilters( * https://github.com/Budibase/budibase/issues/10118 */ export const cleanupQuery = (query: SearchFilters) => { - if (!query) { - return query - } for (let filterField of NoEmptyFilterStrings) { if (!query[filterField]) { continue @@ -311,7 +308,7 @@ export class ColumnSplitter { * @param filter the builder filter structure */ -const buildCondition = (expression: SearchFilter) => { +const buildCondition = (expression: LegacyFilter) => { // Filter body let query: SearchFilters = { string: {}, @@ -437,8 +434,13 @@ const buildCondition = (expression: SearchFilter) => { } export const buildQueryLegacy = ( - filter?: SearchFilterGroup | SearchFilter[] + filter?: LegacyFilter[] | SearchFilters ): SearchFilters | undefined => { + // this is of type SearchFilters or is undefined + if (!Array.isArray(filter)) { + return filter + } + let query: SearchFilters = { string: {}, fuzzy: {}, @@ -572,7 +574,7 @@ export const buildQueryLegacy = ( */ export const buildQuery = ( - filter?: SearchFilterGroup | SearchFilter[] + filter?: SearchFilterGroup | LegacyFilter[] ): SearchFilters | undefined => { const parsedFilter: SearchFilterGroup | undefined = processSearchFilters(filter) @@ -594,7 +596,7 @@ export const buildQuery = ( const globalOperator: LogicalOperator = operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator] - const coreRequest: SearchFilters = { + return { ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}), [globalOperator]: { conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => { @@ -608,7 +610,6 @@ export const buildQuery = ( }), }, } - return coreRequest } // The frontend can send single values for array fields sometimes, so to handle diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index 5b4d439984..14b3c84425 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -1,5 +1,5 @@ import { - SearchFilter, + LegacyFilter, SearchFilterGroup, FilterGroupLogicalOperator, SearchFilters, @@ -9,6 +9,10 @@ import { import * as Constants from "./constants" import { removeKeyNumbering } from "./filters" +// an array of keys from filter type to properties that are in the type +// this can then be converted using .fromEntries to an object +type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][] + export function unreachable( value: never, message = `No such case in exhaustive switch: ${value}` @@ -87,106 +91,6 @@ export function trimOtherProps(object: any, allowedProps: string[]) { return result } -/** - * Processes the filter config. Filters are migrated from - * SearchFilter[] to SearchFilterGroup - * - * If filters is not an array, the migration is skipped - * - * @param {SearchFilter[] | SearchFilterGroup} filters - */ -export const processSearchFilters = ( - filters: SearchFilter[] | SearchFilterGroup | undefined -): SearchFilterGroup | undefined => { - if (!filters) { - return - } - - // Base search config. - const defaultCfg: SearchFilterGroup = { - logicalOperator: FilterGroupLogicalOperator.ALL, - groups: [], - } - - const filterWhitelistKeys = [ - "field", - "operator", - "value", - "type", - "externalType", - "valueType", - "noValue", - "formulaType", - ] - - if (Array.isArray(filters)) { - let baseGroup: SearchFilterGroup = { - filters: [], - logicalOperator: FilterGroupLogicalOperator.ALL, - } - - const migratedSetting: SearchFilterGroup = filters.reduce( - (acc: SearchFilterGroup, filter: SearchFilter) => { - // Sort the properties for easier debugging - const filterEntries = Object.entries(filter) - .sort((a, b) => { - return a[0].localeCompare(b[0]) - }) - .filter(x => x[1] ?? false) - - if (filterEntries.length == 1) { - const [key, value] = filterEntries[0] - // Global - if (key === "onEmptyFilter") { - // unset otherwise - acc.onEmptyFilter = value - } else if (key === "operator" && value === "allOr") { - // Group 1 logical operator - baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY - } - - return acc - } - - const whiteListedFilterSettings: [string, any][] = filterEntries.reduce( - (acc: [string, any][], entry: [string, any]) => { - const [key, value] = entry - - if (filterWhitelistKeys.includes(key)) { - if (key === "field") { - acc.push([key, removeKeyNumbering(value)]) - } else { - acc.push([key, value]) - } - } - return acc - }, - [] - ) - - const migratedFilter: SearchFilter = Object.fromEntries( - whiteListedFilterSettings - ) as SearchFilter - - baseGroup.filters!.push(migratedFilter) - - if (!acc.groups || !acc.groups.length) { - // init the base group - acc.groups = [baseGroup] - } - - return acc - }, - defaultCfg - ) - - return migratedSetting - } else if (!filters?.groups) { - return - } - return filters -} - export function isSupportedUserSearch(query: SearchFilters) { const allowed = [ { op: BasicOperator.STRING, key: "email" }, @@ -212,3 +116,98 @@ export function isSupportedUserSearch(query: SearchFilters) { } return true } + +/** + * Processes the filter config. Filters are migrated from + * SearchFilter[] to SearchFilterGroup + * + * If filters is not an array, the migration is skipped + * + * @param {LegacyFilter[] | SearchFilterGroup} filters + */ +export const processSearchFilters = ( + filters: LegacyFilter[] | SearchFilterGroup | undefined +): SearchFilterGroup | undefined => { + if (!filters) { + return + } + + // Base search config. + const defaultCfg: SearchFilterGroup = { + logicalOperator: FilterGroupLogicalOperator.ALL, + groups: [], + } + + const filterAllowedKeys = [ + "field", + "operator", + "value", + "type", + "externalType", + "valueType", + "noValue", + "formulaType", + ] + + if (Array.isArray(filters)) { + let baseGroup: SearchFilterGroup = { + filters: [], + logicalOperator: FilterGroupLogicalOperator.ALL, + } + + return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => { + // Sort the properties for easier debugging + const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[]) + .sort((a, b) => { + return a.localeCompare(b) + }) + .filter(key => key in filter) + + if (filterPropertyKeys.length == 1) { + const key = filterPropertyKeys[0], + value = filter[key] + // Global + if (key === "onEmptyFilter") { + // unset otherwise + acc.onEmptyFilter = value + } else if (key === "operator" && value === "allOr") { + // Group 1 logical operator + baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY + } + + return acc + } + + const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce( + (acc: AllowedFilters, key) => { + const value = filter[key] + if (filterAllowedKeys.includes(key)) { + if (key === "field") { + acc.push([key, removeKeyNumbering(value)]) + } else { + acc.push([key, value]) + } + } + return acc + }, + [] + ) + + const migratedFilter: LegacyFilter = Object.fromEntries( + allowedFilterSettings + ) as LegacyFilter + + baseGroup.filters!.push(migratedFilter) + + if (!acc.groups || !acc.groups.length) { + // init the base group + acc.groups = [baseGroup] + } + + return acc + }, defaultCfg) + } else if (!filters?.groups) { + return + } + return filters +} diff --git a/packages/types/src/api/web/app/view.ts b/packages/types/src/api/web/app/view.ts index a6be5e2986..a99f2938ab 100644 --- a/packages/types/src/api/web/app/view.ts +++ b/packages/types/src/api/web/app/view.ts @@ -9,6 +9,7 @@ export interface ViewResponseEnriched { data: ViewV2Enriched } -export interface CreateViewRequest extends Omit {} +export interface CreateViewRequest + extends Omit {} -export interface UpdateViewRequest extends ViewV2 {} +export interface UpdateViewRequest extends Omit {} diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts index b3d577f0c8..23c599027e 100644 --- a/packages/types/src/api/web/searchFilter.ts +++ b/packages/types/src/api/web/searchFilter.ts @@ -5,7 +5,7 @@ import { SearchFilters, } from "../../sdk" -export type SearchFilter = { +export type LegacyFilter = { operator: keyof SearchFilters | "rangeLow" | "rangeHigh" onEmptyFilter?: EmptyFilterOption field: string @@ -14,9 +14,10 @@ export type SearchFilter = { externalType?: string } +// this is a type purely used by the UI export type SearchFilterGroup = { logicalOperator: FilterGroupLogicalOperator onEmptyFilter?: EmptyFilterOption groups?: SearchFilterGroup[] - filters?: SearchFilter[] + filters?: LegacyFilter[] } diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index 16817f177a..87667a71e0 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -1,7 +1,7 @@ -import { SearchFilter, SearchFilterGroup, SortOrder, SortType } from "../../api" +import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api" import { UIFieldMetadata } from "./table" import { Document } from "../document" -import { DBView } from "../../sdk" +import { DBView, SearchFilters } from "../../sdk" export type ViewTemplateOpts = { field: string @@ -65,7 +65,9 @@ export interface ViewV2 { name: string primaryDisplay?: string tableId: string - query?: SearchFilter[] | SearchFilterGroup + query?: LegacyFilter[] | SearchFilters + // duplicate to store UI information about filters + queryUI?: SearchFilterGroup sort?: { field: string order?: SortOrder