From 6e660151bdef6cd5c9130e09a695bc47b80d4b0c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 30 Sep 2024 18:06:47 +0100 Subject: [PATCH 01/14] backport of V3 backend changes for search filters on view, giving this the correct type to support conditionals. --- .../server/src/api/controllers/row/views.ts | 48 ++-- .../src/api/controllers/view/viewsV2.ts | 4 +- packages/shared-core/src/filters.ts | 208 +++++++++++++++++- packages/shared-core/src/utils.ts | 110 ++++++++- packages/types/src/api/web/app/view.ts | 5 +- packages/types/src/api/web/searchFilter.ts | 16 +- packages/types/src/documents/app/view.ts | 8 +- packages/types/src/sdk/search.ts | 5 + 8 files changed, 372 insertions(+), 32 deletions(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 68958da8e7..398121f49b 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -5,6 +5,9 @@ import { SearchViewRowRequest, SearchFilterKey, LogicalOperator, + RequiredKeys, + RowSearchParams, + LegacyFilter, } from "@budibase/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../../sdk" @@ -17,7 +20,7 @@ export async function searchView( ) { const { viewId } = ctx.params - const view = await sdk.views.get(viewId) + const view: ViewV2 = await sdk.views.get(viewId) if (!view) { ctx.throw(404, `View ${viewId} not found`) } @@ -25,23 +28,35 @@ export async function searchView( ctx.throw(400, `This method only supports viewsV2`) } + const viewFields = Object.entries(view.schema || {}) + .filter(([_, value]) => value.visible) + .map(([key]) => key) const { body } = ctx.request + const sqsEnabled = await features.flags.isEnabled("SQS") + const supportsLogicalOperators = isExternalTableID(view.tableId) || sqsEnabled + // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as // that could let users find rows they should not be allowed to access. - let query = dataFilters.buildQuery(view.query || []) + let query = dataFilters.buildQueryLegacy(view.query) + + delete query?.onEmptyFilter + if (body.query) { // Delete extraneous search params that cannot be overridden delete body.query.onEmptyFilter - if ( - !isExternalTableID(view.tableId) && - !(await features.flags.isEnabled("SQS")) - ) { + if (!supportsLogicalOperators) { + // In the unlikely event that a Grouped Filter is in a non-SQS environment + // It needs to be ignored entirely + let queryFilters: LegacyFilter[] = Array.isArray(view.query) + ? view.query + : [] + // Extract existing fields const existingFields = - view.query + queryFilters ?.filter(filter => filter.field) .map(filter => db.removeKeyNumbering(filter.field)) || [] @@ -49,15 +64,16 @@ export async function searchView( Object.keys(body.query).forEach(key => { const operator = key as Exclude Object.keys(body.query[operator] || {}).forEach(field => { - if (!existingFields.includes(db.removeKeyNumbering(field))) { + if (query && !existingFields.includes(db.removeKeyNumbering(field))) { query[operator]![field] = body.query[operator]![field] } }) }) } else { + const conditions = query ? [query] : [] query = { $and: { - conditions: [query, body.query], + conditions: [...conditions, body.query], }, } } @@ -65,25 +81,29 @@ export async function searchView( await context.ensureSnippetContext(true) - const enrichedQuery = await enrichSearchContext(query, { + const enrichedQuery = await enrichSearchContext(query || {}, { user: sdk.users.getUserContextBindings(ctx.user), }) - const result = await sdk.rows.search({ - viewId: view.id, + const searchOptions: RequiredKeys & + RequiredKeys< + Pick + > = { tableId: view.tableId, + viewId: view.id, query: enrichedQuery, + fields: viewFields, ...getSortOptions(body, view), limit: body.limit, bookmark: body.bookmark, paginate: body.paginate, countRows: body.countRows, - }) + } + const result = await sdk.rows.search(searchOptions) result.rows.forEach(r => (r._viewId = view.id)) ctx.body = result } - function getSortOptions(request: SearchViewRowRequest, view: ViewV2) { if (request.sort) { return { diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 7f6f638541..3df7172de2 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -99,7 +99,7 @@ export async function create(ctx: Ctx) { const schema = await parseSchema(view) - const parsedView: Omit, "id" | "version"> = { + const parsedView: Omit, "id" | "version" | "queryUI"> = { name: view.name, tableId: view.tableId, query: view.query, @@ -132,7 +132,7 @@ export async function update(ctx: Ctx) { const { tableId } = view const schema = await parseSchema(view) - const parsedView: RequiredKeys = { + const parsedView: RequiredKeys> = { id: view.id, name: view.name, version: view.version, diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index ef0500b01a..18ce4b6ed7 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -3,7 +3,7 @@ import { BBReferenceFieldSubType, FieldType, FormulaType, - SearchFilter, + LegacyFilter, SearchFilters, SearchQueryFields, ArrayOperator, @@ -19,9 +19,12 @@ import { RangeOperator, LogicalOperator, isLogicalSearchOperator, + SearchFilterGroup, + FilterGroupLogicalOperator, } from "@budibase/types" import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" +import { processSearchFilters } from "./utils" import { deepGet, schema } from "./helpers" import { isPlainObject, isEmpty } from "lodash" import { decodeNonAscii } from "./helpers/schema" @@ -124,7 +127,7 @@ export function recurseLogicalOperators( fn: (f: SearchFilters) => SearchFilters ) { for (const logical of LOGICAL_OPERATORS) { - if (filters?.[logical]) { + if (filters[logical]) { filters[logical]!.conditions = filters[logical]!.conditions.map( condition => fn(condition) ) @@ -304,10 +307,143 @@ export class ColumnSplitter { } /** - * Builds a JSON query from the filter structure generated in the builder + * Builds a JSON query from the filter a SearchFilter definition * @param filter the builder filter structure */ -export const buildQuery = (filter: SearchFilter[]) => { + +const buildCondition = (expression: LegacyFilter) => { + // Filter body + let query: SearchFilters = { + string: {}, + fuzzy: {}, + range: {}, + equal: {}, + notEqual: {}, + empty: {}, + notEmpty: {}, + contains: {}, + notContains: {}, + oneOf: {}, + containsAny: {}, + } + let { operator, field, type, value, externalType, onEmptyFilter } = expression + + if (!operator || !field) { + return + } + + const queryOperator = operator as SearchFilterOperator + const isHbs = + typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0 + // Parse all values into correct types + if (operator === "allOr") { + query.allOr = true + return + } + if (onEmptyFilter) { + query.onEmptyFilter = onEmptyFilter + return + } + + // Default the value for noValue fields to ensure they are correctly added + // to the final query + if (queryOperator === "empty" || queryOperator === "notEmpty") { + value = null + } + + if ( + type === "datetime" && + !isHbs && + queryOperator !== "empty" && + queryOperator !== "notEmpty" + ) { + // Ensure date value is a valid date and parse into correct format + if (!value) { + return + } + try { + value = new Date(value).toISOString() + } catch (error) { + return + } + } + if (type === "number" && typeof value === "string" && !isHbs) { + if (queryOperator === "oneOf") { + value = value.split(",").map(item => parseFloat(item)) + } else { + value = parseFloat(value) + } + } + if (type === "boolean") { + value = `${value}`?.toLowerCase() === "true" + } + if ( + ["contains", "notContains", "containsAny"].includes( + operator.toLocaleString() + ) && + type === "array" && + typeof value === "string" + ) { + value = value.split(",") + } + if (operator.toLocaleString().startsWith("range") && query.range) { + const minint = + SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap] + ?.min || Number.MIN_SAFE_INTEGER + const maxint = + SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap] + ?.max || Number.MAX_SAFE_INTEGER + if (!query.range[field]) { + query.range[field] = { + low: type === "number" ? minint : "0000-00-00T00:00:00.000Z", + high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z", + } + } + if (operator === "rangeLow" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + low: value, + } + } else if (operator === "rangeHigh" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + high: value, + } + } + } else if (isLogicalSearchOperator(queryOperator)) { + // TODO + } else if (query[queryOperator] && operator !== "onEmptyFilter") { + if (type === "boolean") { + // Transform boolean filters to cope with null. + // "equals false" needs to be "not equals true" + // "not equals false" needs to be "equals true" + if (queryOperator === "equal" && value === false) { + query.notEqual = query.notEqual || {} + query.notEqual[field] = true + } else if (queryOperator === "notEqual" && value === false) { + query.equal = query.equal || {} + query.equal[field] = true + } else { + query[queryOperator] ??= {} + query[queryOperator]![field] = value + } + } else { + query[queryOperator] ??= {} + query[queryOperator]![field] = value + } + } + + return query +} + +export const buildQueryLegacy = ( + filter?: LegacyFilter[] | SearchFilters +): SearchFilters | undefined => { + // this is of type SearchFilters or is undefined + if (!Array.isArray(filter)) { + return filter + } + let query: SearchFilters = { string: {}, fuzzy: {}, @@ -368,13 +504,15 @@ export const buildQuery = (filter: SearchFilter[]) => { value = `${value}`?.toLowerCase() === "true" } if ( - ["contains", "notContains", "containsAny"].includes(operator) && + ["contains", "notContains", "containsAny"].includes( + operator.toLocaleString() + ) && type === "array" && typeof value === "string" ) { value = value.split(",") } - if (operator.startsWith("range") && query.range) { + if (operator.toLocaleString().startsWith("range") && query.range) { const minint = SqlNumberTypeRangeMap[ externalType as keyof typeof SqlNumberTypeRangeMap @@ -401,7 +539,7 @@ export const buildQuery = (filter: SearchFilter[]) => { } } } else if (isLogicalSearchOperator(queryOperator)) { - // TODO + // ignore } else if (query[queryOperator] && operator !== "onEmptyFilter") { if (type === "boolean") { // Transform boolean filters to cope with null. @@ -423,14 +561,68 @@ export const buildQuery = (filter: SearchFilter[]) => { } } }) - return query } +/** + * Converts a **SearchFilterGroup** filter definition into a grouped + * search query of type **SearchFilters** + * + * Legacy support remains for the old **SearchFilter[]** format. + * These will be migrated to an appropriate **SearchFilters** object, if encountered + * + * @param filter + * + * @returns {SearchFilters} + */ + +export const buildQuery = ( + filter?: SearchFilterGroup | LegacyFilter[] +): SearchFilters | undefined => { + const parsedFilter: SearchFilterGroup | undefined = + processSearchFilters(filter) + + if (!parsedFilter) { + return + } + + const operatorMap: { [key in FilterGroupLogicalOperator]: LogicalOperator } = + { + [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND, + [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR, + } + + const globalOnEmpty = parsedFilter.onEmptyFilter + ? parsedFilter.onEmptyFilter + : null + + const globalOperator: LogicalOperator = + operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator] + + const coreRequest: SearchFilters = { + ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}), + [globalOperator]: { + conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => { + return { + [operatorMap[group.logicalOperator]]: { + conditions: group.filters + ?.map(x => buildCondition(x)) + .filter(filter => filter), + }, + } + }), + }, + } + return coreRequest +} + // The frontend can send single values for array fields sometimes, so to handle // this we convert them to arrays at the controller level so that nothing below // this has to worry about the non-array values. export function fixupFilterArrays(filters: SearchFilters) { + if (!filters) { + return filters + } for (const searchField of Object.values(ArrayOperator)) { const field = filters[searchField] if (field == null || !isPlainObject(field)) { diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index 81fab659c6..b441791751 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -1,5 +1,20 @@ -import { ArrayOperator, BasicOperator, SearchFilters } from "@budibase/types" +import { + LegacyFilter, + SearchFilterGroup, + FilterGroupLogicalOperator, + SearchFilters, + BasicOperator, + ArrayOperator, +} from "@budibase/types" import * as Constants from "./constants" +import { removeKeyNumbering } from "./filters" + +// an array of keys from filter type to properties that are in the type +// this can then be converted using .fromEntries to an object +type WhitelistedFilters = [ + keyof LegacyFilter, + LegacyFilter[keyof LegacyFilter] +][] export function unreachable( value: never, @@ -104,3 +119,96 @@ export function isSupportedUserSearch(query: SearchFilters) { } return true } + +/** + * Processes the filter config. Filters are migrated from + * SearchFilter[] to SearchFilterGroup + * + * If filters is not an array, the migration is skipped + * + * @param {LegacyFilter[] | SearchFilterGroup} filters + */ +export const processSearchFilters = ( + filters: LegacyFilter[] | SearchFilterGroup | undefined +): SearchFilterGroup | undefined => { + if (!filters) { + return + } + + // Base search config. + const defaultCfg: SearchFilterGroup = { + logicalOperator: FilterGroupLogicalOperator.ALL, + groups: [], + } + + const filterWhitelistKeys = [ + "field", + "operator", + "value", + "type", + "externalType", + "valueType", + "noValue", + "formulaType", + ] + + if (Array.isArray(filters)) { + let baseGroup: SearchFilterGroup = { + filters: [], + logicalOperator: FilterGroupLogicalOperator.ALL, + } + + return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => { + // Sort the properties for easier debugging + const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[]) + .sort((a, b) => { + return a.localeCompare(b) + }) + .filter(key => key in filter) + + if (filterPropertyKeys.length == 1) { + const key = filterPropertyKeys[0], + value = filter[key] + // Global + if (key === "onEmptyFilter") { + // unset otherwise + acc.onEmptyFilter = value + } else if (key === "operator" && value === "allOr") { + // Group 1 logical operator + baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY + } + + return acc + } + + const whiteListedFilterSettings: WhitelistedFilters = + filterPropertyKeys.reduce((acc: WhitelistedFilters, key) => { + const value = filter[key] + if (filterWhitelistKeys.includes(key)) { + if (key === "field") { + acc.push([key, removeKeyNumbering(value)]) + } else { + acc.push([key, value]) + } + } + return acc + }, []) + + const migratedFilter: LegacyFilter = Object.fromEntries( + whiteListedFilterSettings + ) as LegacyFilter + + baseGroup.filters!.push(migratedFilter) + + if (!acc.groups || !acc.groups.length) { + // init the base group + acc.groups = [baseGroup] + } + + return acc + }, defaultCfg) + } else if (!filters?.groups) { + return + } + return filters +} diff --git a/packages/types/src/api/web/app/view.ts b/packages/types/src/api/web/app/view.ts index a6be5e2986..a99f2938ab 100644 --- a/packages/types/src/api/web/app/view.ts +++ b/packages/types/src/api/web/app/view.ts @@ -9,6 +9,7 @@ export interface ViewResponseEnriched { data: ViewV2Enriched } -export interface CreateViewRequest extends Omit {} +export interface CreateViewRequest + extends Omit {} -export interface UpdateViewRequest extends ViewV2 {} +export interface UpdateViewRequest extends Omit {} diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts index 5223204a7f..23c599027e 100644 --- a/packages/types/src/api/web/searchFilter.ts +++ b/packages/types/src/api/web/searchFilter.ts @@ -1,7 +1,11 @@ import { FieldType } from "../../documents" -import { EmptyFilterOption, SearchFilters } from "../../sdk" +import { + EmptyFilterOption, + FilterGroupLogicalOperator, + SearchFilters, +} from "../../sdk" -export type SearchFilter = { +export type LegacyFilter = { operator: keyof SearchFilters | "rangeLow" | "rangeHigh" onEmptyFilter?: EmptyFilterOption field: string @@ -9,3 +13,11 @@ export type SearchFilter = { value: any externalType?: string } + +// this is a type purely used by the UI +export type SearchFilterGroup = { + logicalOperator: FilterGroupLogicalOperator + onEmptyFilter?: EmptyFilterOption + groups?: SearchFilterGroup[] + filters?: LegacyFilter[] +} diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index a957564039..87667a71e0 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -1,7 +1,7 @@ -import { SearchFilter, SortOrder, SortType } from "../../api" +import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api" import { UIFieldMetadata } from "./table" import { Document } from "../document" -import { DBView } from "../../sdk" +import { DBView, SearchFilters } from "../../sdk" export type ViewTemplateOpts = { field: string @@ -65,7 +65,9 @@ export interface ViewV2 { name: string primaryDisplay?: string tableId: string - query?: SearchFilter[] + query?: LegacyFilter[] | SearchFilters + // duplicate to store UI information about filters + queryUI?: SearchFilterGroup sort?: { field: string order?: SortOrder diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 647a9e7d00..d41bb0fb99 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -191,6 +191,11 @@ export enum EmptyFilterOption { RETURN_NONE = "none", } +export enum FilterGroupLogicalOperator { + ALL = "all", + ANY = "any", +} + export enum SqlClient { MS_SQL = "mssql", POSTGRES = "pg", From 975e348de5b9bed62030ed73b7db00cef700df0d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 10:25:15 +0100 Subject: [PATCH 02/14] Check options.fields are in the table. --- packages/server/src/sdk/app/rows/search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 809bd73d1f..eb04e9fe62 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -89,7 +89,7 @@ export async function search( if (options.query) { const visibleFields = ( options.fields || Object.keys(table.schema) - ).filter(field => table.schema[field].visible !== false) + ).filter(field => table.schema[field]?.visible !== false) const queryableFields = await getQueryableFields(table, visibleFields) options.query = removeInvalidFilters(options.query, queryableFields) From 119767a30e970220a64f7aa8a43c25cc349c56b0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 1 Oct 2024 12:20:18 +0200 Subject: [PATCH 03/14] Cleanup --- packages/server/src/api/controllers/row/views.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index b8d01424f2..622688deb6 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -14,7 +14,7 @@ export async function searchView( ) { const { viewId } = ctx.params - const view: ViewV2 = await sdk.views.get(viewId) + const view = await sdk.views.get(viewId) if (!view) { ctx.throw(404, `View ${viewId} not found`) } From 522941abf004cae1e04f650a0128df73aca2814a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 11:31:41 +0100 Subject: [PATCH 04/14] PR comments. --- packages/shared-core/src/filters.ts | 9 +-------- packages/shared-core/src/utils.ts | 19 +++++++++---------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 18ce4b6ed7..b10375acb0 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -163,9 +163,6 @@ export function recurseSearchFilters( * https://github.com/Budibase/budibase/issues/10118 */ export const cleanupQuery = (query: SearchFilters) => { - if (!query) { - return query - } for (let filterField of NoEmptyFilterStrings) { if (!query[filterField]) { continue @@ -599,7 +596,7 @@ export const buildQuery = ( const globalOperator: LogicalOperator = operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator] - const coreRequest: SearchFilters = { + return { ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}), [globalOperator]: { conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => { @@ -613,16 +610,12 @@ export const buildQuery = ( }), }, } - return coreRequest } // The frontend can send single values for array fields sometimes, so to handle // this we convert them to arrays at the controller level so that nothing below // this has to worry about the non-array values. export function fixupFilterArrays(filters: SearchFilters) { - if (!filters) { - return filters - } for (const searchField of Object.values(ArrayOperator)) { const field = filters[searchField] if (field == null || !isPlainObject(field)) { diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index b441791751..14b3c84425 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -11,10 +11,7 @@ import { removeKeyNumbering } from "./filters" // an array of keys from filter type to properties that are in the type // this can then be converted using .fromEntries to an object -type WhitelistedFilters = [ - keyof LegacyFilter, - LegacyFilter[keyof LegacyFilter] -][] +type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][] export function unreachable( value: never, @@ -141,7 +138,7 @@ export const processSearchFilters = ( groups: [], } - const filterWhitelistKeys = [ + const filterAllowedKeys = [ "field", "operator", "value", @@ -181,10 +178,10 @@ export const processSearchFilters = ( return acc } - const whiteListedFilterSettings: WhitelistedFilters = - filterPropertyKeys.reduce((acc: WhitelistedFilters, key) => { + const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce( + (acc: AllowedFilters, key) => { const value = filter[key] - if (filterWhitelistKeys.includes(key)) { + if (filterAllowedKeys.includes(key)) { if (key === "field") { acc.push([key, removeKeyNumbering(value)]) } else { @@ -192,10 +189,12 @@ export const processSearchFilters = ( } } return acc - }, []) + }, + [] + ) const migratedFilter: LegacyFilter = Object.fromEntries( - whiteListedFilterSettings + allowedFilterSettings ) as LegacyFilter baseGroup.filters!.push(migratedFilter) From 19407d5e37bc4a2e937c4e153a8bdf74fd46a83f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 11:38:02 +0100 Subject: [PATCH 05/14] Check filters have been provided. --- packages/server/src/sdk/app/rows/search/utils.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 1dba420a28..90303a6ca7 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -107,7 +107,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) { } return dataFilters.recurseLogicalOperators(filters, checkFilters) } - options.query = checkFilters(options.query) + if (options.query) { + options.query = checkFilters(options.query) + } return options } From d7873c5c6e5cd6908f4fc75bc44a53b9af6436aa Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 11:42:16 +0100 Subject: [PATCH 06/14] Test fix. --- .../server/src/api/routes/tests/search.spec.ts | 14 +++++++------- packages/server/src/sdk/app/rows/search.ts | 5 +++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 1ec5ca792a..092a851e14 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -45,14 +45,14 @@ import { generateRowIdField } from "../../../integrations/utils" import { cloneDeep } from "lodash/fp" describe.each([ - ["in-memory", undefined], - ["lucene", undefined], + // ["in-memory", undefined], + // ["lucene", undefined], ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 87129fdbc8..7e73a51889 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -81,12 +81,13 @@ export async function search( options.query = {} } + // need to make sure filters in correct shape before checking for view + options = searchInputMapping(table, options) + if (options.viewId) { // Delete extraneous search params that cannot be overridden delete options.query.onEmptyFilter - options = searchInputMapping(table, options) - const view = source as ViewV2 // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as From 4d33106b450781573083fd6359f6c1a3297e374e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 1 Oct 2024 11:42:44 +0100 Subject: [PATCH 07/14] Undo commenting out other DBs. --- .../server/src/api/routes/tests/search.spec.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 092a851e14..1ec5ca792a 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -45,14 +45,14 @@ import { generateRowIdField } from "../../../integrations/utils" import { cloneDeep } from "lodash/fp" describe.each([ - // ["in-memory", undefined], - // ["lucene", undefined], + ["in-memory", undefined], + ["lucene", undefined], ["sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" From 786bfdb0e2584c9e03ad54befa7202e5bd8354b5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 2 Oct 2024 18:09:53 +0100 Subject: [PATCH 08/14] Merging v3-backport branch --- .../server/src/api/controllers/row/views.ts | 38 ++-- .../src/api/controllers/view/viewsV2.ts | 4 +- packages/server/src/sdk/app/rows/search.ts | 45 ++-- .../server/src/sdk/app/rows/search/utils.ts | 4 +- packages/shared-core/src/filters.ts | 21 +- packages/shared-core/src/utils.ts | 201 +++++++++--------- packages/types/src/api/web/app/view.ts | 5 +- packages/types/src/api/web/searchFilter.ts | 5 +- packages/types/src/documents/app/view.ts | 8 +- 9 files changed, 176 insertions(+), 155 deletions(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index f804c650f9..b8d01424f2 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -3,6 +3,8 @@ import { ViewV2, SearchRowResponse, SearchViewRowRequest, + RequiredKeys, + RowSearchParams, } from "@budibase/types" import sdk from "../../../sdk" import { context } from "@budibase/backend-core" @@ -20,30 +22,34 @@ export async function searchView( ctx.throw(400, `This method only supports viewsV2`) } + const viewFields = Object.entries(view.schema || {}) + .filter(([_, value]) => value.visible) + .map(([key]) => key) const { body } = ctx.request await context.ensureSnippetContext(true) - const result = await sdk.rows.search( - { - viewId: view.id, - tableId: view.tableId, - query: body.query, - ...getSortOptions(body, view), - limit: body.limit, - bookmark: body.bookmark, - paginate: body.paginate, - countRows: body.countRows, - }, - { - user: sdk.users.getUserContextBindings(ctx.user), - } - ) + const searchOptions: RequiredKeys & + RequiredKeys< + Pick + > = { + tableId: view.tableId, + viewId: view.id, + query: body.query, + fields: viewFields, + ...getSortOptions(body, view), + limit: body.limit, + bookmark: body.bookmark, + paginate: body.paginate, + countRows: body.countRows, + } + const result = await sdk.rows.search(searchOptions, { + user: sdk.users.getUserContextBindings(ctx.user), + }) result.rows.forEach(r => (r._viewId = view.id)) ctx.body = result } - function getSortOptions(request: SearchViewRowRequest, view: ViewV2) { if (request.sort) { return { diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 7f6f638541..3df7172de2 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -99,7 +99,7 @@ export async function create(ctx: Ctx) { const schema = await parseSchema(view) - const parsedView: Omit, "id" | "version"> = { + const parsedView: Omit, "id" | "version" | "queryUI"> = { name: view.name, tableId: view.tableId, query: view.query, @@ -132,7 +132,7 @@ export async function update(ctx: Ctx) { const { tableId } = view const schema = await parseSchema(view) - const parsedView: RequiredKeys = { + const parsedView: RequiredKeys> = { id: view.id, name: view.name, version: view.version, diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index dae24c6bc0..7e73a51889 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -1,12 +1,10 @@ import { EmptyFilterOption, + LegacyFilter, LogicalOperator, Row, RowSearchParams, - SearchFilter, - SearchFilterGroup, SearchFilterKey, - SearchFilters, SearchResponse, SortOrder, Table, @@ -63,7 +61,6 @@ export async function search( if (options.viewId) { source = await sdk.views.get(options.viewId) table = await sdk.views.getTable(source) - options = searchInputMapping(table, options) } else if (options.tableId) { source = await sdk.tables.getTable(options.tableId) table = source @@ -76,7 +73,7 @@ export async function search( if (options.query) { const visibleFields = ( options.fields || Object.keys(table.schema) - ).filter(field => table.schema[field].visible !== false) + ).filter(field => table.schema[field]?.visible !== false) const queryableFields = await getQueryableFields(table, visibleFields) options.query = removeInvalidFilters(options.query, queryableFields) @@ -84,38 +81,52 @@ export async function search( options.query = {} } + // need to make sure filters in correct shape before checking for view + options = searchInputMapping(table, options) + if (options.viewId) { - const view = await sdk.views.get(options.viewId) + // Delete extraneous search params that cannot be overridden + delete options.query.onEmptyFilter + + const view = source as ViewV2 // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as // that could let users find rows they should not be allowed to access. - let viewQuery = dataFilters.buildQuery(view.query || []) + let viewQuery = dataFilters.buildQueryLegacy(view.query || []) + delete viewQuery?.onEmptyFilter - if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) { - // Lucene does not accept conditional filters, so we need to keep the old logic - const query: SearchFilters = viewQuery || {} - const viewFilters = view.query as SearchFilter[] + const sqsEnabled = await features.flags.isEnabled("SQS") + const supportsLogicalOperators = + isExternalTableID(view.tableId) || sqsEnabled + if (!supportsLogicalOperators) { + // In the unlikely event that a Grouped Filter is in a non-SQS environment + // It needs to be ignored entirely + let queryFilters: LegacyFilter[] = Array.isArray(view.query) + ? view.query + : [] // Extract existing fields const existingFields = - viewFilters + queryFilters ?.filter(filter => filter.field) .map(filter => db.removeKeyNumbering(filter.field)) || [] + viewQuery ??= {} // Carry over filters for unused fields - Object.keys(options.query || {}).forEach(key => { + Object.keys(options.query).forEach(key => { const operator = key as Exclude Object.keys(options.query[operator] || {}).forEach(field => { if (!existingFields.includes(db.removeKeyNumbering(field))) { - query[operator]![field] = options.query[operator]![field] + viewQuery![operator]![field] = options.query[operator]![field] } }) }) - options.query = query + options.query = viewQuery } else { + const conditions = viewQuery ? [viewQuery] : [] options.query = { $and: { - conditions: [viewQuery as SearchFilterGroup, options.query], + conditions: [...conditions, options.query], }, } } @@ -146,8 +157,6 @@ export async function search( options.sortOrder = options.sortOrder.toLowerCase() as SortOrder } - options = searchInputMapping(table, options) - let result: SearchResponse if (isExternalTable) { span?.addTags({ searchType: "external" }) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 1dba420a28..90303a6ca7 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -107,7 +107,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) { } return dataFilters.recurseLogicalOperators(filters, checkFilters) } - options.query = checkFilters(options.query) + if (options.query) { + options.query = checkFilters(options.query) + } return options } diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 23d7c7590c..fce355f7b9 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -3,7 +3,7 @@ import { BBReferenceFieldSubType, FieldType, FormulaType, - SearchFilter, + LegacyFilter, SearchFilters, SearchQueryFields, ArrayOperator, @@ -127,7 +127,7 @@ export function recurseLogicalOperators( fn: (f: SearchFilters) => SearchFilters ) { for (const logical of LOGICAL_OPERATORS) { - if (filters?.[logical]) { + if (filters[logical]) { filters[logical]!.conditions = filters[logical]!.conditions.map( condition => fn(condition) ) @@ -163,9 +163,6 @@ export function recurseSearchFilters( * https://github.com/Budibase/budibase/issues/10118 */ export const cleanupQuery = (query: SearchFilters) => { - if (!query) { - return query - } for (let filterField of NoEmptyFilterStrings) { if (!query[filterField]) { continue @@ -311,7 +308,7 @@ export class ColumnSplitter { * @param filter the builder filter structure */ -const buildCondition = (expression: SearchFilter) => { +const buildCondition = (expression: LegacyFilter) => { // Filter body let query: SearchFilters = { string: {}, @@ -437,8 +434,13 @@ const buildCondition = (expression: SearchFilter) => { } export const buildQueryLegacy = ( - filter?: SearchFilterGroup | SearchFilter[] + filter?: LegacyFilter[] | SearchFilters ): SearchFilters | undefined => { + // this is of type SearchFilters or is undefined + if (!Array.isArray(filter)) { + return filter + } + let query: SearchFilters = { string: {}, fuzzy: {}, @@ -572,7 +574,7 @@ export const buildQueryLegacy = ( */ export const buildQuery = ( - filter?: SearchFilterGroup | SearchFilter[] + filter?: SearchFilterGroup | LegacyFilter[] ): SearchFilters | undefined => { const parsedFilter: SearchFilterGroup | undefined = processSearchFilters(filter) @@ -594,7 +596,7 @@ export const buildQuery = ( const globalOperator: LogicalOperator = operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator] - const coreRequest: SearchFilters = { + return { ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}), [globalOperator]: { conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => { @@ -608,7 +610,6 @@ export const buildQuery = ( }), }, } - return coreRequest } // The frontend can send single values for array fields sometimes, so to handle diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index 5b4d439984..14b3c84425 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -1,5 +1,5 @@ import { - SearchFilter, + LegacyFilter, SearchFilterGroup, FilterGroupLogicalOperator, SearchFilters, @@ -9,6 +9,10 @@ import { import * as Constants from "./constants" import { removeKeyNumbering } from "./filters" +// an array of keys from filter type to properties that are in the type +// this can then be converted using .fromEntries to an object +type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][] + export function unreachable( value: never, message = `No such case in exhaustive switch: ${value}` @@ -87,106 +91,6 @@ export function trimOtherProps(object: any, allowedProps: string[]) { return result } -/** - * Processes the filter config. Filters are migrated from - * SearchFilter[] to SearchFilterGroup - * - * If filters is not an array, the migration is skipped - * - * @param {SearchFilter[] | SearchFilterGroup} filters - */ -export const processSearchFilters = ( - filters: SearchFilter[] | SearchFilterGroup | undefined -): SearchFilterGroup | undefined => { - if (!filters) { - return - } - - // Base search config. - const defaultCfg: SearchFilterGroup = { - logicalOperator: FilterGroupLogicalOperator.ALL, - groups: [], - } - - const filterWhitelistKeys = [ - "field", - "operator", - "value", - "type", - "externalType", - "valueType", - "noValue", - "formulaType", - ] - - if (Array.isArray(filters)) { - let baseGroup: SearchFilterGroup = { - filters: [], - logicalOperator: FilterGroupLogicalOperator.ALL, - } - - const migratedSetting: SearchFilterGroup = filters.reduce( - (acc: SearchFilterGroup, filter: SearchFilter) => { - // Sort the properties for easier debugging - const filterEntries = Object.entries(filter) - .sort((a, b) => { - return a[0].localeCompare(b[0]) - }) - .filter(x => x[1] ?? false) - - if (filterEntries.length == 1) { - const [key, value] = filterEntries[0] - // Global - if (key === "onEmptyFilter") { - // unset otherwise - acc.onEmptyFilter = value - } else if (key === "operator" && value === "allOr") { - // Group 1 logical operator - baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY - } - - return acc - } - - const whiteListedFilterSettings: [string, any][] = filterEntries.reduce( - (acc: [string, any][], entry: [string, any]) => { - const [key, value] = entry - - if (filterWhitelistKeys.includes(key)) { - if (key === "field") { - acc.push([key, removeKeyNumbering(value)]) - } else { - acc.push([key, value]) - } - } - return acc - }, - [] - ) - - const migratedFilter: SearchFilter = Object.fromEntries( - whiteListedFilterSettings - ) as SearchFilter - - baseGroup.filters!.push(migratedFilter) - - if (!acc.groups || !acc.groups.length) { - // init the base group - acc.groups = [baseGroup] - } - - return acc - }, - defaultCfg - ) - - return migratedSetting - } else if (!filters?.groups) { - return - } - return filters -} - export function isSupportedUserSearch(query: SearchFilters) { const allowed = [ { op: BasicOperator.STRING, key: "email" }, @@ -212,3 +116,98 @@ export function isSupportedUserSearch(query: SearchFilters) { } return true } + +/** + * Processes the filter config. Filters are migrated from + * SearchFilter[] to SearchFilterGroup + * + * If filters is not an array, the migration is skipped + * + * @param {LegacyFilter[] | SearchFilterGroup} filters + */ +export const processSearchFilters = ( + filters: LegacyFilter[] | SearchFilterGroup | undefined +): SearchFilterGroup | undefined => { + if (!filters) { + return + } + + // Base search config. + const defaultCfg: SearchFilterGroup = { + logicalOperator: FilterGroupLogicalOperator.ALL, + groups: [], + } + + const filterAllowedKeys = [ + "field", + "operator", + "value", + "type", + "externalType", + "valueType", + "noValue", + "formulaType", + ] + + if (Array.isArray(filters)) { + let baseGroup: SearchFilterGroup = { + filters: [], + logicalOperator: FilterGroupLogicalOperator.ALL, + } + + return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => { + // Sort the properties for easier debugging + const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[]) + .sort((a, b) => { + return a.localeCompare(b) + }) + .filter(key => key in filter) + + if (filterPropertyKeys.length == 1) { + const key = filterPropertyKeys[0], + value = filter[key] + // Global + if (key === "onEmptyFilter") { + // unset otherwise + acc.onEmptyFilter = value + } else if (key === "operator" && value === "allOr") { + // Group 1 logical operator + baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY + } + + return acc + } + + const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce( + (acc: AllowedFilters, key) => { + const value = filter[key] + if (filterAllowedKeys.includes(key)) { + if (key === "field") { + acc.push([key, removeKeyNumbering(value)]) + } else { + acc.push([key, value]) + } + } + return acc + }, + [] + ) + + const migratedFilter: LegacyFilter = Object.fromEntries( + allowedFilterSettings + ) as LegacyFilter + + baseGroup.filters!.push(migratedFilter) + + if (!acc.groups || !acc.groups.length) { + // init the base group + acc.groups = [baseGroup] + } + + return acc + }, defaultCfg) + } else if (!filters?.groups) { + return + } + return filters +} diff --git a/packages/types/src/api/web/app/view.ts b/packages/types/src/api/web/app/view.ts index a6be5e2986..a99f2938ab 100644 --- a/packages/types/src/api/web/app/view.ts +++ b/packages/types/src/api/web/app/view.ts @@ -9,6 +9,7 @@ export interface ViewResponseEnriched { data: ViewV2Enriched } -export interface CreateViewRequest extends Omit {} +export interface CreateViewRequest + extends Omit {} -export interface UpdateViewRequest extends ViewV2 {} +export interface UpdateViewRequest extends Omit {} diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts index b3d577f0c8..23c599027e 100644 --- a/packages/types/src/api/web/searchFilter.ts +++ b/packages/types/src/api/web/searchFilter.ts @@ -5,7 +5,7 @@ import { SearchFilters, } from "../../sdk" -export type SearchFilter = { +export type LegacyFilter = { operator: keyof SearchFilters | "rangeLow" | "rangeHigh" onEmptyFilter?: EmptyFilterOption field: string @@ -14,9 +14,10 @@ export type SearchFilter = { externalType?: string } +// this is a type purely used by the UI export type SearchFilterGroup = { logicalOperator: FilterGroupLogicalOperator onEmptyFilter?: EmptyFilterOption groups?: SearchFilterGroup[] - filters?: SearchFilter[] + filters?: LegacyFilter[] } diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index 16817f177a..87667a71e0 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -1,7 +1,7 @@ -import { SearchFilter, SearchFilterGroup, SortOrder, SortType } from "../../api" +import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api" import { UIFieldMetadata } from "./table" import { Document } from "../document" -import { DBView } from "../../sdk" +import { DBView, SearchFilters } from "../../sdk" export type ViewTemplateOpts = { field: string @@ -65,7 +65,9 @@ export interface ViewV2 { name: string primaryDisplay?: string tableId: string - query?: SearchFilter[] | SearchFilterGroup + query?: LegacyFilter[] | SearchFilters + // duplicate to store UI information about filters + queryUI?: SearchFilterGroup sort?: { field: string order?: SortOrder From 2fed6008fa81ec462532f72b7c23a9a58cb10900 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 2 Oct 2024 18:41:27 +0100 Subject: [PATCH 09/14] Updating view stores. --- .../builder/src/stores/builder/viewsV2.js | 22 ++++++++++++++++++- .../grid/stores/datasources/viewV2.js | 12 +++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/stores/builder/viewsV2.js b/packages/builder/src/stores/builder/viewsV2.js index 9bd32f4a24..a3ceb623a9 100644 --- a/packages/builder/src/stores/builder/viewsV2.js +++ b/packages/builder/src/stores/builder/viewsV2.js @@ -1,6 +1,23 @@ import { writable, derived, get } from "svelte/store" import { tables } from "./tables" import { API } from "api" +import { dataFilters } from "@budibase/shared-core" + +function convertToSearchFilters(view) { + // convert from SearchFilterGroup type + if (view.query) { + view.queryUI = view.query + view.query = dataFilters.buildQuery(view.query) + } + return view +} + +function convertToSearchFilterGroup(view) { + if (view.queryUI) { + view.query = view.queryUI + } + return view +} export function createViewsV2Store() { const store = writable({ @@ -12,7 +29,7 @@ export function createViewsV2Store() { const views = Object.values(table?.views || {}).filter(view => { return view.version === 2 }) - list = list.concat(views) + list = list.concat(views.map(view => convertToSearchFilterGroup(view))) }) return { ...$store, @@ -34,6 +51,7 @@ export function createViewsV2Store() { } const create = async view => { + view = convertToSearchFilters(view) const savedViewResponse = await API.viewV2.create(view) const savedView = savedViewResponse.data replaceView(savedView.id, savedView) @@ -41,6 +59,7 @@ export function createViewsV2Store() { } const save = async view => { + view = convertToSearchFilters(view) const res = await API.viewV2.update(view) const savedView = res?.data replaceView(view.id, savedView) @@ -51,6 +70,7 @@ export function createViewsV2Store() { if (!viewId) { return } + view = convertToSearchFilterGroup(view) const existingView = get(derivedStore).list.find(view => view.id === viewId) const tableIndex = get(tables).list.findIndex(table => { return table._id === view?.tableId || table._id === existingView?.tableId diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js index bf5e7c2628..13c46f9d34 100644 --- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js +++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js @@ -1,4 +1,14 @@ import { get } from "svelte/store" +import { dataFilters } from "@budibase/shared-core" + +function convertToSearchFilters(view) { + // convert from SearchFilterGroup type + if (view.query) { + view.queryUI = view.query + view.query = dataFilters.buildQuery(view.query) + } + return view +} const SuppressErrors = true @@ -6,7 +16,7 @@ export const createActions = context => { const { API, datasource, columns } = context const saveDefinition = async newDefinition => { - await API.viewV2.update(newDefinition) + await API.viewV2.update(convertToSearchFilters(newDefinition)) } const saveRow = async row => { From 9e7ed0471905eaa5eb5ba3fdf7444c733e6c7858 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 2 Oct 2024 18:43:07 +0100 Subject: [PATCH 10/14] Support saving queryUI. --- packages/server/src/api/controllers/view/viewsV2.ts | 6 ++++-- packages/types/src/api/web/app/view.ts | 5 ++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 3df7172de2..0257c86ded 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -99,10 +99,11 @@ export async function create(ctx: Ctx) { const schema = await parseSchema(view) - const parsedView: Omit, "id" | "version" | "queryUI"> = { + const parsedView: Omit, "id" | "version"> = { name: view.name, tableId: view.tableId, query: view.query, + queryUI: view.queryUI, sort: view.sort, schema, primaryDisplay: view.primaryDisplay, @@ -132,12 +133,13 @@ export async function update(ctx: Ctx) { const { tableId } = view const schema = await parseSchema(view) - const parsedView: RequiredKeys> = { + const parsedView: RequiredKeys = { id: view.id, name: view.name, version: view.version, tableId: view.tableId, query: view.query, + queryUI: view.queryUI, sort: view.sort, schema, primaryDisplay: view.primaryDisplay, diff --git a/packages/types/src/api/web/app/view.ts b/packages/types/src/api/web/app/view.ts index a99f2938ab..a6be5e2986 100644 --- a/packages/types/src/api/web/app/view.ts +++ b/packages/types/src/api/web/app/view.ts @@ -9,7 +9,6 @@ export interface ViewResponseEnriched { data: ViewV2Enriched } -export interface CreateViewRequest - extends Omit {} +export interface CreateViewRequest extends Omit {} -export interface UpdateViewRequest extends Omit {} +export interface UpdateViewRequest extends ViewV2 {} From bcb940b7eb3cc2ebe40f2788c3494f9f861e9122 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 2 Oct 2024 18:45:47 +0100 Subject: [PATCH 11/14] Merge issue. --- packages/server/src/api/controllers/view/viewsV2.ts | 4 ++-- packages/types/src/api/web/app/view.ts | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 0d74a2f5d0..0257c86ded 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -99,7 +99,7 @@ export async function create(ctx: Ctx) { const schema = await parseSchema(view) - const parsedView: Omit, "id" | "version" | "queryUI"> = { + const parsedView: Omit, "id" | "version"> = { name: view.name, tableId: view.tableId, query: view.query, @@ -133,7 +133,7 @@ export async function update(ctx: Ctx) { const { tableId } = view const schema = await parseSchema(view) - const parsedView: RequiredKeys> = { + const parsedView: RequiredKeys = { id: view.id, name: view.name, version: view.version, diff --git a/packages/types/src/api/web/app/view.ts b/packages/types/src/api/web/app/view.ts index a99f2938ab..a6be5e2986 100644 --- a/packages/types/src/api/web/app/view.ts +++ b/packages/types/src/api/web/app/view.ts @@ -9,7 +9,6 @@ export interface ViewResponseEnriched { data: ViewV2Enriched } -export interface CreateViewRequest - extends Omit {} +export interface CreateViewRequest extends Omit {} -export interface UpdateViewRequest extends Omit {} +export interface UpdateViewRequest extends ViewV2 {} From 932413c4c1df3e99ee64c5ecbccd68d222f0e6fc Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 2 Oct 2024 18:57:40 +0100 Subject: [PATCH 12/14] Getting re-loading of view queries working again. --- packages/builder/src/stores/builder/viewsV2.js | 1 + .../src/components/grid/stores/datasources/viewV2.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/stores/builder/viewsV2.js b/packages/builder/src/stores/builder/viewsV2.js index a3ceb623a9..0c7a03414e 100644 --- a/packages/builder/src/stores/builder/viewsV2.js +++ b/packages/builder/src/stores/builder/viewsV2.js @@ -15,6 +15,7 @@ function convertToSearchFilters(view) { function convertToSearchFilterGroup(view) { if (view.queryUI) { view.query = view.queryUI + delete view.queryUI } return view } diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js index 13c46f9d34..8e4321be96 100644 --- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js +++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js @@ -135,7 +135,7 @@ export const initialise = context => { } // Only override filter state if we don't have an initial filter if (!get(initialFilter)) { - filter.set($definition.query) + filter.set($definition.queryUI || $definition.query) } }) ) From 98ebd8f18cf45a026aad06a5b31bc0b538440005 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 2 Oct 2024 18:59:20 +0100 Subject: [PATCH 13/14] Omitting in tests. --- packages/server/src/api/routes/tests/viewV2.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 1d6c1d50cd..da2354017a 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -154,7 +154,7 @@ describe.each([ }) it("can persist views with all fields", async () => { - const newView: Required = { + const newView: Required> = { name: generator.name(), tableId: table._id!, primaryDisplay: "id", @@ -584,7 +584,7 @@ describe.each([ it("can update all fields", async () => { const tableId = table._id! - const updatedData: Required = { + const updatedData: Required> = { version: view.version, id: view.id, tableId, From 9c70ed92ba58094abba9b53d28a991f3e89db559 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 3 Oct 2024 14:47:31 +0100 Subject: [PATCH 14/14] Small build fix. --- packages/server/src/sdk/app/rows/search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 7e73a51889..1a15eb5839 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -92,7 +92,7 @@ export async function search( // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as // that could let users find rows they should not be allowed to access. - let viewQuery = dataFilters.buildQueryLegacy(view.query || []) + let viewQuery = dataFilters.buildQueryLegacy(view.query) || {} delete viewQuery?.onEmptyFilter const sqsEnabled = await features.flags.isEnabled("SQS")