diff --git a/packages/builder/src/stores/builder/viewsV2.js b/packages/builder/src/stores/builder/viewsV2.js index 9bd32f4a24..0c7a03414e 100644 --- a/packages/builder/src/stores/builder/viewsV2.js +++ b/packages/builder/src/stores/builder/viewsV2.js @@ -1,6 +1,24 @@ import { writable, derived, get } from "svelte/store" import { tables } from "./tables" import { API } from "api" +import { dataFilters } from "@budibase/shared-core" + +function convertToSearchFilters(view) { + // convert from SearchFilterGroup type + if (view.query) { + view.queryUI = view.query + view.query = dataFilters.buildQuery(view.query) + } + return view +} + +function convertToSearchFilterGroup(view) { + if (view.queryUI) { + view.query = view.queryUI + delete view.queryUI + } + return view +} export function createViewsV2Store() { const store = writable({ @@ -12,7 +30,7 @@ export function createViewsV2Store() { const views = Object.values(table?.views || {}).filter(view => { return view.version === 2 }) - list = list.concat(views) + list = list.concat(views.map(view => convertToSearchFilterGroup(view))) }) return { ...$store, @@ -34,6 +52,7 @@ export function createViewsV2Store() { } const create = async view => { + view = convertToSearchFilters(view) const savedViewResponse = await API.viewV2.create(view) const savedView = savedViewResponse.data replaceView(savedView.id, savedView) @@ -41,6 +60,7 @@ export function createViewsV2Store() { } const save = async view => { + view = convertToSearchFilters(view) const res = await API.viewV2.update(view) const savedView = res?.data replaceView(view.id, savedView) @@ -51,6 +71,7 @@ export function createViewsV2Store() { if (!viewId) { return } + view = convertToSearchFilterGroup(view) const existingView = get(derivedStore).list.find(view => view.id === viewId) const tableIndex = get(tables).list.findIndex(table => { return table._id === view?.tableId || table._id === existingView?.tableId diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js index bf5e7c2628..8e4321be96 100644 --- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js +++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js @@ -1,4 +1,14 @@ import { get } from "svelte/store" +import { dataFilters } from "@budibase/shared-core" + +function convertToSearchFilters(view) { + // convert from SearchFilterGroup type + if (view.query) { + view.queryUI = view.query + view.query = dataFilters.buildQuery(view.query) + } + return view +} const SuppressErrors = true @@ -6,7 +16,7 @@ export const createActions = context => { const { API, datasource, columns } = context const saveDefinition = async newDefinition => { - await API.viewV2.update(newDefinition) + await API.viewV2.update(convertToSearchFilters(newDefinition)) } const saveRow = async row => { @@ -125,7 +135,7 @@ export const initialise = context => { } // Only override filter state if we don't have an initial filter if (!get(initialFilter)) { - filter.set($definition.query) + filter.set($definition.queryUI || $definition.query) } }) ) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index f804c650f9..b8d01424f2 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -3,6 +3,8 @@ import { ViewV2, SearchRowResponse, SearchViewRowRequest, + RequiredKeys, + RowSearchParams, } from "@budibase/types" import sdk from "../../../sdk" import { context } from "@budibase/backend-core" @@ -20,30 +22,34 @@ export async function searchView( ctx.throw(400, `This method only supports viewsV2`) } + const viewFields = Object.entries(view.schema || {}) + .filter(([_, value]) => value.visible) + .map(([key]) => key) const { body } = ctx.request await context.ensureSnippetContext(true) - const result = await sdk.rows.search( - { - viewId: view.id, - tableId: view.tableId, - query: body.query, - ...getSortOptions(body, view), - limit: body.limit, - bookmark: body.bookmark, - paginate: body.paginate, - countRows: body.countRows, - }, - { - user: sdk.users.getUserContextBindings(ctx.user), - } - ) + const searchOptions: RequiredKeys & + RequiredKeys< + Pick + > = { + tableId: view.tableId, + viewId: view.id, + query: body.query, + fields: viewFields, + ...getSortOptions(body, view), + limit: body.limit, + bookmark: body.bookmark, + paginate: body.paginate, + countRows: body.countRows, + } + const result = await sdk.rows.search(searchOptions, { + user: sdk.users.getUserContextBindings(ctx.user), + }) result.rows.forEach(r => (r._viewId = view.id)) ctx.body = result } - function getSortOptions(request: SearchViewRowRequest, view: ViewV2) { if (request.sort) { return { diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 7f6f638541..0257c86ded 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -103,6 +103,7 @@ export async function create(ctx: Ctx) { name: view.name, tableId: view.tableId, query: view.query, + queryUI: view.queryUI, sort: view.sort, schema, primaryDisplay: view.primaryDisplay, @@ -138,6 +139,7 @@ export async function update(ctx: Ctx) { version: view.version, tableId: view.tableId, query: view.query, + queryUI: view.queryUI, sort: view.sort, schema, primaryDisplay: view.primaryDisplay, diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 1780b8ff27..83b26bc687 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -155,7 +155,7 @@ describe.each([ }) it("can persist views with all fields", async () => { - const newView: Required = { + const newView: Required> = { name: generator.name(), tableId: table._id!, primaryDisplay: "id", @@ -612,7 +612,7 @@ describe.each([ it("can update all fields", async () => { const tableId = table._id! - const updatedData: Required = { + const updatedData: Required> = { version: view.version, id: view.id, tableId, diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 297be85d47..f68f78f0bc 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -1,12 +1,10 @@ import { EmptyFilterOption, + LegacyFilter, LogicalOperator, Row, RowSearchParams, - SearchFilter, - SearchFilterGroup, SearchFilterKey, - SearchFilters, SearchResponse, SortOrder, Table, @@ -63,7 +61,6 @@ export async function search( if (options.viewId) { source = await sdk.views.get(options.viewId) table = await sdk.views.getTable(source) - options = searchInputMapping(table, options) } else if (options.tableId) { source = await sdk.tables.getTable(options.tableId) table = source @@ -76,7 +73,7 @@ export async function search( if (options.query) { const visibleFields = ( options.fields || Object.keys(table.schema) - ).filter(field => table.schema[field].visible !== false) + ).filter(field => table.schema[field]?.visible !== false) const queryableFields = await getQueryableFields(table, visibleFields) options.query = removeInvalidFilters(options.query, queryableFields) @@ -84,40 +81,54 @@ export async function search( options.query = {} } + // need to make sure filters in correct shape before checking for view + options = searchInputMapping(table, options) + if (options.viewId) { - const view = await sdk.views.get(options.viewId) + // Delete extraneous search params that cannot be overridden + delete options.query.onEmptyFilter + + const view = source as ViewV2 // Enrich saved query with ephemeral query params. // We prevent searching on any fields that are saved as part of the query, as // that could let users find rows they should not be allowed to access. - let viewQuery = dataFilters.buildQuery(view.query || []) + let viewQuery = dataFilters.buildQueryLegacy(view.query) || {} + delete viewQuery?.onEmptyFilter - if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) { - // Lucene does not accept conditional filters, so we need to keep the old logic - const query: SearchFilters = viewQuery || {} - const viewFilters = view.query as SearchFilter[] + const sqsEnabled = await features.flags.isEnabled("SQS") + const supportsLogicalOperators = + isExternalTableID(view.tableId) || sqsEnabled + if (!supportsLogicalOperators) { + // In the unlikely event that a Grouped Filter is in a non-SQS environment + // It needs to be ignored entirely + let queryFilters: LegacyFilter[] = Array.isArray(view.query) + ? view.query + : [] delete options.query.onEmptyFilter // Extract existing fields const existingFields = - viewFilters + queryFilters ?.filter(filter => filter.field) .map(filter => db.removeKeyNumbering(filter.field)) || [] + viewQuery ??= {} // Carry over filters for unused fields - Object.keys(options.query || {}).forEach(key => { + Object.keys(options.query).forEach(key => { const operator = key as Exclude Object.keys(options.query[operator] || {}).forEach(field => { if (!existingFields.includes(db.removeKeyNumbering(field))) { - query[operator]![field] = options.query[operator]![field] + viewQuery![operator]![field] = options.query[operator]![field] } }) }) - options.query = query + options.query = viewQuery } else { + const conditions = viewQuery ? [viewQuery] : [] options.query = { $and: { - conditions: [viewQuery as SearchFilterGroup, options.query], + conditions: [...conditions, options.query], }, } if (viewQuery.onEmptyFilter) { @@ -151,8 +162,6 @@ export async function search( options.sortOrder = options.sortOrder.toLowerCase() as SortOrder } - options = searchInputMapping(table, options) - let result: SearchResponse if (isExternalTable) { span?.addTags({ searchType: "external" }) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index 1dba420a28..90303a6ca7 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -107,7 +107,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) { } return dataFilters.recurseLogicalOperators(filters, checkFilters) } - options.query = checkFilters(options.query) + if (options.query) { + options.query = checkFilters(options.query) + } return options } diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 23d7c7590c..fce355f7b9 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -3,7 +3,7 @@ import { BBReferenceFieldSubType, FieldType, FormulaType, - SearchFilter, + LegacyFilter, SearchFilters, SearchQueryFields, ArrayOperator, @@ -127,7 +127,7 @@ export function recurseLogicalOperators( fn: (f: SearchFilters) => SearchFilters ) { for (const logical of LOGICAL_OPERATORS) { - if (filters?.[logical]) { + if (filters[logical]) { filters[logical]!.conditions = filters[logical]!.conditions.map( condition => fn(condition) ) @@ -163,9 +163,6 @@ export function recurseSearchFilters( * https://github.com/Budibase/budibase/issues/10118 */ export const cleanupQuery = (query: SearchFilters) => { - if (!query) { - return query - } for (let filterField of NoEmptyFilterStrings) { if (!query[filterField]) { continue @@ -311,7 +308,7 @@ export class ColumnSplitter { * @param filter the builder filter structure */ -const buildCondition = (expression: SearchFilter) => { +const buildCondition = (expression: LegacyFilter) => { // Filter body let query: SearchFilters = { string: {}, @@ -437,8 +434,13 @@ const buildCondition = (expression: SearchFilter) => { } export const buildQueryLegacy = ( - filter?: SearchFilterGroup | SearchFilter[] + filter?: LegacyFilter[] | SearchFilters ): SearchFilters | undefined => { + // this is of type SearchFilters or is undefined + if (!Array.isArray(filter)) { + return filter + } + let query: SearchFilters = { string: {}, fuzzy: {}, @@ -572,7 +574,7 @@ export const buildQueryLegacy = ( */ export const buildQuery = ( - filter?: SearchFilterGroup | SearchFilter[] + filter?: SearchFilterGroup | LegacyFilter[] ): SearchFilters | undefined => { const parsedFilter: SearchFilterGroup | undefined = processSearchFilters(filter) @@ -594,7 +596,7 @@ export const buildQuery = ( const globalOperator: LogicalOperator = operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator] - const coreRequest: SearchFilters = { + return { ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}), [globalOperator]: { conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => { @@ -608,7 +610,6 @@ export const buildQuery = ( }), }, } - return coreRequest } // The frontend can send single values for array fields sometimes, so to handle diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts index 5b4d439984..14b3c84425 100644 --- a/packages/shared-core/src/utils.ts +++ b/packages/shared-core/src/utils.ts @@ -1,5 +1,5 @@ import { - SearchFilter, + LegacyFilter, SearchFilterGroup, FilterGroupLogicalOperator, SearchFilters, @@ -9,6 +9,10 @@ import { import * as Constants from "./constants" import { removeKeyNumbering } from "./filters" +// an array of keys from filter type to properties that are in the type +// this can then be converted using .fromEntries to an object +type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][] + export function unreachable( value: never, message = `No such case in exhaustive switch: ${value}` @@ -87,106 +91,6 @@ export function trimOtherProps(object: any, allowedProps: string[]) { return result } -/** - * Processes the filter config. Filters are migrated from - * SearchFilter[] to SearchFilterGroup - * - * If filters is not an array, the migration is skipped - * - * @param {SearchFilter[] | SearchFilterGroup} filters - */ -export const processSearchFilters = ( - filters: SearchFilter[] | SearchFilterGroup | undefined -): SearchFilterGroup | undefined => { - if (!filters) { - return - } - - // Base search config. - const defaultCfg: SearchFilterGroup = { - logicalOperator: FilterGroupLogicalOperator.ALL, - groups: [], - } - - const filterWhitelistKeys = [ - "field", - "operator", - "value", - "type", - "externalType", - "valueType", - "noValue", - "formulaType", - ] - - if (Array.isArray(filters)) { - let baseGroup: SearchFilterGroup = { - filters: [], - logicalOperator: FilterGroupLogicalOperator.ALL, - } - - const migratedSetting: SearchFilterGroup = filters.reduce( - (acc: SearchFilterGroup, filter: SearchFilter) => { - // Sort the properties for easier debugging - const filterEntries = Object.entries(filter) - .sort((a, b) => { - return a[0].localeCompare(b[0]) - }) - .filter(x => x[1] ?? false) - - if (filterEntries.length == 1) { - const [key, value] = filterEntries[0] - // Global - if (key === "onEmptyFilter") { - // unset otherwise - acc.onEmptyFilter = value - } else if (key === "operator" && value === "allOr") { - // Group 1 logical operator - baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY - } - - return acc - } - - const whiteListedFilterSettings: [string, any][] = filterEntries.reduce( - (acc: [string, any][], entry: [string, any]) => { - const [key, value] = entry - - if (filterWhitelistKeys.includes(key)) { - if (key === "field") { - acc.push([key, removeKeyNumbering(value)]) - } else { - acc.push([key, value]) - } - } - return acc - }, - [] - ) - - const migratedFilter: SearchFilter = Object.fromEntries( - whiteListedFilterSettings - ) as SearchFilter - - baseGroup.filters!.push(migratedFilter) - - if (!acc.groups || !acc.groups.length) { - // init the base group - acc.groups = [baseGroup] - } - - return acc - }, - defaultCfg - ) - - return migratedSetting - } else if (!filters?.groups) { - return - } - return filters -} - export function isSupportedUserSearch(query: SearchFilters) { const allowed = [ { op: BasicOperator.STRING, key: "email" }, @@ -212,3 +116,98 @@ export function isSupportedUserSearch(query: SearchFilters) { } return true } + +/** + * Processes the filter config. Filters are migrated from + * SearchFilter[] to SearchFilterGroup + * + * If filters is not an array, the migration is skipped + * + * @param {LegacyFilter[] | SearchFilterGroup} filters + */ +export const processSearchFilters = ( + filters: LegacyFilter[] | SearchFilterGroup | undefined +): SearchFilterGroup | undefined => { + if (!filters) { + return + } + + // Base search config. + const defaultCfg: SearchFilterGroup = { + logicalOperator: FilterGroupLogicalOperator.ALL, + groups: [], + } + + const filterAllowedKeys = [ + "field", + "operator", + "value", + "type", + "externalType", + "valueType", + "noValue", + "formulaType", + ] + + if (Array.isArray(filters)) { + let baseGroup: SearchFilterGroup = { + filters: [], + logicalOperator: FilterGroupLogicalOperator.ALL, + } + + return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => { + // Sort the properties for easier debugging + const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[]) + .sort((a, b) => { + return a.localeCompare(b) + }) + .filter(key => key in filter) + + if (filterPropertyKeys.length == 1) { + const key = filterPropertyKeys[0], + value = filter[key] + // Global + if (key === "onEmptyFilter") { + // unset otherwise + acc.onEmptyFilter = value + } else if (key === "operator" && value === "allOr") { + // Group 1 logical operator + baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY + } + + return acc + } + + const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce( + (acc: AllowedFilters, key) => { + const value = filter[key] + if (filterAllowedKeys.includes(key)) { + if (key === "field") { + acc.push([key, removeKeyNumbering(value)]) + } else { + acc.push([key, value]) + } + } + return acc + }, + [] + ) + + const migratedFilter: LegacyFilter = Object.fromEntries( + allowedFilterSettings + ) as LegacyFilter + + baseGroup.filters!.push(migratedFilter) + + if (!acc.groups || !acc.groups.length) { + // init the base group + acc.groups = [baseGroup] + } + + return acc + }, defaultCfg) + } else if (!filters?.groups) { + return + } + return filters +} diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts index b3d577f0c8..23c599027e 100644 --- a/packages/types/src/api/web/searchFilter.ts +++ b/packages/types/src/api/web/searchFilter.ts @@ -5,7 +5,7 @@ import { SearchFilters, } from "../../sdk" -export type SearchFilter = { +export type LegacyFilter = { operator: keyof SearchFilters | "rangeLow" | "rangeHigh" onEmptyFilter?: EmptyFilterOption field: string @@ -14,9 +14,10 @@ export type SearchFilter = { externalType?: string } +// this is a type purely used by the UI export type SearchFilterGroup = { logicalOperator: FilterGroupLogicalOperator onEmptyFilter?: EmptyFilterOption groups?: SearchFilterGroup[] - filters?: SearchFilter[] + filters?: LegacyFilter[] } diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index 846de47a40..77bbb3e4d1 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -1,7 +1,7 @@ -import { SearchFilter, SearchFilterGroup, SortOrder, SortType } from "../../api" +import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api" import { UIFieldMetadata } from "./table" import { Document } from "../document" -import { DBView } from "../../sdk" +import { DBView, SearchFilters } from "../../sdk" export type ViewTemplateOpts = { field: string @@ -65,7 +65,9 @@ export interface ViewV2 { name: string primaryDisplay?: string tableId: string - query?: SearchFilter[] | SearchFilterGroup + query?: LegacyFilter[] | SearchFilters + // duplicate to store UI information about filters + queryUI?: SearchFilterGroup sort?: { field: string order?: SortOrder