backport of V3 backend changes for search filters on view, giving this the correct type to support conditionals.
This commit is contained in:
parent
6b3867faee
commit
6e660151bd
|
@ -5,6 +5,9 @@ import {
|
|||
SearchViewRowRequest,
|
||||
SearchFilterKey,
|
||||
LogicalOperator,
|
||||
RequiredKeys,
|
||||
RowSearchParams,
|
||||
LegacyFilter,
|
||||
} from "@budibase/types"
|
||||
import { dataFilters } from "@budibase/shared-core"
|
||||
import sdk from "../../../sdk"
|
||||
|
@ -17,7 +20,7 @@ export async function searchView(
|
|||
) {
|
||||
const { viewId } = ctx.params
|
||||
|
||||
const view = await sdk.views.get(viewId)
|
||||
const view: ViewV2 = await sdk.views.get(viewId)
|
||||
if (!view) {
|
||||
ctx.throw(404, `View ${viewId} not found`)
|
||||
}
|
||||
|
@ -25,23 +28,35 @@ export async function searchView(
|
|||
ctx.throw(400, `This method only supports viewsV2`)
|
||||
}
|
||||
|
||||
const viewFields = Object.entries(view.schema || {})
|
||||
.filter(([_, value]) => value.visible)
|
||||
.map(([key]) => key)
|
||||
const { body } = ctx.request
|
||||
|
||||
const sqsEnabled = await features.flags.isEnabled("SQS")
|
||||
const supportsLogicalOperators = isExternalTableID(view.tableId) || sqsEnabled
|
||||
|
||||
// Enrich saved query with ephemeral query params.
|
||||
// We prevent searching on any fields that are saved as part of the query, as
|
||||
// that could let users find rows they should not be allowed to access.
|
||||
let query = dataFilters.buildQuery(view.query || [])
|
||||
let query = dataFilters.buildQueryLegacy(view.query)
|
||||
|
||||
delete query?.onEmptyFilter
|
||||
|
||||
if (body.query) {
|
||||
// Delete extraneous search params that cannot be overridden
|
||||
delete body.query.onEmptyFilter
|
||||
|
||||
if (
|
||||
!isExternalTableID(view.tableId) &&
|
||||
!(await features.flags.isEnabled("SQS"))
|
||||
) {
|
||||
if (!supportsLogicalOperators) {
|
||||
// In the unlikely event that a Grouped Filter is in a non-SQS environment
|
||||
// It needs to be ignored entirely
|
||||
let queryFilters: LegacyFilter[] = Array.isArray(view.query)
|
||||
? view.query
|
||||
: []
|
||||
|
||||
// Extract existing fields
|
||||
const existingFields =
|
||||
view.query
|
||||
queryFilters
|
||||
?.filter(filter => filter.field)
|
||||
.map(filter => db.removeKeyNumbering(filter.field)) || []
|
||||
|
||||
|
@ -49,15 +64,16 @@ export async function searchView(
|
|||
Object.keys(body.query).forEach(key => {
|
||||
const operator = key as Exclude<SearchFilterKey, LogicalOperator>
|
||||
Object.keys(body.query[operator] || {}).forEach(field => {
|
||||
if (!existingFields.includes(db.removeKeyNumbering(field))) {
|
||||
if (query && !existingFields.includes(db.removeKeyNumbering(field))) {
|
||||
query[operator]![field] = body.query[operator]![field]
|
||||
}
|
||||
})
|
||||
})
|
||||
} else {
|
||||
const conditions = query ? [query] : []
|
||||
query = {
|
||||
$and: {
|
||||
conditions: [query, body.query],
|
||||
conditions: [...conditions, body.query],
|
||||
},
|
||||
}
|
||||
}
|
||||
|
@ -65,25 +81,29 @@ export async function searchView(
|
|||
|
||||
await context.ensureSnippetContext(true)
|
||||
|
||||
const enrichedQuery = await enrichSearchContext(query, {
|
||||
const enrichedQuery = await enrichSearchContext(query || {}, {
|
||||
user: sdk.users.getUserContextBindings(ctx.user),
|
||||
})
|
||||
|
||||
const result = await sdk.rows.search({
|
||||
viewId: view.id,
|
||||
const searchOptions: RequiredKeys<SearchViewRowRequest> &
|
||||
RequiredKeys<
|
||||
Pick<RowSearchParams, "tableId" | "viewId" | "query" | "fields">
|
||||
> = {
|
||||
tableId: view.tableId,
|
||||
viewId: view.id,
|
||||
query: enrichedQuery,
|
||||
fields: viewFields,
|
||||
...getSortOptions(body, view),
|
||||
limit: body.limit,
|
||||
bookmark: body.bookmark,
|
||||
paginate: body.paginate,
|
||||
countRows: body.countRows,
|
||||
})
|
||||
}
|
||||
|
||||
const result = await sdk.rows.search(searchOptions)
|
||||
result.rows.forEach(r => (r._viewId = view.id))
|
||||
ctx.body = result
|
||||
}
|
||||
|
||||
function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
|
||||
if (request.sort) {
|
||||
return {
|
||||
|
|
|
@ -99,7 +99,7 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
|
|||
|
||||
const schema = await parseSchema(view)
|
||||
|
||||
const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version"> = {
|
||||
const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version" | "queryUI"> = {
|
||||
name: view.name,
|
||||
tableId: view.tableId,
|
||||
query: view.query,
|
||||
|
@ -132,7 +132,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
|
|||
const { tableId } = view
|
||||
|
||||
const schema = await parseSchema(view)
|
||||
const parsedView: RequiredKeys<ViewV2> = {
|
||||
const parsedView: RequiredKeys<Omit<ViewV2, "queryUI">> = {
|
||||
id: view.id,
|
||||
name: view.name,
|
||||
version: view.version,
|
||||
|
|
|
@ -3,7 +3,7 @@ import {
|
|||
BBReferenceFieldSubType,
|
||||
FieldType,
|
||||
FormulaType,
|
||||
SearchFilter,
|
||||
LegacyFilter,
|
||||
SearchFilters,
|
||||
SearchQueryFields,
|
||||
ArrayOperator,
|
||||
|
@ -19,9 +19,12 @@ import {
|
|||
RangeOperator,
|
||||
LogicalOperator,
|
||||
isLogicalSearchOperator,
|
||||
SearchFilterGroup,
|
||||
FilterGroupLogicalOperator,
|
||||
} from "@budibase/types"
|
||||
import dayjs from "dayjs"
|
||||
import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
|
||||
import { processSearchFilters } from "./utils"
|
||||
import { deepGet, schema } from "./helpers"
|
||||
import { isPlainObject, isEmpty } from "lodash"
|
||||
import { decodeNonAscii } from "./helpers/schema"
|
||||
|
@ -124,7 +127,7 @@ export function recurseLogicalOperators(
|
|||
fn: (f: SearchFilters) => SearchFilters
|
||||
) {
|
||||
for (const logical of LOGICAL_OPERATORS) {
|
||||
if (filters?.[logical]) {
|
||||
if (filters[logical]) {
|
||||
filters[logical]!.conditions = filters[logical]!.conditions.map(
|
||||
condition => fn(condition)
|
||||
)
|
||||
|
@ -304,10 +307,143 @@ export class ColumnSplitter {
|
|||
}
|
||||
|
||||
/**
|
||||
* Builds a JSON query from the filter structure generated in the builder
|
||||
* Builds a JSON query from the filter a SearchFilter definition
|
||||
* @param filter the builder filter structure
|
||||
*/
|
||||
export const buildQuery = (filter: SearchFilter[]) => {
|
||||
|
||||
const buildCondition = (expression: LegacyFilter) => {
|
||||
// Filter body
|
||||
let query: SearchFilters = {
|
||||
string: {},
|
||||
fuzzy: {},
|
||||
range: {},
|
||||
equal: {},
|
||||
notEqual: {},
|
||||
empty: {},
|
||||
notEmpty: {},
|
||||
contains: {},
|
||||
notContains: {},
|
||||
oneOf: {},
|
||||
containsAny: {},
|
||||
}
|
||||
let { operator, field, type, value, externalType, onEmptyFilter } = expression
|
||||
|
||||
if (!operator || !field) {
|
||||
return
|
||||
}
|
||||
|
||||
const queryOperator = operator as SearchFilterOperator
|
||||
const isHbs =
|
||||
typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
|
||||
// Parse all values into correct types
|
||||
if (operator === "allOr") {
|
||||
query.allOr = true
|
||||
return
|
||||
}
|
||||
if (onEmptyFilter) {
|
||||
query.onEmptyFilter = onEmptyFilter
|
||||
return
|
||||
}
|
||||
|
||||
// Default the value for noValue fields to ensure they are correctly added
|
||||
// to the final query
|
||||
if (queryOperator === "empty" || queryOperator === "notEmpty") {
|
||||
value = null
|
||||
}
|
||||
|
||||
if (
|
||||
type === "datetime" &&
|
||||
!isHbs &&
|
||||
queryOperator !== "empty" &&
|
||||
queryOperator !== "notEmpty"
|
||||
) {
|
||||
// Ensure date value is a valid date and parse into correct format
|
||||
if (!value) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
value = new Date(value).toISOString()
|
||||
} catch (error) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if (type === "number" && typeof value === "string" && !isHbs) {
|
||||
if (queryOperator === "oneOf") {
|
||||
value = value.split(",").map(item => parseFloat(item))
|
||||
} else {
|
||||
value = parseFloat(value)
|
||||
}
|
||||
}
|
||||
if (type === "boolean") {
|
||||
value = `${value}`?.toLowerCase() === "true"
|
||||
}
|
||||
if (
|
||||
["contains", "notContains", "containsAny"].includes(
|
||||
operator.toLocaleString()
|
||||
) &&
|
||||
type === "array" &&
|
||||
typeof value === "string"
|
||||
) {
|
||||
value = value.split(",")
|
||||
}
|
||||
if (operator.toLocaleString().startsWith("range") && query.range) {
|
||||
const minint =
|
||||
SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap]
|
||||
?.min || Number.MIN_SAFE_INTEGER
|
||||
const maxint =
|
||||
SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap]
|
||||
?.max || Number.MAX_SAFE_INTEGER
|
||||
if (!query.range[field]) {
|
||||
query.range[field] = {
|
||||
low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
|
||||
high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
|
||||
}
|
||||
}
|
||||
if (operator === "rangeLow" && value != null && value !== "") {
|
||||
query.range[field] = {
|
||||
...query.range[field],
|
||||
low: value,
|
||||
}
|
||||
} else if (operator === "rangeHigh" && value != null && value !== "") {
|
||||
query.range[field] = {
|
||||
...query.range[field],
|
||||
high: value,
|
||||
}
|
||||
}
|
||||
} else if (isLogicalSearchOperator(queryOperator)) {
|
||||
// TODO
|
||||
} else if (query[queryOperator] && operator !== "onEmptyFilter") {
|
||||
if (type === "boolean") {
|
||||
// Transform boolean filters to cope with null.
|
||||
// "equals false" needs to be "not equals true"
|
||||
// "not equals false" needs to be "equals true"
|
||||
if (queryOperator === "equal" && value === false) {
|
||||
query.notEqual = query.notEqual || {}
|
||||
query.notEqual[field] = true
|
||||
} else if (queryOperator === "notEqual" && value === false) {
|
||||
query.equal = query.equal || {}
|
||||
query.equal[field] = true
|
||||
} else {
|
||||
query[queryOperator] ??= {}
|
||||
query[queryOperator]![field] = value
|
||||
}
|
||||
} else {
|
||||
query[queryOperator] ??= {}
|
||||
query[queryOperator]![field] = value
|
||||
}
|
||||
}
|
||||
|
||||
return query
|
||||
}
|
||||
|
||||
export const buildQueryLegacy = (
|
||||
filter?: LegacyFilter[] | SearchFilters
|
||||
): SearchFilters | undefined => {
|
||||
// this is of type SearchFilters or is undefined
|
||||
if (!Array.isArray(filter)) {
|
||||
return filter
|
||||
}
|
||||
|
||||
let query: SearchFilters = {
|
||||
string: {},
|
||||
fuzzy: {},
|
||||
|
@ -368,13 +504,15 @@ export const buildQuery = (filter: SearchFilter[]) => {
|
|||
value = `${value}`?.toLowerCase() === "true"
|
||||
}
|
||||
if (
|
||||
["contains", "notContains", "containsAny"].includes(operator) &&
|
||||
["contains", "notContains", "containsAny"].includes(
|
||||
operator.toLocaleString()
|
||||
) &&
|
||||
type === "array" &&
|
||||
typeof value === "string"
|
||||
) {
|
||||
value = value.split(",")
|
||||
}
|
||||
if (operator.startsWith("range") && query.range) {
|
||||
if (operator.toLocaleString().startsWith("range") && query.range) {
|
||||
const minint =
|
||||
SqlNumberTypeRangeMap[
|
||||
externalType as keyof typeof SqlNumberTypeRangeMap
|
||||
|
@ -401,7 +539,7 @@ export const buildQuery = (filter: SearchFilter[]) => {
|
|||
}
|
||||
}
|
||||
} else if (isLogicalSearchOperator(queryOperator)) {
|
||||
// TODO
|
||||
// ignore
|
||||
} else if (query[queryOperator] && operator !== "onEmptyFilter") {
|
||||
if (type === "boolean") {
|
||||
// Transform boolean filters to cope with null.
|
||||
|
@ -423,14 +561,68 @@ export const buildQuery = (filter: SearchFilter[]) => {
|
|||
}
|
||||
}
|
||||
})
|
||||
|
||||
return query
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a **SearchFilterGroup** filter definition into a grouped
|
||||
* search query of type **SearchFilters**
|
||||
*
|
||||
* Legacy support remains for the old **SearchFilter[]** format.
|
||||
* These will be migrated to an appropriate **SearchFilters** object, if encountered
|
||||
*
|
||||
* @param filter
|
||||
*
|
||||
* @returns {SearchFilters}
|
||||
*/
|
||||
|
||||
export const buildQuery = (
|
||||
filter?: SearchFilterGroup | LegacyFilter[]
|
||||
): SearchFilters | undefined => {
|
||||
const parsedFilter: SearchFilterGroup | undefined =
|
||||
processSearchFilters(filter)
|
||||
|
||||
if (!parsedFilter) {
|
||||
return
|
||||
}
|
||||
|
||||
const operatorMap: { [key in FilterGroupLogicalOperator]: LogicalOperator } =
|
||||
{
|
||||
[FilterGroupLogicalOperator.ALL]: LogicalOperator.AND,
|
||||
[FilterGroupLogicalOperator.ANY]: LogicalOperator.OR,
|
||||
}
|
||||
|
||||
const globalOnEmpty = parsedFilter.onEmptyFilter
|
||||
? parsedFilter.onEmptyFilter
|
||||
: null
|
||||
|
||||
const globalOperator: LogicalOperator =
|
||||
operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator]
|
||||
|
||||
const coreRequest: SearchFilters = {
|
||||
...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
|
||||
[globalOperator]: {
|
||||
conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => {
|
||||
return {
|
||||
[operatorMap[group.logicalOperator]]: {
|
||||
conditions: group.filters
|
||||
?.map(x => buildCondition(x))
|
||||
.filter(filter => filter),
|
||||
},
|
||||
}
|
||||
}),
|
||||
},
|
||||
}
|
||||
return coreRequest
|
||||
}
|
||||
|
||||
// The frontend can send single values for array fields sometimes, so to handle
|
||||
// this we convert them to arrays at the controller level so that nothing below
|
||||
// this has to worry about the non-array values.
|
||||
export function fixupFilterArrays(filters: SearchFilters) {
|
||||
if (!filters) {
|
||||
return filters
|
||||
}
|
||||
for (const searchField of Object.values(ArrayOperator)) {
|
||||
const field = filters[searchField]
|
||||
if (field == null || !isPlainObject(field)) {
|
||||
|
|
|
@ -1,5 +1,20 @@
|
|||
import { ArrayOperator, BasicOperator, SearchFilters } from "@budibase/types"
|
||||
import {
|
||||
LegacyFilter,
|
||||
SearchFilterGroup,
|
||||
FilterGroupLogicalOperator,
|
||||
SearchFilters,
|
||||
BasicOperator,
|
||||
ArrayOperator,
|
||||
} from "@budibase/types"
|
||||
import * as Constants from "./constants"
|
||||
import { removeKeyNumbering } from "./filters"
|
||||
|
||||
// an array of keys from filter type to properties that are in the type
|
||||
// this can then be converted using .fromEntries to an object
|
||||
type WhitelistedFilters = [
|
||||
keyof LegacyFilter,
|
||||
LegacyFilter[keyof LegacyFilter]
|
||||
][]
|
||||
|
||||
export function unreachable(
|
||||
value: never,
|
||||
|
@ -104,3 +119,96 @@ export function isSupportedUserSearch(query: SearchFilters) {
|
|||
}
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes the filter config. Filters are migrated from
|
||||
* SearchFilter[] to SearchFilterGroup
|
||||
*
|
||||
* If filters is not an array, the migration is skipped
|
||||
*
|
||||
* @param {LegacyFilter[] | SearchFilterGroup} filters
|
||||
*/
|
||||
export const processSearchFilters = (
|
||||
filters: LegacyFilter[] | SearchFilterGroup | undefined
|
||||
): SearchFilterGroup | undefined => {
|
||||
if (!filters) {
|
||||
return
|
||||
}
|
||||
|
||||
// Base search config.
|
||||
const defaultCfg: SearchFilterGroup = {
|
||||
logicalOperator: FilterGroupLogicalOperator.ALL,
|
||||
groups: [],
|
||||
}
|
||||
|
||||
const filterWhitelistKeys = [
|
||||
"field",
|
||||
"operator",
|
||||
"value",
|
||||
"type",
|
||||
"externalType",
|
||||
"valueType",
|
||||
"noValue",
|
||||
"formulaType",
|
||||
]
|
||||
|
||||
if (Array.isArray(filters)) {
|
||||
let baseGroup: SearchFilterGroup = {
|
||||
filters: [],
|
||||
logicalOperator: FilterGroupLogicalOperator.ALL,
|
||||
}
|
||||
|
||||
return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => {
|
||||
// Sort the properties for easier debugging
|
||||
const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[])
|
||||
.sort((a, b) => {
|
||||
return a.localeCompare(b)
|
||||
})
|
||||
.filter(key => key in filter)
|
||||
|
||||
if (filterPropertyKeys.length == 1) {
|
||||
const key = filterPropertyKeys[0],
|
||||
value = filter[key]
|
||||
// Global
|
||||
if (key === "onEmptyFilter") {
|
||||
// unset otherwise
|
||||
acc.onEmptyFilter = value
|
||||
} else if (key === "operator" && value === "allOr") {
|
||||
// Group 1 logical operator
|
||||
baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
|
||||
}
|
||||
|
||||
return acc
|
||||
}
|
||||
|
||||
const whiteListedFilterSettings: WhitelistedFilters =
|
||||
filterPropertyKeys.reduce((acc: WhitelistedFilters, key) => {
|
||||
const value = filter[key]
|
||||
if (filterWhitelistKeys.includes(key)) {
|
||||
if (key === "field") {
|
||||
acc.push([key, removeKeyNumbering(value)])
|
||||
} else {
|
||||
acc.push([key, value])
|
||||
}
|
||||
}
|
||||
return acc
|
||||
}, [])
|
||||
|
||||
const migratedFilter: LegacyFilter = Object.fromEntries(
|
||||
whiteListedFilterSettings
|
||||
) as LegacyFilter
|
||||
|
||||
baseGroup.filters!.push(migratedFilter)
|
||||
|
||||
if (!acc.groups || !acc.groups.length) {
|
||||
// init the base group
|
||||
acc.groups = [baseGroup]
|
||||
}
|
||||
|
||||
return acc
|
||||
}, defaultCfg)
|
||||
} else if (!filters?.groups) {
|
||||
return
|
||||
}
|
||||
return filters
|
||||
}
|
||||
|
|
|
@ -9,6 +9,7 @@ export interface ViewResponseEnriched {
|
|||
data: ViewV2Enriched
|
||||
}
|
||||
|
||||
export interface CreateViewRequest extends Omit<ViewV2, "version" | "id"> {}
|
||||
export interface CreateViewRequest
|
||||
extends Omit<ViewV2, "version" | "id" | "queryUI"> {}
|
||||
|
||||
export interface UpdateViewRequest extends ViewV2 {}
|
||||
export interface UpdateViewRequest extends Omit<ViewV2, "queryUI"> {}
|
||||
|
|
|
@ -1,7 +1,11 @@
|
|||
import { FieldType } from "../../documents"
|
||||
import { EmptyFilterOption, SearchFilters } from "../../sdk"
|
||||
import {
|
||||
EmptyFilterOption,
|
||||
FilterGroupLogicalOperator,
|
||||
SearchFilters,
|
||||
} from "../../sdk"
|
||||
|
||||
export type SearchFilter = {
|
||||
export type LegacyFilter = {
|
||||
operator: keyof SearchFilters | "rangeLow" | "rangeHigh"
|
||||
onEmptyFilter?: EmptyFilterOption
|
||||
field: string
|
||||
|
@ -9,3 +13,11 @@ export type SearchFilter = {
|
|||
value: any
|
||||
externalType?: string
|
||||
}
|
||||
|
||||
// this is a type purely used by the UI
|
||||
export type SearchFilterGroup = {
|
||||
logicalOperator: FilterGroupLogicalOperator
|
||||
onEmptyFilter?: EmptyFilterOption
|
||||
groups?: SearchFilterGroup[]
|
||||
filters?: LegacyFilter[]
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
import { SearchFilter, SortOrder, SortType } from "../../api"
|
||||
import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
|
||||
import { UIFieldMetadata } from "./table"
|
||||
import { Document } from "../document"
|
||||
import { DBView } from "../../sdk"
|
||||
import { DBView, SearchFilters } from "../../sdk"
|
||||
|
||||
export type ViewTemplateOpts = {
|
||||
field: string
|
||||
|
@ -65,7 +65,9 @@ export interface ViewV2 {
|
|||
name: string
|
||||
primaryDisplay?: string
|
||||
tableId: string
|
||||
query?: SearchFilter[]
|
||||
query?: LegacyFilter[] | SearchFilters
|
||||
// duplicate to store UI information about filters
|
||||
queryUI?: SearchFilterGroup
|
||||
sort?: {
|
||||
field: string
|
||||
order?: SortOrder
|
||||
|
|
|
@ -191,6 +191,11 @@ export enum EmptyFilterOption {
|
|||
RETURN_NONE = "none",
|
||||
}
|
||||
|
||||
export enum FilterGroupLogicalOperator {
|
||||
ALL = "all",
|
||||
ANY = "any",
|
||||
}
|
||||
|
||||
export enum SqlClient {
|
||||
MS_SQL = "mssql",
|
||||
POSTGRES = "pg",
|
||||
|
|
Loading…
Reference in New Issue