MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merging v3-backport branch

This commit is contained in:
mike12345567 2024-10-02 18:09:53 +01:00
parent 5709c5f8d5
commit 786bfdb0e2
9 changed files with 176 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
packages/server/src/api/controllers/row/views.ts
View File

@ -3,6 +3,8 @@ import {
ViewV2,
SearchRowResponse,
SearchViewRowRequest,
RequiredKeys,
RowSearchParams,
} from "@budibase/types"
import sdk from "../../../sdk"
import { context } from "@budibase/backend-core"
@ -20,30 +22,34 @@ export async function searchView(
ctx.throw(400, `This method only supports viewsV2`)
}
const viewFields = Object.entries(view.schema || {})
.filter(([_, value]) => value.visible)
.map(([key]) => key)
const { body } = ctx.request
await context.ensureSnippetContext(true)
const result = await sdk.rows.search(
{
viewId: view.id,
const searchOptions: RequiredKeys<SearchViewRowRequest> &
RequiredKeys<
Pick<RowSearchParams, "tableId" | "viewId" | "query" | "fields">
> = {
tableId: view.tableId,
viewId: view.id,
query: body.query,
fields: viewFields,
...getSortOptions(body, view),
limit: body.limit,
bookmark: body.bookmark,
paginate: body.paginate,
countRows: body.countRows,
},
{
user: sdk.users.getUserContextBindings(ctx.user),
}
)
const result = await sdk.rows.search(searchOptions, {
user: sdk.users.getUserContextBindings(ctx.user),
})
result.rows.forEach(r => (r._viewId = view.id))
ctx.body = result
}
function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
if (request.sort) {
return {

4
packages/server/src/api/controllers/view/viewsV2.ts
View File

@ -99,7 +99,7 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
const schema = await parseSchema(view)
const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version"> = {
const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version" | "queryUI"> = {
name: view.name,
tableId: view.tableId,
query: view.query,
@ -132,7 +132,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
const { tableId } = view
const schema = await parseSchema(view)
const parsedView: RequiredKeys<ViewV2> = {
const parsedView: RequiredKeys<Omit<ViewV2, "queryUI">> = {
id: view.id,
name: view.name,
version: view.version,

45
packages/server/src/sdk/app/rows/search.ts
View File

@ -1,12 +1,10 @@
import {
EmptyFilterOption,
LegacyFilter,
LogicalOperator,
Row,
RowSearchParams,
SearchFilter,
SearchFilterGroup,
SearchFilterKey,
SearchFilters,
SearchResponse,
SortOrder,
Table,
@ -63,7 +61,6 @@ export async function search(
if (options.viewId) {
source = await sdk.views.get(options.viewId)
table = await sdk.views.getTable(source)
options = searchInputMapping(table, options)
} else if (options.tableId) {
source = await sdk.tables.getTable(options.tableId)
table = source
@ -76,7 +73,7 @@ export async function search(
if (options.query) {
const visibleFields = (
options.fields || Object.keys(table.schema)
).filter(field => table.schema[field].visible !== false)
).filter(field => table.schema[field]?.visible !== false)
const queryableFields = await getQueryableFields(table, visibleFields)
options.query = removeInvalidFilters(options.query, queryableFields)
@ -84,38 +81,52 @@ export async function search(
options.query = {}
}
// need to make sure filters in correct shape before checking for view
options = searchInputMapping(table, options)
if (options.viewId) {
const view = await sdk.views.get(options.viewId)
// Delete extraneous search params that cannot be overridden
delete options.query.onEmptyFilter
const view = source as ViewV2
// Enrich saved query with ephemeral query params.
// We prevent searching on any fields that are saved as part of the query, as
// that could let users find rows they should not be allowed to access.
let viewQuery = dataFilters.buildQuery(view.query || [])
let viewQuery = dataFilters.buildQueryLegacy(view.query || [])
delete viewQuery?.onEmptyFilter
if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) {
// Lucene does not accept conditional filters, so we need to keep the old logic
const query: SearchFilters = viewQuery || {}
const viewFilters = view.query as SearchFilter[]
const sqsEnabled = await features.flags.isEnabled("SQS")
const supportsLogicalOperators =
isExternalTableID(view.tableId) || sqsEnabled
if (!supportsLogicalOperators) {
// In the unlikely event that a Grouped Filter is in a non-SQS environment
// It needs to be ignored entirely
let queryFilters: LegacyFilter[] = Array.isArray(view.query)
? view.query
: []
// Extract existing fields
const existingFields =
viewFilters
queryFilters
?.filter(filter => filter.field)
.map(filter => db.removeKeyNumbering(filter.field)) || []
viewQuery ??= {}
// Carry over filters for unused fields
Object.keys(options.query || {}).forEach(key => {
Object.keys(options.query).forEach(key => {
const operator = key as Exclude<SearchFilterKey, LogicalOperator>
Object.keys(options.query[operator] || {}).forEach(field => {
if (!existingFields.includes(db.removeKeyNumbering(field))) {
query[operator]![field] = options.query[operator]![field]
viewQuery![operator]![field] = options.query[operator]![field]
}
})
})
options.query = query
options.query = viewQuery
} else {
const conditions = viewQuery ? [viewQuery] : []
options.query = {
$and: {
conditions: [viewQuery as SearchFilterGroup, options.query],
conditions: [...conditions, options.query],
},
}
}
@ -146,8 +157,6 @@ export async function search(
options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
}
options = searchInputMapping(table, options)
let result: SearchResponse<Row>
if (isExternalTable) {
span?.addTags({ searchType: "external" })

2
packages/server/src/sdk/app/rows/search/utils.ts
View File

@ -107,7 +107,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) {
}
return dataFilters.recurseLogicalOperators(filters, checkFilters)
}
if (options.query) {
options.query = checkFilters(options.query)
}
return options
}

21
packages/shared-core/src/filters.ts
View File

@ -3,7 +3,7 @@ import {
BBReferenceFieldSubType,
FieldType,
FormulaType,
SearchFilter,
LegacyFilter,
SearchFilters,
SearchQueryFields,
ArrayOperator,
@ -127,7 +127,7 @@ export function recurseLogicalOperators(
fn: (f: SearchFilters) => SearchFilters
) {
for (const logical of LOGICAL_OPERATORS) {
if (filters?.[logical]) {
if (filters[logical]) {
filters[logical]!.conditions = filters[logical]!.conditions.map(
condition => fn(condition)
)
@ -163,9 +163,6 @@ export function recurseSearchFilters(
* https://github.com/Budibase/budibase/issues/10118
*/
export const cleanupQuery = (query: SearchFilters) => {
if (!query) {
return query
}
for (let filterField of NoEmptyFilterStrings) {
if (!query[filterField]) {
continue
@ -311,7 +308,7 @@ export class ColumnSplitter {
* @param filter the builder filter structure
*/
const buildCondition = (expression: SearchFilter) => {
const buildCondition = (expression: LegacyFilter) => {
// Filter body
let query: SearchFilters = {
string: {},
@ -437,8 +434,13 @@ const buildCondition = (expression: SearchFilter) => {
}
export const buildQueryLegacy = (
filter?: SearchFilterGroup | SearchFilter[]
filter?: LegacyFilter[] | SearchFilters
): SearchFilters | undefined => {
// this is of type SearchFilters or is undefined
if (!Array.isArray(filter)) {
return filter
}
let query: SearchFilters = {
string: {},
fuzzy: {},
@ -572,7 +574,7 @@ export const buildQueryLegacy = (
*/
export const buildQuery = (
filter?: SearchFilterGroup | SearchFilter[]
filter?: SearchFilterGroup | LegacyFilter[]
): SearchFilters | undefined => {
const parsedFilter: SearchFilterGroup | undefined =
processSearchFilters(filter)
@ -594,7 +596,7 @@ export const buildQuery = (
const globalOperator: LogicalOperator =
operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator]
const coreRequest: SearchFilters = {
return {
...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
[globalOperator]: {
conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => {
@ -608,7 +610,6 @@ export const buildQuery = (
}),
},
}
return coreRequest
}
// The frontend can send single values for array fields sometimes, so to handle

201
packages/shared-core/src/utils.ts
View File

@ -1,5 +1,5 @@
import {
SearchFilter,
LegacyFilter,
SearchFilterGroup,
FilterGroupLogicalOperator,
SearchFilters,
@ -9,6 +9,10 @@ import {
import * as Constants from "./constants"
import { removeKeyNumbering } from "./filters"
// an array of keys from filter type to properties that are in the type
// this can then be converted using .fromEntries to an object
type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][]
export function unreachable(
value: never,
message = `No such case in exhaustive switch: ${value}`
@ -87,106 +91,6 @@ export function trimOtherProps(object: any, allowedProps: string[]) {
return result
}
/**
* Processes the filter config. Filters are migrated from
* SearchFilter[] to SearchFilterGroup
*
* If filters is not an array, the migration is skipped
*
* @param {SearchFilter[] | SearchFilterGroup} filters
*/
export const processSearchFilters = (
filters: SearchFilter[] | SearchFilterGroup | undefined
): SearchFilterGroup | undefined => {
if (!filters) {
return
}
// Base search config.
const defaultCfg: SearchFilterGroup = {
logicalOperator: FilterGroupLogicalOperator.ALL,
groups: [],
}
const filterWhitelistKeys = [
"field",
"operator",
"value",
"type",
"externalType",
"valueType",
"noValue",
"formulaType",
]
if (Array.isArray(filters)) {
let baseGroup: SearchFilterGroup = {
filters: [],
logicalOperator: FilterGroupLogicalOperator.ALL,
}
const migratedSetting: SearchFilterGroup = filters.reduce(
(acc: SearchFilterGroup, filter: SearchFilter) => {
// Sort the properties for easier debugging
const filterEntries = Object.entries(filter)
.sort((a, b) => {
return a[0].localeCompare(b[0])
})
.filter(x => x[1] ?? false)
if (filterEntries.length == 1) {
const [key, value] = filterEntries[0]
// Global
if (key === "onEmptyFilter") {
// unset otherwise
acc.onEmptyFilter = value
} else if (key === "operator" && value === "allOr") {
// Group 1 logical operator
baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
}
return acc
}
const whiteListedFilterSettings: [string, any][] = filterEntries.reduce(
(acc: [string, any][], entry: [string, any]) => {
const [key, value] = entry
if (filterWhitelistKeys.includes(key)) {
if (key === "field") {
acc.push([key, removeKeyNumbering(value)])
} else {
acc.push([key, value])
}
}
return acc
},
[]
)
const migratedFilter: SearchFilter = Object.fromEntries(
whiteListedFilterSettings
) as SearchFilter
baseGroup.filters!.push(migratedFilter)
if (!acc.groups || !acc.groups.length) {
// init the base group
acc.groups = [baseGroup]
}
return acc
},
defaultCfg
)
return migratedSetting
} else if (!filters?.groups) {
return
}
return filters
}
export function isSupportedUserSearch(query: SearchFilters) {
const allowed = [
{ op: BasicOperator.STRING, key: "email" },
@ -212,3 +116,98 @@ export function isSupportedUserSearch(query: SearchFilters) {
}
return true
}
/**
* Processes the filter config. Filters are migrated from
* SearchFilter[] to SearchFilterGroup
*
* If filters is not an array, the migration is skipped
*
* @param {LegacyFilter[] | SearchFilterGroup} filters
*/
export const processSearchFilters = (
filters: LegacyFilter[] | SearchFilterGroup | undefined
): SearchFilterGroup | undefined => {
if (!filters) {
return
}
// Base search config.
const defaultCfg: SearchFilterGroup = {
logicalOperator: FilterGroupLogicalOperator.ALL,
groups: [],
}
const filterAllowedKeys = [
"field",
"operator",
"value",
"type",
"externalType",
"valueType",
"noValue",
"formulaType",
]
if (Array.isArray(filters)) {
let baseGroup: SearchFilterGroup = {
filters: [],
logicalOperator: FilterGroupLogicalOperator.ALL,
}
return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => {
// Sort the properties for easier debugging
const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[])
.sort((a, b) => {
return a.localeCompare(b)
})
.filter(key => key in filter)
if (filterPropertyKeys.length == 1) {
const key = filterPropertyKeys[0],
value = filter[key]
// Global
if (key === "onEmptyFilter") {
// unset otherwise
acc.onEmptyFilter = value
} else if (key === "operator" && value === "allOr") {
// Group 1 logical operator
baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
}
return acc
}
const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce(
(acc: AllowedFilters, key) => {
const value = filter[key]
if (filterAllowedKeys.includes(key)) {
if (key === "field") {
acc.push([key, removeKeyNumbering(value)])
} else {
acc.push([key, value])
}
}
return acc
},
[]
)
const migratedFilter: LegacyFilter = Object.fromEntries(
allowedFilterSettings
) as LegacyFilter
baseGroup.filters!.push(migratedFilter)
if (!acc.groups || !acc.groups.length) {
// init the base group
acc.groups = [baseGroup]
}
return acc
}, defaultCfg)
} else if (!filters?.groups) {
return
}
return filters
}

5
packages/types/src/api/web/app/view.ts
View File

@ -9,6 +9,7 @@ export interface ViewResponseEnriched {
data: ViewV2Enriched
}
export interface CreateViewRequest extends Omit<ViewV2, "version" | "id"> {}
export interface CreateViewRequest
extends Omit<ViewV2, "version" | "id" | "queryUI"> {}
export interface UpdateViewRequest extends ViewV2 {}
export interface UpdateViewRequest extends Omit<ViewV2, "queryUI"> {}

5
packages/types/src/api/web/searchFilter.ts
View File

@ -5,7 +5,7 @@ import {
SearchFilters,
} from "../../sdk"
export type SearchFilter = {
export type LegacyFilter = {
operator: keyof SearchFilters | "rangeLow" | "rangeHigh"
onEmptyFilter?: EmptyFilterOption
field: string
@ -14,9 +14,10 @@ export type SearchFilter = {
externalType?: string
}
// this is a type purely used by the UI
export type SearchFilterGroup = {
logicalOperator: FilterGroupLogicalOperator
onEmptyFilter?: EmptyFilterOption
groups?: SearchFilterGroup[]
filters?: SearchFilter[]
filters?: LegacyFilter[]
}

8
packages/types/src/documents/app/view.ts
View File

@ -1,7 +1,7 @@
import { SearchFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
import { UIFieldMetadata } from "./table"
import { Document } from "../document"
import { DBView } from "../../sdk"
import { DBView, SearchFilters } from "../../sdk"
export type ViewTemplateOpts = {
field: string
@ -65,7 +65,9 @@ export interface ViewV2 {
name: string
primaryDisplay?: string
tableId: string
query?: SearchFilter[] | SearchFilterGroup
query?: LegacyFilter[] | SearchFilters
// duplicate to store UI information about filters
queryUI?: SearchFilterGroup
sort?: {
field: string
order?: SortOrder