MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge master.

This commit is contained in:
Sam Rose 2024-10-03 16:33:35 +01:00
parent 7ed28593fb f10aa42374
commit 725096c9d1
No known key found for this signature in database
18 changed files with 3176 additions and 2795 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/deploy-featurebranch.yml vendored
View File

@ -23,6 +23,7 @@ jobs:
PAYLOAD_BRANCH: ${{ github.head_ref }} PAYLOAD_BRANCH: ${{ github.head_ref }}
PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }} PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
PAYLOAD_LICENSE_TYPE: "free" PAYLOAD_LICENSE_TYPE: "free"
PAYLOAD_DEPLOY: "true"
with: with:
repository: budibase/budibase-deploys repository: budibase/budibase-deploys
event: featurebranch-qa-deploy event: featurebranch-qa-deploy

2
lerna.json
View File

@ -1,6 +1,6 @@
{ {
"$schema": "node_modules/lerna/schemas/lerna-schema.json", "$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.32.10", "version": "2.32.11",
"npmClient": "yarn", "npmClient": "yarn",
"packages": [ "packages": [
"packages/*", "packages/*",

6
packages/server/src/api/controllers/row/utils/sqlUtils.ts
View File

@ -124,9 +124,11 @@ export async function buildSqlFieldList(
([columnName, column]) => ([columnName, column]) =>
column.type !== FieldType.LINK && column.type !== FieldType.LINK &&
column.type !== FieldType.FORMULA && column.type !== FieldType.FORMULA &&
!existing.find((field: string) => field === columnName) !existing.find(
(field: string) => field === `${table.name}.${columnName}`
) )
.map(column => `${table.name}.${column[0]}`) )
.map(([columnName]) => `${table.name}.${columnName}`)
} }
let fields: string[] = [] let fields: string[] = []

22
packages/server/src/api/controllers/row/views.ts
View File

@ -3,6 +3,8 @@ import {
ViewV2, ViewV2,
SearchRowResponse, SearchRowResponse,
SearchViewRowRequest, SearchViewRowRequest,
RequiredKeys,
RowSearchParams,
} from "@budibase/types" } from "@budibase/types"
import sdk from "../../../sdk" import sdk from "../../../sdk"
import { context } from "@budibase/backend-core" import { context } from "@budibase/backend-core"
@ -20,30 +22,34 @@ export async function searchView(
ctx.throw(400, `This method only supports viewsV2`) ctx.throw(400, `This method only supports viewsV2`)
} }
const viewFields = Object.entries(view.schema || {})
.filter(([_, value]) => value.visible)
.map(([key]) => key)
const { body } = ctx.request const { body } = ctx.request
await context.ensureSnippetContext(true) await context.ensureSnippetContext(true)
const result = await sdk.rows.search( const searchOptions: RequiredKeys<SearchViewRowRequest> &
{ RequiredKeys<
viewId: view.id, Pick<RowSearchParams, "tableId" | "viewId" | "query" | "fields">
> = {
tableId: view.tableId, tableId: view.tableId,
viewId: view.id,
query: body.query, query: body.query,
fields: viewFields,
...getSortOptions(body, view), ...getSortOptions(body, view),
limit: body.limit, limit: body.limit,
bookmark: body.bookmark, bookmark: body.bookmark,
paginate: body.paginate, paginate: body.paginate,
countRows: body.countRows, countRows: body.countRows,
},
{
user: sdk.users.getUserContextBindings(ctx.user),
} }
)
const result = await sdk.rows.search(searchOptions, {
user: sdk.users.getUserContextBindings(ctx.user),
})
result.rows.forEach(r => (r._viewId = view.id)) result.rows.forEach(r => (r._viewId = view.id))
ctx.body = result ctx.body = result
} }
function getSortOptions(request: SearchViewRowRequest, view: ViewV2) { function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
if (request.sort) { if (request.sort) {
return { return {

2
packages/server/src/api/controllers/view/viewsV2.ts
View File

@ -129,6 +129,7 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
name: view.name, name: view.name,
tableId: view.tableId, tableId: view.tableId,
query: view.query, query: view.query,
queryUI: view.queryUI,
sort: view.sort, sort: view.sort,
schema, schema,
primaryDisplay: view.primaryDisplay, primaryDisplay: view.primaryDisplay,
@ -164,6 +165,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
version: view.version, version: view.version,
tableId: view.tableId, tableId: view.tableId,
query: view.query, query: view.query,
queryUI: view.queryUI,
sort: view.sort, sort: view.sort,
schema, schema,
primaryDisplay: view.primaryDisplay, primaryDisplay: view.primaryDisplay,

371
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -28,11 +28,13 @@ import {
RowSearchParams, RowSearchParams,
SearchFilters, SearchFilters,
SearchResponse, SearchResponse,
SearchRowRequest,
SortOrder, SortOrder,
SortType, SortType,
Table, Table,
TableSchema, TableSchema,
User, User,
ViewV2Schema,
} from "@budibase/types" } from "@budibase/types"
import _ from "lodash" import _ from "lodash"
import tk from "timekeeper" import tk from "timekeeper"
@ -64,18 +66,14 @@ describe.each([
let envCleanup: (() => void) | undefined let envCleanup: (() => void) | undefined
let datasource: Datasource | undefined let datasource: Datasource | undefined
let client: Knex | undefined let client: Knex | undefined
let table: Table let tableOrViewId: string
let rows: Row[] let rows: Row[]
async function basicRelationshipTables(type: RelationshipType) { async function basicRelationshipTables(type: RelationshipType) {
const relatedTable = await createTable( const relatedTable = await createTable({
{
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
}, })
generator.guid().substring(0, 10) const tableId = await createTable({
)
table = await createTable(
{
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
//@ts-ignore - API accepts this structure, will build out rest of definition //@ts-ignore - API accepts this structure, will build out rest of definition
productCat: { productCat: {
@ -83,17 +81,15 @@ describe.each([
relationshipType: type, relationshipType: type,
name: "productCat", name: "productCat",
fieldName: "product", fieldName: "product",
tableId: relatedTable._id!, tableId: relatedTable,
constraints: { constraints: {
type: "array", type: "array",
}, },
}, },
}, })
generator.guid().substring(0, 10)
)
return { return {
relatedTable: await config.api.table.get(relatedTable._id!), relatedTable: await config.api.table.get(relatedTable),
table, tableId,
} }
} }
@ -136,32 +132,69 @@ describe.each([
} }
}) })
async function createTable(schema: TableSchema, name?: string) { async function createTable(schema: TableSchema) {
return await config.api.table.save( const table = await config.api.table.save(
tableForDatasource(datasource, { schema, name }) tableForDatasource(datasource, { schema })
) )
return table._id!
}
async function createView(tableId: string, schema: ViewV2Schema) {
const view = await config.api.viewV2.create({
tableId: tableId,
name: generator.guid(),
schema,
})
return view.id
} }
async function createRows(arr: Record<string, any>[]) { async function createRows(arr: Record<string, any>[]) {
// Shuffling to avoid false positives given a fixed order // Shuffling to avoid false positives given a fixed order
await config.api.row.bulkImport(table._id!, { for (const row of _.shuffle(arr)) {
rows: _.shuffle(arr), await config.api.row.save(tableOrViewId, row)
}) }
rows = await config.api.row.fetch(table._id!) rows = await config.api.row.fetch(tableOrViewId)
}
describe.each([
["table", createTable],
[
"view",
async (schema: TableSchema) => {
const tableId = await createTable(schema)
const viewId = await createView(
tableId,
Object.keys(schema).reduce<ViewV2Schema>((viewSchema, fieldName) => {
const field = schema[fieldName]
viewSchema[fieldName] = {
visible: field.visible ?? true,
readonly: false,
}
return viewSchema
}, {})
)
return viewId
},
],
])("from %s", (sourceType, createTableOrView) => {
const isView = sourceType === "view"
if (isView && isLucene) {
// Some tests don't have the expected result in views via lucene, and given that it is getting deprecated, we exclude them from the tests
return
} }
class SearchAssertion { class SearchAssertion {
constructor(private readonly query: RowSearchParams) {} constructor(private readonly query: SearchRowRequest) {}
private async performSearch(): Promise<SearchResponse<Row>> { private async performSearch(): Promise<SearchResponse<Row>> {
if (isInMemory) { if (isInMemory) {
return dataFilters.search(_.cloneDeep(rows), this.query) return dataFilters.search(_.cloneDeep(rows), {
...this.query,
tableId: tableOrViewId,
})
} else { } else {
const sourceId = this.query.viewId || this.query.tableId return config.api.row.search(tableOrViewId, this.query)
if (!sourceId) {
throw new Error("No source ID provided")
}
return config.api.row.search(sourceId, this.query)
} }
} }
@ -331,8 +364,8 @@ describe.each([
} }
} }
function expectSearch(query: Omit<RowSearchParams, "tableId">) { function expectSearch(query: SearchRowRequest) {
return new SearchAssertion({ ...query, tableId: table._id! }) return new SearchAssertion(query)
} }
function expectQuery(query: SearchFilters) { function expectQuery(query: SearchFilters) {
@ -341,7 +374,7 @@ describe.each([
describe("boolean", () => { describe("boolean", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
isTrue: { name: "isTrue", type: FieldType.BOOLEAN }, isTrue: { name: "isTrue", type: FieldType.BOOLEAN },
}) })
await createRows([{ isTrue: true }, { isTrue: false }]) await createRows([{ isTrue: true }, { isTrue: false }])
@ -429,38 +462,35 @@ describe.each([
{ name: "serverDate", appointment: serverTime.toISOString() }, { name: "serverDate", appointment: serverTime.toISOString() },
{ {
name: "single user, session user", name: "single user, session user",
single_user: JSON.stringify(currentUser), single_user: currentUser,
}, },
{ {
name: "single user", name: "single user",
single_user: JSON.stringify(globalUsers[0]), single_user: globalUsers[0],
}, },
{ {
name: "deprecated single user, session user", name: "deprecated single user, session user",
deprecated_single_user: JSON.stringify([currentUser]), deprecated_single_user: [currentUser],
}, },
{ {
name: "deprecated single user", name: "deprecated single user",
deprecated_single_user: JSON.stringify([globalUsers[0]]), deprecated_single_user: [globalUsers[0]],
}, },
{ {
name: "multi user", name: "multi user",
multi_user: JSON.stringify(globalUsers), multi_user: globalUsers,
}, },
{ {
name: "multi user with session user", name: "multi user with session user",
multi_user: JSON.stringify([...globalUsers, currentUser]), multi_user: [...globalUsers, currentUser],
}, },
{ {
name: "deprecated multi user", name: "deprecated multi user",
deprecated_multi_user: JSON.stringify(globalUsers), deprecated_multi_user: globalUsers,
}, },
{ {
name: "deprecated multi user with session user", name: "deprecated multi user with session user",
deprecated_multi_user: JSON.stringify([ deprecated_multi_user: [...globalUsers, currentUser],
...globalUsers,
currentUser,
]),
}, },
] ]
} }
@ -482,7 +512,7 @@ describe.each([
}) })
) )
table = await createTable({ tableOrViewId = await createTableOrView({
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
appointment: { name: "appointment", type: FieldType.DATETIME }, appointment: { name: "appointment", type: FieldType.DATETIME },
single_user: { single_user: {
@ -632,9 +662,11 @@ describe.each([
}) })
it("should match the session user id in a multi user field", async () => { it("should match the session user id in a multi user field", async () => {
const allUsers = [...globalUsers, config.getUser()].map((user: any) => { const allUsers = [...globalUsers, config.getUser()].map(
(user: any) => {
return { _id: user._id } return { _id: user._id }
}) }
)
await expectQuery({ await expectQuery({
contains: { multi_user: ["{{ [user]._id }}"] }, contains: { multi_user: ["{{ [user]._id }}"] },
@ -647,9 +679,11 @@ describe.each([
}) })
it("should match the session user id in a deprecated multi user field", async () => { it("should match the session user id in a deprecated multi user field", async () => {
const allUsers = [...globalUsers, config.getUser()].map((user: any) => { const allUsers = [...globalUsers, config.getUser()].map(
(user: any) => {
return { _id: user._id } return { _id: user._id }
}) }
)
await expectQuery({ await expectQuery({
contains: { deprecated_multi_user: ["{{ [user]._id }}"] }, contains: { deprecated_multi_user: ["{{ [user]._id }}"] },
@ -764,7 +798,7 @@ describe.each([
describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => { describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
}) })
await createRows([{ name: "foo" }, { name: "bar" }]) await createRows([{ name: "foo" }, { name: "bar" }])
@ -791,6 +825,8 @@ describe.each([
}).toContainExactly([{ name: "foo" }, { name: "bar" }]) }).toContainExactly([{ name: "foo" }, { name: "bar" }])
}) })
// onEmptyFilter cannot be sent to view searches
!isView &&
it("should return nothing if onEmptyFilter is RETURN_NONE", async () => { it("should return nothing if onEmptyFilter is RETURN_NONE", async () => {
await expectQuery({ await expectQuery({
onEmptyFilter: EmptyFilterOption.RETURN_NONE, onEmptyFilter: EmptyFilterOption.RETURN_NONE,
@ -891,6 +927,8 @@ describe.each([
}).toContainExactly([{ name: "foo" }, { name: "bar" }]) }).toContainExactly([{ name: "foo" }, { name: "bar" }])
}) })
// onEmptyFilter cannot be sent to view searches
!isView &&
it("empty arrays returns all when onEmptyFilter is set to return 'none'", async () => { it("empty arrays returns all when onEmptyFilter is set to return 'none'", async () => {
await expectQuery({ await expectQuery({
onEmptyFilter: EmptyFilterOption.RETURN_NONE, onEmptyFilter: EmptyFilterOption.RETURN_NONE,
@ -1055,7 +1093,7 @@ describe.each([
datasourceId: datasource!._id!, datasourceId: datasource!._id!,
}) })
table = resp.datasource.entities![tableName] tableOrViewId = resp.datasource.entities![tableName]._id!
await createRows([{ name: "foo" }, { name: "bar" }]) await createRows([{ name: "foo" }, { name: "bar" }])
}) })
@ -1079,7 +1117,7 @@ describe.each([
describe("numbers", () => { describe("numbers", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
age: { name: "age", type: FieldType.NUMBER }, age: { name: "age", type: FieldType.NUMBER },
}) })
await createRows([{ age: 1 }, { age: 10 }]) await createRows([{ age: 1 }, { age: 10 }])
@ -1087,7 +1125,9 @@ describe.each([
describe("equal", () => { describe("equal", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ equal: { age: 1 } }).toContainExactly([{ age: 1 }]) await expectQuery({ equal: { age: 1 } }).toContainExactly([
{ age: 1 },
])
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
@ -1252,7 +1292,7 @@ describe.each([
const JAN_10TH = "2020-01-10T00:00:00.000Z" const JAN_10TH = "2020-01-10T00:00:00.000Z"
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
dob: { name: "dob", type: FieldType.DATETIME }, dob: { name: "dob", type: FieldType.DATETIME },
}) })
@ -1324,25 +1364,33 @@ describe.each([
it("greater than equal to", async () => { it("greater than equal to", async () => {
await expectQuery({ await expectQuery({
range: { dob: { low: JAN_10TH, high: MAX_VALID_DATE.toISOString() } }, range: {
dob: { low: JAN_10TH, high: MAX_VALID_DATE.toISOString() },
},
}).toContainExactly([{ dob: JAN_10TH }]) }).toContainExactly([{ dob: JAN_10TH }])
}) })
it("greater than", async () => { it("greater than", async () => {
await expectQuery({ await expectQuery({
range: { dob: { low: JAN_5TH, high: MAX_VALID_DATE.toISOString() } }, range: {
dob: { low: JAN_5TH, high: MAX_VALID_DATE.toISOString() },
},
}).toContainExactly([{ dob: JAN_10TH }]) }).toContainExactly([{ dob: JAN_10TH }])
}) })
it("less than equal to", async () => { it("less than equal to", async () => {
await expectQuery({ await expectQuery({
range: { dob: { high: JAN_1ST, low: MIN_VALID_DATE.toISOString() } }, range: {
dob: { high: JAN_1ST, low: MIN_VALID_DATE.toISOString() },
},
}).toContainExactly([{ dob: JAN_1ST }]) }).toContainExactly([{ dob: JAN_1ST }])
}) })
it("less than", async () => { it("less than", async () => {
await expectQuery({ await expectQuery({
range: { dob: { high: JAN_5TH, low: MIN_VALID_DATE.toISOString() } }, range: {
dob: { high: JAN_5TH, low: MIN_VALID_DATE.toISOString() },
},
}).toContainExactly([{ dob: JAN_1ST }]) }).toContainExactly([{ dob: JAN_1ST }])
}) })
}) })
@ -1399,7 +1447,7 @@ describe.each([
const NULL_TIME__ID = `null_time__id` const NULL_TIME__ID = `null_time__id`
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
timeid: { name: "timeid", type: FieldType.STRING }, timeid: { name: "timeid", type: FieldType.STRING },
time: { name: "time", type: FieldType.DATETIME, timeOnly: true }, time: { name: "time", type: FieldType.DATETIME, timeOnly: true },
}) })
@ -1560,7 +1608,7 @@ describe.each([
describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => { describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
numbers: { numbers: {
name: "numbers", name: "numbers",
type: FieldType.ARRAY, type: FieldType.ARRAY,
@ -1575,9 +1623,9 @@ describe.each([
describe("contains", () => { describe("contains", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ contains: { numbers: ["one"] } }).toContainExactly([ await expectQuery({
{ numbers: ["one", "two"] }, contains: { numbers: ["one"] },
]) }).toContainExactly([{ numbers: ["one", "two"] }])
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
@ -1657,7 +1705,7 @@ describe.each([
let BIG = "9223372036854775807" let BIG = "9223372036854775807"
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
num: { name: "num", type: FieldType.BIGINT }, num: { name: "num", type: FieldType.BIGINT },
}) })
await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }]) await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }])
@ -1762,7 +1810,7 @@ describe.each([
isInternal && isInternal &&
describe("auto", () => { describe("auto", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
auto: { auto: {
name: "auto", name: "auto",
type: FieldType.AUTO, type: FieldType.AUTO,
@ -1913,9 +1961,9 @@ describe.each([
// to specify an order for pagination to work. // to specify an order for pagination to work.
it("is stable without a sort specified", async () => { it("is stable without a sort specified", async () => {
let { rows: fullRowList } = await config.api.row.search( let { rows: fullRowList } = await config.api.row.search(
table._id!, tableOrViewId,
{ {
tableId: table._id!, tableId: tableOrViewId,
query: {}, query: {},
} }
) )
@ -1925,8 +1973,8 @@ describe.each([
hasNextPage: boolean | undefined = true, hasNextPage: boolean | undefined = true,
rowCount: number = 0 rowCount: number = 0
do { do {
const response = await config.api.row.search(table._id!, { const response = await config.api.row.search(tableOrViewId, {
tableId: table._id!, tableId: tableOrViewId,
limit: 1, limit: 1,
paginate: true, paginate: true,
query: {}, query: {},
@ -1949,8 +1997,8 @@ describe.each([
// eslint-disable-next-line no-constant-condition // eslint-disable-next-line no-constant-condition
while (true) { while (true) {
const response = await config.api.row.search(table._id!, { const response = await config.api.row.search(tableOrViewId, {
tableId: table._id!, tableId: tableOrViewId,
limit: 3, limit: 3,
query: {}, query: {},
bookmark, bookmark,
@ -1973,7 +2021,7 @@ describe.each([
describe("field name 1:name", () => { describe("field name 1:name", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
"1:name": { name: "1:name", type: FieldType.STRING }, "1:name": { name: "1:name", type: FieldType.STRING },
}) })
await createRows([{ "1:name": "bar" }, { "1:name": "foo" }]) await createRows([{ "1:name": "bar" }, { "1:name": "foo" }])
@ -1993,8 +2041,7 @@ describe.each([
isSql && isSql &&
describe("related formulas", () => { describe("related formulas", () => {
beforeAll(async () => { beforeAll(async () => {
const arrayTable = await createTable( const arrayTable = await createTable({
{
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
array: { array: {
name: "array", name: "array",
@ -2004,17 +2051,14 @@ describe.each([
inclusion: ["option 1", "option 2"], inclusion: ["option 1", "option 2"],
}, },
}, },
}, })
"array" tableOrViewId = await createTableOrView({
)
table = await createTable(
{
relationship: { relationship: {
type: FieldType.LINK, type: FieldType.LINK,
relationshipType: RelationshipType.MANY_TO_ONE, relationshipType: RelationshipType.MANY_TO_ONE,
name: "relationship", name: "relationship",
fieldName: "relate", fieldName: "relate",
tableId: arrayTable._id!, tableId: arrayTable,
constraints: { constraints: {
type: "array", type: "array",
}, },
@ -2026,21 +2070,19 @@ describe.each([
`let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.sort().join(",")` `let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.sort().join(",")`
), ),
}, },
}, })
"main"
)
const arrayRows = await Promise.all([ const arrayRows = await Promise.all([
config.api.row.save(arrayTable._id!, { config.api.row.save(arrayTable, {
name: "foo", name: "foo",
array: ["option 1"], array: ["option 1"],
}), }),
config.api.row.save(arrayTable._id!, { config.api.row.save(arrayTable, {
name: "bar", name: "bar",
array: ["option 2"], array: ["option 2"],
}), }),
]) ])
await Promise.all([ await Promise.all([
config.api.row.save(table._id!, { config.api.row.save(tableOrViewId, {
relationship: [arrayRows[0]._id, arrayRows[1]._id], relationship: [arrayRows[0]._id, arrayRows[1]._id],
}), }),
]) ])
@ -2059,7 +2101,7 @@ describe.each([
user1 = await config.createUser({ _id: `us_${utils.newid()}` }) user1 = await config.createUser({ _id: `us_${utils.newid()}` })
user2 = await config.createUser({ _id: `us_${utils.newid()}` }) user2 = await config.createUser({ _id: `us_${utils.newid()}` })
table = await createTable({ tableOrViewId = await createTableOrView({
user: { user: {
name: "user", name: "user",
type: FieldType.BB_REFERENCE_SINGLE, type: FieldType.BB_REFERENCE_SINGLE,
@ -2067,11 +2109,7 @@ describe.each([
}, },
}) })
await createRows([ await createRows([{ user: user1 }, { user: user2 }, { user: null }])
{ user: JSON.stringify(user1) },
{ user: JSON.stringify(user2) },
{ user: null },
])
}) })
describe("equal", () => { describe("equal", () => {
@ -2088,18 +2126,15 @@ describe.each([
describe("notEqual", () => { describe("notEqual", () => {
it("successfully finds a row", async () => { it("successfully finds a row", async () => {
await expectQuery({ notEqual: { user: user1._id } }).toContainExactly([ await expectQuery({ notEqual: { user: user1._id } }).toContainExactly(
{ user: { _id: user2._id } }, [{ user: { _id: user2._id } }, {}]
{}, )
])
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
await expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([ await expectQuery({ notEqual: { user: "us_none" } }).toContainExactly(
{ user: { _id: user1._id } }, [{ user: { _id: user1._id } }, { user: { _id: user2._id } }, {}]
{ user: { _id: user2._id } }, )
{},
])
}) })
}) })
@ -2139,7 +2174,7 @@ describe.each([
user1 = await config.createUser({ _id: `us_${utils.newid()}` }) user1 = await config.createUser({ _id: `us_${utils.newid()}` })
user2 = await config.createUser({ _id: `us_${utils.newid()}` }) user2 = await config.createUser({ _id: `us_${utils.newid()}` })
table = await createTable({ tableOrViewId = await createTableOrView({
users: { users: {
name: "users", name: "users",
type: FieldType.BB_REFERENCE, type: FieldType.BB_REFERENCE,
@ -2153,10 +2188,10 @@ describe.each([
}) })
await createRows([ await createRows([
{ number: 1, users: JSON.stringify([user1]) }, { number: 1, users: [user1] },
{ number: 2, users: JSON.stringify([user2]) }, { number: 2, users: [user2] },
{ number: 3, users: JSON.stringify([user1, user2]) }, { number: 3, users: [user1, user2] },
{ number: 4, users: JSON.stringify([]) }, { number: 4, users: [] },
]) ])
}) })
@ -2182,7 +2217,9 @@ describe.each([
}) })
it("fails to find nonexistent row", async () => { it("fails to find nonexistent row", async () => {
await expectQuery({ contains: { users: ["us_none"] } }).toFindNothing() await expectQuery({
contains: { users: ["us_none"] },
}).toFindNothing()
}) })
}) })
@ -2249,9 +2286,10 @@ describe.each([
let productCategoryTable: Table, productCatRows: Row[] let productCategoryTable: Table, productCatRows: Row[]
beforeAll(async () => { beforeAll(async () => {
const { relatedTable } = await basicRelationshipTables( const { relatedTable, tableId } = await basicRelationshipTables(
RelationshipType.ONE_TO_MANY RelationshipType.ONE_TO_MANY
) )
tableOrViewId = tableId
productCategoryTable = relatedTable productCategoryTable = relatedTable
productCatRows = await Promise.all([ productCatRows = await Promise.all([
@ -2260,15 +2298,15 @@ describe.each([
]) ])
await Promise.all([ await Promise.all([
config.api.row.save(table._id!, { config.api.row.save(tableOrViewId, {
name: "foo", name: "foo",
productCat: [productCatRows[0]._id], productCat: [productCatRows[0]._id],
}), }),
config.api.row.save(table._id!, { config.api.row.save(tableOrViewId, {
name: "bar", name: "bar",
productCat: [productCatRows[1]._id], productCat: [productCatRows[1]._id],
}), }),
config.api.row.save(table._id!, { config.api.row.save(tableOrViewId, {
name: "baz", name: "baz",
productCat: [], productCat: [],
}), }),
@ -2301,10 +2339,11 @@ describe.each([
isSql && isSql &&
describe("big relations", () => { describe("big relations", () => {
beforeAll(async () => { beforeAll(async () => {
const { relatedTable } = await basicRelationshipTables( const { relatedTable, tableId } = await basicRelationshipTables(
RelationshipType.MANY_TO_ONE RelationshipType.MANY_TO_ONE
) )
const mainRow = await config.api.row.save(table._id!, { tableOrViewId = tableId
const mainRow = await config.api.row.save(tableOrViewId, {
name: "foo", name: "foo",
}) })
for (let i = 0; i < 11; i++) { for (let i = 0; i < 11; i++) {
@ -2329,45 +2368,42 @@ describe.each([
}) })
;(isSqs || isLucene) && ;(isSqs || isLucene) &&
describe("relations to same table", () => { describe("relations to same table", () => {
let relatedTable: Table, relatedRows: Row[] let relatedTable: string, relatedRows: Row[]
beforeAll(async () => { beforeAll(async () => {
relatedTable = await createTable( relatedTable = await createTable({
{
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
}, })
"productCategory" tableOrViewId = await createTableOrView({
)
table = await createTable({
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
related1: { related1: {
type: FieldType.LINK, type: FieldType.LINK,
name: "related1", name: "related1",
fieldName: "main1", fieldName: "main1",
tableId: relatedTable._id!, tableId: relatedTable,
relationshipType: RelationshipType.MANY_TO_MANY, relationshipType: RelationshipType.MANY_TO_MANY,
}, },
related2: { related2: {
type: FieldType.LINK, type: FieldType.LINK,
name: "related2", name: "related2",
fieldName: "main2", fieldName: "main2",
tableId: relatedTable._id!, tableId: relatedTable,
relationshipType: RelationshipType.MANY_TO_MANY, relationshipType: RelationshipType.MANY_TO_MANY,
}, },
}) })
relatedRows = await Promise.all([ relatedRows = await Promise.all([
config.api.row.save(relatedTable._id!, { name: "foo" }), config.api.row.save(relatedTable, { name: "foo" }),
config.api.row.save(relatedTable._id!, { name: "bar" }), config.api.row.save(relatedTable, { name: "bar" }),
config.api.row.save(relatedTable._id!, { name: "baz" }), config.api.row.save(relatedTable, { name: "baz" }),
config.api.row.save(relatedTable._id!, { name: "boo" }), config.api.row.save(relatedTable, { name: "boo" }),
]) ])
await Promise.all([ await Promise.all([
config.api.row.save(table._id!, { config.api.row.save(tableOrViewId, {
name: "test", name: "test",
related1: [relatedRows[0]._id!], related1: [relatedRows[0]._id!],
related2: [relatedRows[1]._id!], related2: [relatedRows[1]._id!],
}), }),
config.api.row.save(table._id!, { config.api.row.save(tableOrViewId, {
name: "test2", name: "test2",
related1: [relatedRows[2]._id!], related1: [relatedRows[2]._id!],
related2: [relatedRows[3]._id!], related2: [relatedRows[3]._id!],
@ -2430,7 +2466,7 @@ describe.each([
isInternal && isInternal &&
describe("no column error backwards compat", () => { describe("no column error backwards compat", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
name: { name: {
name: "name", name: "name",
type: FieldType.STRING, type: FieldType.STRING,
@ -2453,7 +2489,7 @@ describe.each([
!isLucene && !isLucene &&
describe("row counting", () => { describe("row counting", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
name: { name: {
name: "name", name: "name",
type: FieldType.STRING, type: FieldType.STRING,
@ -2488,7 +2524,7 @@ describe.each([
describe("Invalid column definitions", () => { describe("Invalid column definitions", () => {
beforeAll(async () => { beforeAll(async () => {
// need to create an invalid table - means ignoring typescript // need to create an invalid table - means ignoring typescript
table = await createTable({ tableOrViewId = await createTableOrView({
// @ts-ignore // @ts-ignore
invalid: { invalid: {
type: FieldType.STRING, type: FieldType.STRING,
@ -2518,7 +2554,7 @@ describe.each([
"special (%s) case", "special (%s) case",
column => { column => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
[column]: { [column]: {
name: column, name: column,
type: FieldType.STRING, type: FieldType.STRING,
@ -2543,8 +2579,8 @@ describe.each([
describe("sample data", () => { describe("sample data", () => {
beforeAll(async () => { beforeAll(async () => {
await config.api.application.addSampleData(config.appId!) await config.api.application.addSampleData(config.appId!)
table = DEFAULT_EMPLOYEE_TABLE_SCHEMA tableOrViewId = DEFAULT_EMPLOYEE_TABLE_SCHEMA._id!
rows = await config.api.row.fetch(table._id!) rows = await config.api.row.fetch(tableOrViewId)
}) })
it("should be able to search sample data", async () => { it("should be able to search sample data", async () => {
@ -2567,7 +2603,7 @@ describe.each([
const earlyDate = "2024-07-03T10:00:00.000Z", const earlyDate = "2024-07-03T10:00:00.000Z",
laterDate = "2024-07-03T11:00:00.000Z" laterDate = "2024-07-03T11:00:00.000Z"
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
date: { date: {
name: "date", name: "date",
type: FieldType.DATETIME, type: FieldType.DATETIME,
@ -2610,7 +2646,7 @@ describe.each([
"ชื่อผู้ใช้", // Thai for "username" "ชื่อผู้ใช้", // Thai for "username"
])("non-ascii column name: %s", name => { ])("non-ascii column name: %s", name => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
[name]: { [name]: {
name, name,
type: FieldType.STRING, type: FieldType.STRING,
@ -2637,7 +2673,7 @@ describe.each([
isInternal && isInternal &&
describe("space at end of column name", () => { describe("space at end of column name", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
"name ": { "name ": {
name: "name ", name: "name ",
type: FieldType.STRING, type: FieldType.STRING,
@ -2672,7 +2708,7 @@ describe.each([
;(isSqs || isInMemory) && ;(isSqs || isInMemory) &&
describe("space at start of column name", () => { describe("space at start of column name", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
" name": { " name": {
name: " name", name: " name",
type: FieldType.STRING, type: FieldType.STRING,
@ -2703,9 +2739,10 @@ describe.each([
}) })
isSqs && isSqs &&
!isView &&
describe("duplicate columns", () => { describe("duplicate columns", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
name: { name: {
name: "name", name: "name",
type: FieldType.STRING, type: FieldType.STRING,
@ -2713,7 +2750,7 @@ describe.each([
}) })
await context.doInAppContext(config.getAppId(), async () => { await context.doInAppContext(config.getAppId(), async () => {
const db = context.getAppDB() const db = context.getAppDB()
const tableDoc = await db.get<Table>(table._id!) const tableDoc = await db.get<Table>(tableOrViewId)
tableDoc.schema.Name = { tableDoc.schema.Name = {
name: "Name", name: "Name",
type: FieldType.STRING, type: FieldType.STRING,
@ -2747,7 +2784,7 @@ describe.each([
type: FieldType.STRING, type: FieldType.STRING,
}, },
}) })
table = await createTable({ tableOrViewId = await createTableOrView({
name: { name: {
name: "name", name: "name",
type: FieldType.STRING, type: FieldType.STRING,
@ -2756,15 +2793,15 @@ describe.each([
name: "rel", name: "rel",
type: FieldType.LINK, type: FieldType.LINK,
relationshipType: RelationshipType.MANY_TO_MANY, relationshipType: RelationshipType.MANY_TO_MANY,
tableId: toRelateTable._id!, tableId: toRelateTable,
fieldName: "rel", fieldName: "rel",
}, },
}) })
const [row1, row2] = await Promise.all([ const [row1, row2] = await Promise.all([
config.api.row.save(toRelateTable._id!, { name: "tag 1" }), config.api.row.save(toRelateTable, { name: "tag 1" }),
config.api.row.save(toRelateTable._id!, { name: "tag 2" }), config.api.row.save(toRelateTable, { name: "tag 2" }),
]) ])
row = await config.api.row.save(table._id!, { row = await config.api.row.save(tableOrViewId, {
name: "product 1", name: "product 1",
rel: [row1._id, row2._id], rel: [row1._id, row2._id],
}) })
@ -2783,7 +2820,7 @@ describe.each([
!isInternal && !isInternal &&
describe("search by composite key", () => { describe("search by composite key", () => {
beforeAll(async () => { beforeAll(async () => {
table = await config.api.table.save( const table = await config.api.table.save(
tableForDatasource(datasource, { tableForDatasource(datasource, {
schema: { schema: {
idColumn1: { idColumn1: {
@ -2798,6 +2835,7 @@ describe.each([
primary: ["idColumn1", "idColumn2"], primary: ["idColumn1", "idColumn2"],
}) })
) )
tableOrViewId = table._id!
await createRows([{ idColumn1: 1, idColumn2: 2 }]) await createRows([{ idColumn1: 1, idColumn2: 2 }])
}) })
@ -2819,15 +2857,13 @@ describe.each([
isSql && isSql &&
describe("primaryDisplay", () => { describe("primaryDisplay", () => {
beforeAll(async () => { beforeAll(async () => {
let toRelateTable = await createTable({ let toRelateTableId = await createTable({
name: { name: {
name: "name", name: "name",
type: FieldType.STRING, type: FieldType.STRING,
}, },
}) })
table = await config.api.table.save( tableOrViewId = await createTableOrView({
tableForDatasource(datasource, {
schema: {
name: { name: {
name: "name", name: "name",
type: FieldType.STRING, type: FieldType.STRING,
@ -2836,33 +2872,30 @@ describe.each([
name: "link", name: "link",
type: FieldType.LINK, type: FieldType.LINK,
relationshipType: RelationshipType.MANY_TO_ONE, relationshipType: RelationshipType.MANY_TO_ONE,
tableId: toRelateTable._id!, tableId: toRelateTableId,
fieldName: "link", fieldName: "link",
}, },
},
}) })
)
toRelateTable = await config.api.table.get(toRelateTable._id!) const toRelateTable = await config.api.table.get(toRelateTableId)
await config.api.table.save({ await config.api.table.save({
...toRelateTable, ...toRelateTable,
primaryDisplay: "link", primaryDisplay: "link",
}) })
const relatedRows = await Promise.all([ const relatedRows = await Promise.all([
config.api.row.save(toRelateTable._id!, { name: "test" }), config.api.row.save(toRelateTable._id!, { name: "related" }),
]) ])
await Promise.all([ await config.api.row.save(tableOrViewId, {
config.api.row.save(table._id!, {
name: "test", name: "test",
link: relatedRows.map(row => row._id), link: relatedRows.map(row => row._id),
}), })
])
}) })
it("should be able to query, primary display on related table shouldn't be used", async () => { it("should be able to query, primary display on related table shouldn't be used", async () => {
// this test makes sure that if a relationship has been specified as the primary display on a table // this test makes sure that if a relationship has been specified as the primary display on a table
// it is ignored and another column is used instead // it is ignored and another column is used instead
await expectQuery({}).toContain([ await expectQuery({}).toContain([
{ name: "test", link: [{ primaryDisplay: "test" }] }, { name: "test", link: [{ primaryDisplay: "related" }] },
]) ])
}) })
}) })
@ -2870,7 +2903,7 @@ describe.each([
!isLucene && !isLucene &&
describe("$and", () => { describe("$and", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
age: { name: "age", type: FieldType.NUMBER }, age: { name: "age", type: FieldType.NUMBER },
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
}) })
@ -2945,7 +2978,10 @@ describe.each([
await expect( await expect(
expectQuery({ expectQuery({
$and: { $and: {
conditions: [{ equal: { age: 10 } }, "invalidCondition" as any], conditions: [
{ equal: { age: 10 } },
"invalidCondition" as any,
],
}, },
}).toFindNothing() }).toFindNothing()
).rejects.toThrow( ).rejects.toThrow(
@ -2973,6 +3009,8 @@ describe.each([
) )
}) })
// onEmptyFilter cannot be sent to view searches
!isView &&
it("returns no rows when onEmptyFilter set to none", async () => { it("returns no rows when onEmptyFilter set to none", async () => {
await expectSearch({ await expectSearch({
query: { query: {
@ -2999,7 +3037,7 @@ describe.each([
!isLucene && !isLucene &&
describe("$or", () => { describe("$or", () => {
beforeAll(async () => { beforeAll(async () => {
table = await createTable({ tableOrViewId = await createTableOrView({
age: { name: "age", type: FieldType.NUMBER }, age: { name: "age", type: FieldType.NUMBER },
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
}) })
@ -3123,6 +3161,8 @@ describe.each([
}).toContainExactly([{ age: 1, name: "Jane" }]) }).toContainExactly([{ age: 1, name: "Jane" }])
}) })
// onEmptyFilter cannot be sent to view searches
!isView &&
it("returns no rows when onEmptyFilter set to none", async () => { it("returns no rows when onEmptyFilter set to none", async () => {
await expectSearch({ await expectSearch({
query: { query: {
@ -3159,20 +3199,20 @@ describe.each([
row[name] = i row[name] = i
} }
const relatedTable = await createTable(relatedSchema) const relatedTable = await createTable(relatedSchema)
table = await createTable({ tableOrViewId = await createTableOrView({
name: { name: "name", type: FieldType.STRING }, name: { name: "name", type: FieldType.STRING },
related1: { related1: {
type: FieldType.LINK, type: FieldType.LINK,
name: "related1", name: "related1",
fieldName: "main1", fieldName: "main1",
tableId: relatedTable._id!, tableId: relatedTable,
relationshipType: RelationshipType.MANY_TO_MANY, relationshipType: RelationshipType.MANY_TO_MANY,
}, },
}) })
relatedRows = await Promise.all([ relatedRows = await Promise.all([
config.api.row.save(relatedTable._id!, row), config.api.row.save(relatedTable, row),
]) ])
await config.api.row.save(table._id!, { await config.api.row.save(tableOrViewId, {
name: "foo", name: "foo",
related1: [relatedRows[0]._id], related1: [relatedRows[0]._id],
}) })
@ -3187,4 +3227,5 @@ describe.each([
]) ])
}) })
}) })
})
}) })

11
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -22,10 +22,10 @@ import {
RelationshipType, RelationshipType,
TableSchema, TableSchema,
RenameColumn, RenameColumn,
ViewFieldMetadata,
FeatureFlag, FeatureFlag,
BBReferenceFieldSubType, BBReferenceFieldSubType,
NumericCalculationFieldMetadata, NumericCalculationFieldMetadata,
ViewV2Schema,
} from "@budibase/types" } from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests" import { generator, mocks } from "@budibase/backend-core/tests"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@ -155,7 +155,7 @@ describe.each([
}) })
it("can persist views with all fields", async () => { it("can persist views with all fields", async () => {
const newView: Required<CreateViewRequest> = { const newView: Required<Omit<CreateViewRequest, "queryUI">> = {
name: generator.name(), name: generator.name(),
tableId: table._id!, tableId: table._id!,
primaryDisplay: "id", primaryDisplay: "id",
@ -612,7 +612,7 @@ describe.each([
it("can update all fields", async () => { it("can update all fields", async () => {
const tableId = table._id! const tableId = table._id!
const updatedData: Required<UpdateViewRequest> = { const updatedData: Required<Omit<UpdateViewRequest, "queryUI">> = {
version: view.version, version: view.version,
id: view.id, id: view.id,
tableId, tableId,
@ -1359,10 +1359,7 @@ describe.each([
return table return table
} }
const createView = async ( const createView = async (tableId: string, schema: ViewV2Schema) =>
tableId: string,
schema: Record<string, ViewFieldMetadata>
) =>
await config.api.viewV2.create({ await config.api.viewV2.create({
name: generator.guid(), name: generator.guid(),
tableId, tableId,

4
packages/server/src/db/linkedRows/index.ts
View File

@ -23,8 +23,8 @@ import {
Row, Row,
Table, Table,
TableSchema, TableSchema,
ViewFieldMetadata,
ViewV2, ViewV2,
ViewV2Schema,
} from "@budibase/types" } from "@budibase/types"
import sdk from "../../sdk" import sdk from "../../sdk"
import { helpers } from "@budibase/shared-core" import { helpers } from "@budibase/shared-core"
@ -262,7 +262,7 @@ export async function squashLinks<T = Row[] | Row>(
FeatureFlag.ENRICHED_RELATIONSHIPS FeatureFlag.ENRICHED_RELATIONSHIPS
) )
let viewSchema: Record<string, ViewFieldMetadata> = {} let viewSchema: ViewV2Schema = {}
if (sdk.views.isView(source)) { if (sdk.views.isView(source)) {
if (helpers.views.isCalculationView(source)) { if (helpers.views.isCalculationView(source)) {
return enriched return enriched

47
packages/server/src/sdk/app/rows/search.ts
View File

@ -1,10 +1,10 @@
import { import {
EmptyFilterOption, EmptyFilterOption,
LegacyFilter,
LogicalOperator, LogicalOperator,
Row, Row,
RowSearchParams, RowSearchParams,
SearchFilterKey, SearchFilterKey,
SearchFilters,
SearchResponse, SearchResponse,
SortOrder, SortOrder,
Table, Table,
@ -61,7 +61,6 @@ export async function search(
if (options.viewId) { if (options.viewId) {
source = await sdk.views.get(options.viewId) source = await sdk.views.get(options.viewId)
table = await sdk.views.getTable(source) table = await sdk.views.getTable(source)
options = searchInputMapping(table, options)
} else if (options.tableId) { } else if (options.tableId) {
source = await sdk.tables.getTable(options.tableId) source = await sdk.tables.getTable(options.tableId)
table = source table = source
@ -74,7 +73,7 @@ export async function search(
if (options.query) { if (options.query) {
const visibleFields = ( const visibleFields = (
options.fields || Object.keys(table.schema) options.fields || Object.keys(table.schema)
).filter(field => table.schema[field].visible !== false) ).filter(field => table.schema[field]?.visible !== false)
const queryableFields = await getQueryableFields(table, visibleFields) const queryableFields = await getQueryableFields(table, visibleFields)
options.query = removeInvalidFilters(options.query, queryableFields) options.query = removeInvalidFilters(options.query, queryableFields)
@ -82,39 +81,59 @@ export async function search(
options.query = {} options.query = {}
} }
// need to make sure filters in correct shape before checking for view
options = searchInputMapping(table, options)
if (options.viewId) { if (options.viewId) {
const view = await sdk.views.get(options.viewId) // Delete extraneous search params that cannot be overridden
delete options.query.onEmptyFilter
const view = source as ViewV2
// Enrich saved query with ephemeral query params. // Enrich saved query with ephemeral query params.
// We prevent searching on any fields that are saved as part of the query, as // We prevent searching on any fields that are saved as part of the query, as
// that could let users find rows they should not be allowed to access. // that could let users find rows they should not be allowed to access.
let viewQuery = dataFilters.buildQuery(view.query || []) let viewQuery = dataFilters.buildQueryLegacy(view.query) || {}
delete viewQuery?.onEmptyFilter
if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) { const sqsEnabled = await features.flags.isEnabled("SQS")
// Lucene does not accept conditional filters, so we need to keep the old logic const supportsLogicalOperators =
const query: SearchFilters = viewQuery isExternalTableID(view.tableId) || sqsEnabled
if (!supportsLogicalOperators) {
// In the unlikely event that a Grouped Filter is in a non-SQS environment
// It needs to be ignored entirely
let queryFilters: LegacyFilter[] = Array.isArray(view.query)
? view.query
: []
delete options.query.onEmptyFilter
// Extract existing fields // Extract existing fields
const existingFields = const existingFields =
view.query queryFilters
?.filter(filter => filter.field) ?.filter(filter => filter.field)
.map(filter => db.removeKeyNumbering(filter.field)) || [] .map(filter => db.removeKeyNumbering(filter.field)) || []
viewQuery ??= {}
// Carry over filters for unused fields // Carry over filters for unused fields
Object.keys(options.query || {}).forEach(key => { Object.keys(options.query).forEach(key => {
const operator = key as Exclude<SearchFilterKey, LogicalOperator> const operator = key as Exclude<SearchFilterKey, LogicalOperator>
Object.keys(options.query[operator] || {}).forEach(field => { Object.keys(options.query[operator] || {}).forEach(field => {
if (!existingFields.includes(db.removeKeyNumbering(field))) { if (!existingFields.includes(db.removeKeyNumbering(field))) {
query[operator]![field] = options.query[operator]![field] viewQuery![operator]![field] = options.query[operator]![field]
} }
}) })
}) })
options.query = query options.query = viewQuery
} else { } else {
const conditions = viewQuery ? [viewQuery] : []
options.query = { options.query = {
$and: { $and: {
conditions: [viewQuery, options.query], conditions: [...conditions, options.query],
}, },
} }
if (viewQuery.onEmptyFilter) {
options.query.onEmptyFilter = viewQuery.onEmptyFilter
}
} }
} }
@ -143,8 +162,6 @@ export async function search(
options.sortOrder = options.sortOrder.toLowerCase() as SortOrder options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
} }
options = searchInputMapping(table, options)
let result: SearchResponse<Row> let result: SearchResponse<Row>
if (isExternalTable) { if (isExternalTable) {
span?.addTags({ searchType: "external" }) span?.addTags({ searchType: "external" })

2
packages/server/src/sdk/app/rows/search/utils.ts
View File

@ -107,7 +107,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) {
} }
return dataFilters.recurseLogicalOperators(filters, checkFilters) return dataFilters.recurseLogicalOperators(filters, checkFilters)
} }
if (options.query) {
options.query = checkFilters(options.query) options.query = checkFilters(options.query)
}
return options return options
} }

2
packages/server/src/sdk/app/views/index.ts
View File

@ -317,7 +317,7 @@ export async function enrichSchema(
// if nothing specified in view, then it is not visible // if nothing specified in view, then it is not visible
const ui = viewSchema[key] || { visible: false } const ui = viewSchema[key] || { visible: false }
schema[key] = { schema[key] = {
...(tableSchema[key] || {}), ...tableSchema[key],
...ui, ...ui,
order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key]?.order, order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key]?.order,
columns: undefined, columns: undefined,

4
packages/server/src/tests/utilities/api/row.ts
View File

@ -7,11 +7,11 @@ import {
BulkImportRequest, BulkImportRequest,
BulkImportResponse, BulkImportResponse,
SearchRowResponse, SearchRowResponse,
RowSearchParams,
DeleteRows, DeleteRows,
DeleteRow, DeleteRow,
PaginatedSearchRowResponse, PaginatedSearchRowResponse,
RowExportFormat, RowExportFormat,
SearchRowRequest,
} from "@budibase/types" } from "@budibase/types"
import { Expectations, TestAPI } from "./base" import { Expectations, TestAPI } from "./base"
@ -136,7 +136,7 @@ export class RowAPI extends TestAPI {
) )
} }
search = async <T extends RowSearchParams>( search = async <T extends SearchRowRequest>(
sourceId: string, sourceId: string,
params?: T, params?: T,
expectations?: Expectations expectations?: Expectations

207
packages/shared-core/src/filters.ts
View File

@ -3,7 +3,7 @@ import {
BBReferenceFieldSubType, BBReferenceFieldSubType,
FieldType, FieldType,
FormulaType, FormulaType,
SearchFilter, LegacyFilter,
SearchFilters, SearchFilters,
SearchQueryFields, SearchQueryFields,
ArrayOperator, ArrayOperator,
@ -19,9 +19,12 @@ import {
RangeOperator, RangeOperator,
LogicalOperator, LogicalOperator,
isLogicalSearchOperator, isLogicalSearchOperator,
SearchFilterGroup,
FilterGroupLogicalOperator,
} from "@budibase/types" } from "@budibase/types"
import dayjs from "dayjs" import dayjs from "dayjs"
import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
import { processSearchFilters } from "./utils"
import { deepGet, schema } from "./helpers" import { deepGet, schema } from "./helpers"
import { isPlainObject, isEmpty } from "lodash" import { isPlainObject, isEmpty } from "lodash"
import { decodeNonAscii } from "./helpers/schema" import { decodeNonAscii } from "./helpers/schema"
@ -124,7 +127,7 @@ export function recurseLogicalOperators(
fn: (f: SearchFilters) => SearchFilters fn: (f: SearchFilters) => SearchFilters
) { ) {
for (const logical of LOGICAL_OPERATORS) { for (const logical of LOGICAL_OPERATORS) {
if (filters?.[logical]) { if (filters[logical]) {
filters[logical]!.conditions = filters[logical]!.conditions.map( filters[logical]!.conditions = filters[logical]!.conditions.map(
condition => fn(condition) condition => fn(condition)
) )
@ -160,9 +163,6 @@ export function recurseSearchFilters(
* https://github.com/Budibase/budibase/issues/10118 * https://github.com/Budibase/budibase/issues/10118
*/ */
export const cleanupQuery = (query: SearchFilters) => { export const cleanupQuery = (query: SearchFilters) => {
if (!query) {
return query
}
for (let filterField of NoEmptyFilterStrings) { for (let filterField of NoEmptyFilterStrings) {
if (!query[filterField]) { if (!query[filterField]) {
continue continue
@ -304,10 +304,143 @@ export class ColumnSplitter {
} }
/** /**
* Builds a JSON query from the filter structure generated in the builder * Builds a JSON query from the filter a SearchFilter definition
* @param filter the builder filter structure * @param filter the builder filter structure
*/ */
export const buildQuery = (filter: SearchFilter[]) => {
const buildCondition = (expression: LegacyFilter) => {
// Filter body
let query: SearchFilters = {
string: {},
fuzzy: {},
range: {},
equal: {},
notEqual: {},
empty: {},
notEmpty: {},
contains: {},
notContains: {},
oneOf: {},
containsAny: {},
}
let { operator, field, type, value, externalType, onEmptyFilter } = expression
if (!operator || !field) {
return
}
const queryOperator = operator as SearchFilterOperator
const isHbs =
typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
// Parse all values into correct types
if (operator === "allOr") {
query.allOr = true
return
}
if (onEmptyFilter) {
query.onEmptyFilter = onEmptyFilter
return
}
// Default the value for noValue fields to ensure they are correctly added
// to the final query
if (queryOperator === "empty" || queryOperator === "notEmpty") {
value = null
}
if (
type === "datetime" &&
!isHbs &&
queryOperator !== "empty" &&
queryOperator !== "notEmpty"
) {
// Ensure date value is a valid date and parse into correct format
if (!value) {
return
}
try {
value = new Date(value).toISOString()
} catch (error) {
return
}
}
if (type === "number" && typeof value === "string" && !isHbs) {
if (queryOperator === "oneOf") {
value = value.split(",").map(item => parseFloat(item))
} else {
value = parseFloat(value)
}
}
if (type === "boolean") {
value = `${value}`?.toLowerCase() === "true"
}
if (
["contains", "notContains", "containsAny"].includes(
operator.toLocaleString()
) &&
type === "array" &&
typeof value === "string"
) {
value = value.split(",")
}
if (operator.toLocaleString().startsWith("range") && query.range) {
const minint =
SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap]
?.min || Number.MIN_SAFE_INTEGER
const maxint =
SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap]
?.max || Number.MAX_SAFE_INTEGER
if (!query.range[field]) {
query.range[field] = {
low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
}
}
if (operator === "rangeLow" && value != null && value !== "") {
query.range[field] = {
...query.range[field],
low: value,
}
} else if (operator === "rangeHigh" && value != null && value !== "") {
query.range[field] = {
...query.range[field],
high: value,
}
}
} else if (isLogicalSearchOperator(queryOperator)) {
// TODO
} else if (query[queryOperator] && operator !== "onEmptyFilter") {
if (type === "boolean") {
// Transform boolean filters to cope with null.
// "equals false" needs to be "not equals true"
// "not equals false" needs to be "equals true"
if (queryOperator === "equal" && value === false) {
query.notEqual = query.notEqual || {}
query.notEqual[field] = true
} else if (queryOperator === "notEqual" && value === false) {
query.equal = query.equal || {}
query.equal[field] = true
} else {
query[queryOperator] ??= {}
query[queryOperator]![field] = value
}
} else {
query[queryOperator] ??= {}
query[queryOperator]![field] = value
}
}
return query
}
export const buildQueryLegacy = (
filter?: LegacyFilter[] | SearchFilters
): SearchFilters | undefined => {
// this is of type SearchFilters or is undefined
if (!Array.isArray(filter)) {
return filter
}
let query: SearchFilters = { let query: SearchFilters = {
string: {}, string: {},
fuzzy: {}, fuzzy: {},
@ -368,13 +501,15 @@ export const buildQuery = (filter: SearchFilter[]) => {
value = `${value}`?.toLowerCase() === "true" value = `${value}`?.toLowerCase() === "true"
} }
if ( if (
["contains", "notContains", "containsAny"].includes(operator) && ["contains", "notContains", "containsAny"].includes(
operator.toLocaleString()
) &&
type === "array" && type === "array" &&
typeof value === "string" typeof value === "string"
) { ) {
value = value.split(",") value = value.split(",")
} }
if (operator.startsWith("range") && query.range) { if (operator.toLocaleString().startsWith("range") && query.range) {
const minint = const minint =
SqlNumberTypeRangeMap[ SqlNumberTypeRangeMap[
externalType as keyof typeof SqlNumberTypeRangeMap externalType as keyof typeof SqlNumberTypeRangeMap
@ -401,7 +536,7 @@ export const buildQuery = (filter: SearchFilter[]) => {
} }
} }
} else if (isLogicalSearchOperator(queryOperator)) { } else if (isLogicalSearchOperator(queryOperator)) {
// TODO // ignore
} else if (query[queryOperator] && operator !== "onEmptyFilter") { } else if (query[queryOperator] && operator !== "onEmptyFilter") {
if (type === "boolean") { if (type === "boolean") {
// Transform boolean filters to cope with null. // Transform boolean filters to cope with null.
@ -423,10 +558,60 @@ export const buildQuery = (filter: SearchFilter[]) => {
} }
} }
}) })
return query return query
} }
/**
* Converts a **SearchFilterGroup** filter definition into a grouped
* search query of type **SearchFilters**
*
* Legacy support remains for the old **SearchFilter[]** format.
* These will be migrated to an appropriate **SearchFilters** object, if encountered
*
* @param filter
*
* @returns {SearchFilters}
*/
export const buildQuery = (
filter?: SearchFilterGroup | LegacyFilter[]
): SearchFilters | undefined => {
const parsedFilter: SearchFilterGroup | undefined =
processSearchFilters(filter)
if (!parsedFilter) {
return
}
const operatorMap: { [key in FilterGroupLogicalOperator]: LogicalOperator } =
{
[FilterGroupLogicalOperator.ALL]: LogicalOperator.AND,
[FilterGroupLogicalOperator.ANY]: LogicalOperator.OR,
}
const globalOnEmpty = parsedFilter.onEmptyFilter
? parsedFilter.onEmptyFilter
: null
const globalOperator: LogicalOperator =
operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator]
return {
...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
[globalOperator]: {
conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => {
return {
[operatorMap[group.logicalOperator]]: {
conditions: group.filters
?.map(x => buildCondition(x))
.filter(filter => filter),
},
}
}),
},
}
}
// The frontend can send single values for array fields sometimes, so to handle // The frontend can send single values for array fields sometimes, so to handle
// this we convert them to arrays at the controller level so that nothing below // this we convert them to arrays at the controller level so that nothing below
// this has to worry about the non-array values. // this has to worry about the non-array values.

109
packages/shared-core/src/utils.ts
View File

@ -1,5 +1,17 @@
import { ArrayOperator, BasicOperator, SearchFilters } from "@budibase/types" import {
LegacyFilter,
SearchFilterGroup,
FilterGroupLogicalOperator,
SearchFilters,
BasicOperator,
ArrayOperator,
} from "@budibase/types"
import * as Constants from "./constants" import * as Constants from "./constants"
import { removeKeyNumbering } from "./filters"
// an array of keys from filter type to properties that are in the type
// this can then be converted using .fromEntries to an object
type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][]
export function unreachable( export function unreachable(
value: never, value: never,
@ -104,3 +116,98 @@ export function isSupportedUserSearch(query: SearchFilters) {
} }
return true return true
} }
/**
* Processes the filter config. Filters are migrated from
* SearchFilter[] to SearchFilterGroup
*
* If filters is not an array, the migration is skipped
*
* @param {LegacyFilter[] | SearchFilterGroup} filters
*/
export const processSearchFilters = (
filters: LegacyFilter[] | SearchFilterGroup | undefined
): SearchFilterGroup | undefined => {
if (!filters) {
return
}
// Base search config.
const defaultCfg: SearchFilterGroup = {
logicalOperator: FilterGroupLogicalOperator.ALL,
groups: [],
}
const filterAllowedKeys = [
"field",
"operator",
"value",
"type",
"externalType",
"valueType",
"noValue",
"formulaType",
]
if (Array.isArray(filters)) {
let baseGroup: SearchFilterGroup = {
filters: [],
logicalOperator: FilterGroupLogicalOperator.ALL,
}
return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => {
// Sort the properties for easier debugging
const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[])
.sort((a, b) => {
return a.localeCompare(b)
})
.filter(key => key in filter)
if (filterPropertyKeys.length == 1) {
const key = filterPropertyKeys[0],
value = filter[key]
// Global
if (key === "onEmptyFilter") {
// unset otherwise
acc.onEmptyFilter = value
} else if (key === "operator" && value === "allOr") {
// Group 1 logical operator
baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
}
return acc
}
const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce(
(acc: AllowedFilters, key) => {
const value = filter[key]
if (filterAllowedKeys.includes(key)) {
if (key === "field") {
acc.push([key, removeKeyNumbering(value)])
} else {
acc.push([key, value])
}
}
return acc
},
[]
)
const migratedFilter: LegacyFilter = Object.fromEntries(
allowedFilterSettings
) as LegacyFilter
baseGroup.filters!.push(migratedFilter)
if (!acc.groups || !acc.groups.length) {
// init the base group
acc.groups = [baseGroup]
}
return acc
}, defaultCfg)
} else if (!filters?.groups) {
return
}
return filters
}

16
packages/types/src/api/web/searchFilter.ts
View File

@ -1,7 +1,11 @@
import { FieldType } from "../../documents" import { FieldType } from "../../documents"
import { EmptyFilterOption, SearchFilters } from "../../sdk" import {
EmptyFilterOption,
FilterGroupLogicalOperator,
SearchFilters,
} from "../../sdk"
export type SearchFilter = { export type LegacyFilter = {
operator: keyof SearchFilters | "rangeLow" | "rangeHigh" operator: keyof SearchFilters | "rangeLow" | "rangeHigh"
onEmptyFilter?: EmptyFilterOption onEmptyFilter?: EmptyFilterOption
field: string field: string
@ -9,3 +13,11 @@ export type SearchFilter = {
value: any value: any
externalType?: string externalType?: string
} }
// this is a type purely used by the UI
export type SearchFilterGroup = {
logicalOperator: FilterGroupLogicalOperator
onEmptyFilter?: EmptyFilterOption
groups?: SearchFilterGroup[]
filters?: LegacyFilter[]
}

12
packages/types/src/documents/app/view.ts
View File

@ -1,7 +1,7 @@
import { SearchFilter, SortOrder, SortType } from "../../api" import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
import { UIFieldMetadata } from "./table" import { UIFieldMetadata } from "./table"
import { Document } from "../document" import { Document } from "../document"
import { DBView } from "../../sdk" import { DBView, SearchFilters } from "../../sdk"
export type ViewTemplateOpts = { export type ViewTemplateOpts = {
field: string field: string
@ -85,15 +85,19 @@ export interface ViewV2 {
name: string name: string
primaryDisplay?: string primaryDisplay?: string
tableId: string tableId: string
query?: SearchFilter[] query?: LegacyFilter[] | SearchFilters
// duplicate to store UI information about filters
queryUI?: SearchFilterGroup
sort?: { sort?: {
field: string field: string
order?: SortOrder order?: SortOrder
type?: SortType type?: SortType
} }
schema?: Record<string, ViewFieldMetadata> schema?: ViewV2Schema
} }
export type ViewV2Schema = Record<string, ViewFieldMetadata>
export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema
export interface ViewCountOrSumSchema { export interface ViewCountOrSumSchema {

5
packages/types/src/sdk/search.ts
View File

@ -191,6 +191,11 @@ export enum EmptyFilterOption {
RETURN_NONE = "none", RETURN_NONE = "none",
} }
export enum FilterGroupLogicalOperator {
ALL = "all",
ANY = "any",
}
export enum SqlClient { export enum SqlClient {
MS_SQL = "mssql", MS_SQL = "mssql",
POSTGRES = "pg", POSTGRES = "pg",

22
yarn.lock
View File

@ -2051,7 +2051,7 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@2.32.10": "@budibase/backend-core@2.32.11":
version "0.0.0" version "0.0.0"
dependencies: dependencies:
"@budibase/nano" "10.1.5" "@budibase/nano" "10.1.5"
@ -2132,15 +2132,15 @@
through2 "^2.0.0" through2 "^2.0.0"
"@budibase/pro@npm:@budibase/pro@latest": "@budibase/pro@npm:@budibase/pro@latest":
version "2.32.10" version "2.32.11"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.10.tgz#19fcbb3ced74791a7e96dfdc5a1270165792eea5" resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.11.tgz#c94d534f829ca0ef252677757e157a7e58b87b4d"
integrity sha512-TbVp2bjmA0rHK+TKi9NVW06+O23fhDm7IJ/FlpWPHIBIZW7xDkCYu6LUOhSwSWMbOTcWzaJFuMbpN1HoTc/YjQ== integrity sha512-mOkqJpqHKWsfTWZwWcvBCYFUIluSUHltQNinc1ZRsg9rC3OKoHSDop6gzm744++H/GzGRN8V86kLhCgtNIlkpA==
dependencies: dependencies:
"@anthropic-ai/sdk" "^0.27.3" "@anthropic-ai/sdk" "^0.27.3"
"@budibase/backend-core" "2.32.10" "@budibase/backend-core" "2.32.11"
"@budibase/shared-core" "2.32.10" "@budibase/shared-core" "2.32.11"
"@budibase/string-templates" "2.32.10" "@budibase/string-templates" "2.32.11"
"@budibase/types" "2.32.10" "@budibase/types" "2.32.11"
"@koa/router" "8.0.8" "@koa/router" "8.0.8"
bull "4.10.1" bull "4.10.1"
dd-trace "5.2.0" dd-trace "5.2.0"
@ -2153,13 +2153,13 @@
scim-patch "^0.8.1" scim-patch "^0.8.1"
scim2-parse-filter "^0.2.8" scim2-parse-filter "^0.2.8"
"@budibase/shared-core@2.32.10": "@budibase/shared-core@2.32.11":
version "0.0.0" version "0.0.0"
dependencies: dependencies:
"@budibase/types" "0.0.0" "@budibase/types" "0.0.0"
cron-validate "1.4.5" cron-validate "1.4.5"
"@budibase/string-templates@2.32.10": "@budibase/string-templates@2.32.11":
version "0.0.0" version "0.0.0"
dependencies: dependencies:
"@budibase/handlebars-helpers" "^0.13.2" "@budibase/handlebars-helpers" "^0.13.2"
@ -2167,7 +2167,7 @@
handlebars "^4.7.8" handlebars "^4.7.8"
lodash.clonedeep "^4.5.0" lodash.clonedeep "^4.5.0"
"@budibase/types@2.32.10": "@budibase/types@2.32.11":
version "0.0.0" version "0.0.0"
dependencies: dependencies:
scim-patch "^0.8.1" scim-patch "^0.8.1"