MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14394 from Budibase/fix/logical-operator-filter-cleanup 

Search logical operator fixes
This commit is contained in:
Michael Drury 2024-08-21 13:23:40 +01:00 committed by GitHub
parent 53e01aa241 bbe2763e54
commit f4484f45aa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 84 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
packages/backend-core/src/sql/sql.ts
View File

@ -36,6 +36,7 @@ import {
} from "@budibase/types"
import environment from "../environment"
import { dataFilters, helpers } from "@budibase/shared-core"
import { cloneDeep } from "lodash"
type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any
@ -268,6 +269,7 @@ class InternalBuilder {
}
private parseFilters(filters: SearchFilters): SearchFilters {
filters = cloneDeep(filters)
for (const op of Object.values(BasicOperator)) {
const filter = filters[op]
if (!filter) {
@ -371,10 +373,11 @@ class InternalBuilder {
),
castedTypeValue.values
)
} else if (!opts?.relationship && !isRelationshipField) {
} else if (!isRelationshipField) {
const alias = getTableAlias(tableName)
fn(alias ? `${alias}.${updatedKey}` : updatedKey, value)
} else if (opts?.relationship && isRelationshipField) {
}
if (opts?.relationship && isRelationshipField) {
const [filterTableName, property] = updatedKey.split(".")
const alias = getTableAlias(filterTableName)
fn(alias ? `${alias}.${property}` : property, value)

39
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -45,6 +45,7 @@ import { db as dbCore } from "@budibase/backend-core"
import sdk from "../../../sdk"
import env from "../../../environment"
import { makeExternalQuery } from "../../../integrations/base/query"
import { dataFilters } from "@budibase/shared-core"
export interface ManyRelationship {
tableId?: string
@ -195,29 +196,33 @@ export class ExternalRequest<T extends Operation> {
if (filters) {
// need to map over the filters and make sure the _id field isn't present
let prefix = 1
for (const [operatorType, operator] of Object.entries(filters)) {
const isArrayOp = sdk.rows.utils.isArrayFilter(operatorType)
for (const field of Object.keys(operator || {})) {
if (dbCore.removeKeyNumbering(field) === "_id") {
if (primary) {
const parts = breakRowIdField(operator[field])
if (primary.length > 1 && isArrayOp) {
operator[InternalSearchFilterOperator.COMPLEX_ID_OPERATOR] = {
id: primary,
values: parts[0],
const checkFilters = (innerFilters: SearchFilters): SearchFilters => {
for (const [operatorType, operator] of Object.entries(innerFilters)) {
const isArrayOp = sdk.rows.utils.isArrayFilter(operatorType)
for (const field of Object.keys(operator || {})) {
if (dbCore.removeKeyNumbering(field) === "_id") {
if (primary) {
const parts = breakRowIdField(operator[field])
if (primary.length > 1 && isArrayOp) {
operator[InternalSearchFilterOperator.COMPLEX_ID_OPERATOR] = {
id: primary,
values: parts[0],
}
} else {
for (let field of primary) {
operator[`${prefix}:${field}`] = parts.shift()
}
prefix++
}
} else {
for (let field of primary) {
operator[`${prefix}:${field}`] = parts.shift()
}
prefix++
}
// make sure this field doesn't exist on any filter
delete operator[field]
}
// make sure this field doesn't exist on any filter
delete operator[field]
}
}
return dataFilters.recurseLogicalOperators(innerFilters, checkFilters)
}
checkFilters(filters)
}
// there is no id, just use the user provided filters
if (!idCopy || !table) {

3
packages/server/src/api/controllers/row/views.ts
View File

@ -57,13 +57,12 @@ export async function searchView(
}
})
})
} else {
} else
query = {
$and: {
conditions: [query, body.query],
},
}
}
}
await context.ensureSnippetContext(true)

21
packages/server/src/integrations/tests/sql.spec.ts
View File

@ -194,8 +194,8 @@ describe("SQL query builder", () => {
})
)
expect(query).toEqual({
bindings: ["john%", limit, 5000],
sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
bindings: ["john%", limit, "john%", 5000],
sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" where LOWER("test"."name") LIKE :3 order by "test"."id" asc) where rownum <= :4`,
})
query = new Sql(SqlClient.ORACLE, limit)._query(
@ -208,9 +208,10 @@ describe("SQL query builder", () => {
},
})
)
const filterSet = [`%20%`, `%25%`, `%"john"%`, `%"mary"%`]
expect(query).toEqual({
bindings: ["%20%", "%25%", `%"john"%`, `%"mary"%`, limit, 5000],
sql: `select * from (select * from (select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5) "test" order by "test"."id" asc) where rownum <= :6`,
bindings: [...filterSet, limit, ...filterSet, 5000],
sql: `select * from (select * from (select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5) "test" where COALESCE(LOWER("test"."age"), '') LIKE :6 AND COALESCE(LOWER("test"."age"), '') LIKE :7 and COALESCE(LOWER("test"."name"), '') LIKE :8 AND COALESCE(LOWER("test"."name"), '') LIKE :9 order by "test"."id" asc) where rownum <= :10`,
})
query = new Sql(SqlClient.ORACLE, limit)._query(
@ -223,8 +224,8 @@ describe("SQL query builder", () => {
})
)
expect(query).toEqual({
bindings: [`%jo%`, limit, 5000],
sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
bindings: [`%jo%`, limit, `%jo%`, 5000],
sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" where LOWER("test"."name") LIKE :3 order by "test"."id" asc) where rownum <= :4`,
})
})
@ -241,8 +242,8 @@ describe("SQL query builder", () => {
)
expect(query).toEqual({
bindings: ["John", limit, 5000],
sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
bindings: ["John", limit, "John", 5000],
sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2) "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :3) order by "test"."id" asc) where rownum <= :4`,
})
})
@ -259,8 +260,8 @@ describe("SQL query builder", () => {
)
expect(query).toEqual({
bindings: ["John", limit, 5000],
sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`,
bindings: ["John", limit, "John", 5000],
sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2) "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :3) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :4`,
})
})
})

21
packages/server/src/integrations/tests/sqlAlias.spec.ts
View File

@ -97,13 +97,14 @@ describe("Captures of real examples", () => {
const filters = queryJson.filters?.oneOf?.taskid as number[]
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
expect(query).toEqual({
bindings: [...filters, limit, limit],
sql: multiline(`select "a"."executorid" as "a.executorid", "a"."taskname" as "a.taskname",
"a"."taskid" as "a.taskid", "a"."completed" as "a.completed", "a"."qaid" as "a.qaid",
"b"."productname" as "b.productname", "b"."productid" as "b.productid"
from (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) as "a"
left join "products_tasks" as "c" on "a"."taskid" = "c"."taskid"
left join "products" as "b" on "b"."productid" = "c"."productid" order by "a"."taskid" asc limit $4`),
bindings: [...filters, limit, ...filters, limit],
sql: multiline(
`select "a"."executorid" as "a.executorid", "a"."taskname" as "a.taskname", "a"."taskid" as "a.taskid",
"a"."completed" as "a.completed", "a"."qaid" as "a.qaid", "b"."productname" as "b.productname", "b"."productid" as "b.productid"
from (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) as "a"
left join "products_tasks" as "c" on "a"."taskid" = "c"."taskid" left join "products" as "b" on "b"."productid" = "c"."productid"
where "a"."taskid" in ($4, $5) order by "a"."taskid" asc limit $6`
),
})
})
@ -123,6 +124,7 @@ describe("Captures of real examples", () => {
rangeValue.low,
rangeValue.high,
equalValue,
true,
limit,
],
sql: expect.stringContaining(
@ -186,8 +188,9 @@ describe("Captures of real examples", () => {
}, queryJson)
expect(returningQuery).toEqual({
sql: multiline(`select top (@p0) * from (select top (@p1) * from [people] where CASE WHEN [people].[name] = @p2
THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p3 THEN 1 ELSE 0 END = 1 order by [people].[name] asc) as [people]`),
bindings: [5000, 1, "Test", 22],
THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p3 THEN 1 ELSE 0 END = 1 order by [people].[name] asc) as [people]
where CASE WHEN [people].[name] = @p4 THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p5 THEN 1 ELSE 0 END = 1`),
bindings: [5000, 1, "Test", 22, "Test", 22],
})
})
})

5
packages/server/src/sdk/app/rows/search/filters.ts
View File

@ -5,6 +5,7 @@ import {
Table,
} from "@budibase/types"
import { isPlainObject } from "lodash"
import { dataFilters } from "@budibase/shared-core"
export function getRelationshipColumns(table: Table): {
name: string
@ -58,5 +59,7 @@ export function updateFilterKeys(
}
}
}
return filters
return dataFilters.recurseLogicalOperators(filters, (f: SearchFilters) => {
return updateFilterKeys(f, updates)
})
}

21
packages/server/src/sdk/app/rows/sqlAlias.ts
View File

@ -12,6 +12,7 @@ import { getSQLClient } from "./utils"
import { cloneDeep } from "lodash"
import datasources from "../datasources"
import { BudibaseInternalDB } from "../../../db/utils"
import { dataFilters } from "@budibase/shared-core"
type PerformQueryFunction = (
datasource: Datasource,
@ -199,16 +200,20 @@ export default class AliasTables {
)
}
if (json.filters) {
for (let [filterKey, filter] of Object.entries(json.filters)) {
if (typeof filter !== "object") {
continue
const aliasFilters = (filters: SearchFilters): SearchFilters => {
for (let [filterKey, filter] of Object.entries(filters)) {
if (typeof filter !== "object") {
continue
}
const aliasedFilters: typeof filter = {}
for (let key of Object.keys(filter)) {
aliasedFilters[this.aliasField(key)] = filter[key]
}
filters[filterKey as keyof SearchFilters] = aliasedFilters
}
const aliasedFilters: typeof filter = {}
for (let key of Object.keys(filter)) {
aliasedFilters[this.aliasField(key)] = filter[key]
}
json.filters[filterKey as keyof SearchFilters] = aliasedFilters
return dataFilters.recurseLogicalOperators(filters, aliasFilters)
}
json.filters = aliasFilters(json.filters)
}
if (json.meta?.table) {
this.getAlias(json.meta.table.name)

16
packages/shared-core/src/filters.ts
View File

@ -113,6 +113,20 @@ export const NoEmptyFilterStrings = [
OperatorOptions.In.value,
] as (keyof SearchQueryFields)[]
export function recurseLogicalOperators(
filters: SearchFilters,
fn: (f: SearchFilters) => SearchFilters
) {
for (const logical of Object.values(LogicalOperator)) {
if (filters[logical]) {
filters[logical]!.conditions = filters[logical]!.conditions.map(
condition => fn(condition)
)
}
}
return filters
}
/**
* Removes any fields that contain empty strings that would cause inconsistent
* behaviour with how backend tables are filtered (no value means no filter).
@ -145,6 +159,7 @@ export const cleanupQuery = (query: SearchFilters) => {
}
}
}
query = recurseLogicalOperators(query, cleanupQuery)
return query
}
@ -410,6 +425,7 @@ export function fixupFilterArrays(filters: SearchFilters) {
}
}
}
recurseLogicalOperators(filters, fixupFilterArrays)
return filters
}