MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14795 from Budibase/chore/negated-join-filters 

Fix negated join filters on relationships
This commit is contained in:
Adria Navarro 2024-10-17 08:50:54 +02:00 committed by GitHub
parent a5e0a7155e 0f43c8f46c
commit 6ccf8ee603
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 81 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
packages/backend-core/src/sql/sql.ts
View File

@ -23,12 +23,14 @@ import {
InternalSearchFilterOperator, InternalSearchFilterOperator,
JsonFieldMetadata, JsonFieldMetadata,
JsonTypes, JsonTypes,
LogicalOperator,
Operation, Operation,
prefixed, prefixed,
QueryJson, QueryJson,
QueryOptions, QueryOptions,
RangeOperator, RangeOperator,
RelationshipsJson, RelationshipsJson,
SearchFilterKey,
SearchFilters, SearchFilters,
SortOrder, SortOrder,
SqlClient, SqlClient,
@ -96,6 +98,22 @@ function isSqs(table: Table): boolean {
) )
} }
const allowEmptyRelationships: Record<SearchFilterKey, boolean> = {
[BasicOperator.EQUAL]: false,
[BasicOperator.NOT_EQUAL]: true,
[BasicOperator.EMPTY]: false,
[BasicOperator.NOT_EMPTY]: true,
[BasicOperator.FUZZY]: false,
[BasicOperator.STRING]: false,
[RangeOperator.RANGE]: false,
[ArrayOperator.CONTAINS]: false,
[ArrayOperator.NOT_CONTAINS]: true,
[ArrayOperator.CONTAINS_ANY]: false,
[ArrayOperator.ONE_OF]: false,
[LogicalOperator.AND]: false,
[LogicalOperator.OR]: false,
}
class InternalBuilder { class InternalBuilder {
private readonly client: SqlClient private readonly client: SqlClient
private readonly query: QueryJson private readonly query: QueryJson
@ -405,6 +423,7 @@ class InternalBuilder {
addRelationshipForFilter( addRelationshipForFilter(
query: Knex.QueryBuilder, query: Knex.QueryBuilder,
allowEmptyRelationships: boolean,
filterKey: string, filterKey: string,
whereCb: (filterKey: string, query: Knex.QueryBuilder) => Knex.QueryBuilder whereCb: (filterKey: string, query: Knex.QueryBuilder) => Knex.QueryBuilder
): Knex.QueryBuilder { ): Knex.QueryBuilder {
@ -430,9 +449,10 @@ class InternalBuilder {
relationship.to && relationship.to &&
relationship.tableName relationship.tableName
) { ) {
let subQuery = mainKnex const joinTable = mainKnex
.select(mainKnex.raw(1)) .select(mainKnex.raw(1))
.from({ [toAlias]: relatedTableName }) .from({ [toAlias]: relatedTableName })
let subQuery = joinTable.clone()
const manyToMany = validateManyToMany(relationship) const manyToMany = validateManyToMany(relationship)
let updatedKey let updatedKey
@ -455,7 +475,6 @@ class InternalBuilder {
subQuery = subQuery subQuery = subQuery
// add a join through the junction table // add a join through the junction table
.innerJoin(throughTable, function () { .innerJoin(throughTable, function () {
// @ts-ignore
this.on( this.on(
`${toAlias}.${manyToMany.toPrimary}`, `${toAlias}.${manyToMany.toPrimary}`,
"=", "=",
@ -475,17 +494,38 @@ class InternalBuilder {
if (this.client === SqlClient.SQL_LITE) { if (this.client === SqlClient.SQL_LITE) {
subQuery = this.addJoinFieldCheck(subQuery, manyToMany) subQuery = this.addJoinFieldCheck(subQuery, manyToMany)
} }
query = query.where(q => {
q.whereExists(whereCb(updatedKey, subQuery))
if (allowEmptyRelationships) {
q.orWhereNotExists(
joinTable.clone().innerJoin(throughTable, function () {
this.on(
`${fromAlias}.${manyToMany.fromPrimary}`,
"=",
`${throughAlias}.${manyToMany.from}`
)
})
)
}
})
} else { } else {
const toKey = `${toAlias}.${relationship.to}`
const foreignKey = `${fromAlias}.${relationship.from}`
// "join" to the main table, making sure the ID matches that of the main // "join" to the main table, making sure the ID matches that of the main
subQuery = subQuery.where( subQuery = subQuery.where(
`${toAlias}.${relationship.to}`, toKey,
"=", "=",
mainKnex.raw( mainKnex.raw(this.quotedIdentifier(foreignKey))
this.quotedIdentifier(`${fromAlias}.${relationship.from}`)
)
) )
query = query.where(q => {
q.whereExists(whereCb(updatedKey, subQuery.clone()))
if (allowEmptyRelationships) {
q.orWhereNotExists(subQuery)
}
})
} }
query = query.whereExists(whereCb(updatedKey, subQuery))
} }
} }
return query return query
@ -516,6 +556,7 @@ class InternalBuilder {
} }
function iterate( function iterate(
structure: AnySearchFilter, structure: AnySearchFilter,
operation: SearchFilterKey,
fn: ( fn: (
query: Knex.QueryBuilder, query: Knex.QueryBuilder,
key: string, key: string,
@ -574,6 +615,7 @@ class InternalBuilder {
} }
query = builder.addRelationshipForFilter( query = builder.addRelationshipForFilter(
query, query,
allowEmptyRelationships[operation],
updatedKey, updatedKey,
(updatedKey, q) => { (updatedKey, q) => {
return handleRelationship(q, updatedKey, value) return handleRelationship(q, updatedKey, value)
@ -610,7 +652,7 @@ class InternalBuilder {
return `[${value.join(",")}]` return `[${value.join(",")}]`
} }
if (this.client === SqlClient.POSTGRES) { if (this.client === SqlClient.POSTGRES) {
iterate(mode, (q, key, value) => { iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => {
const wrap = any ? "" : "'" const wrap = any ? "" : "'"
const op = any ? "\\?| array" : "@>" const op = any ? "\\?| array" : "@>"
const fieldNames = key.split(/\./g) const fieldNames = key.split(/\./g)
@ -628,7 +670,7 @@ class InternalBuilder {
this.client === SqlClient.MARIADB this.client === SqlClient.MARIADB
) { ) {
const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS" const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS"
iterate(mode, (q, key, value) => { iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => {
return q[rawFnc]( return q[rawFnc](
`${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray( `${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray(
value value
@ -637,7 +679,7 @@ class InternalBuilder {
}) })
} else { } else {
const andOr = mode === filters?.containsAny ? " OR " : " AND " const andOr = mode === filters?.containsAny ? " OR " : " AND "
iterate(mode, (q, key, value) => { iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => {
let statement = "" let statement = ""
const identifier = this.quotedIdentifier(key) const identifier = this.quotedIdentifier(key)
for (let i in value) { for (let i in value) {
@ -691,6 +733,7 @@ class InternalBuilder {
const fnc = allOr ? "orWhereIn" : "whereIn" const fnc = allOr ? "orWhereIn" : "whereIn"
iterate( iterate(
filters.oneOf, filters.oneOf,
ArrayOperator.ONE_OF,
(q, key: string, array) => { (q, key: string, array) => {
if (this.client === SqlClient.ORACLE) { if (this.client === SqlClient.ORACLE) {
key = this.convertClobs(key) key = this.convertClobs(key)
@ -715,7 +758,7 @@ class InternalBuilder {
) )
} }
if (filters.string) { if (filters.string) {
iterate(filters.string, (q, key, value) => { iterate(filters.string, BasicOperator.STRING, (q, key, value) => {
const fnc = allOr ? "orWhere" : "where" const fnc = allOr ? "orWhere" : "where"
// postgres supports ilike, nothing else does // postgres supports ilike, nothing else does
if (this.client === SqlClient.POSTGRES) { if (this.client === SqlClient.POSTGRES) {
@ -730,10 +773,10 @@ class InternalBuilder {
}) })
} }
if (filters.fuzzy) { if (filters.fuzzy) {
iterate(filters.fuzzy, like) iterate(filters.fuzzy, BasicOperator.FUZZY, like)
} }
if (filters.range) { if (filters.range) {
iterate(filters.range, (q, key, value) => { iterate(filters.range, RangeOperator.RANGE, (q, key, value) => {
const isEmptyObject = (val: any) => { const isEmptyObject = (val: any) => {
return ( return (
val && val &&
@ -799,7 +842,7 @@ class InternalBuilder {
}) })
} }
if (filters.equal) { if (filters.equal) {
iterate(filters.equal, (q, key, value) => { iterate(filters.equal, BasicOperator.EQUAL, (q, key, value) => {
const fnc = allOr ? "orWhereRaw" : "whereRaw" const fnc = allOr ? "orWhereRaw" : "whereRaw"
if (this.client === SqlClient.MS_SQL) { if (this.client === SqlClient.MS_SQL) {
return q[fnc]( return q[fnc](
@ -819,7 +862,7 @@ class InternalBuilder {
}) })
} }
if (filters.notEqual) { if (filters.notEqual) {
iterate(filters.notEqual, (q, key, value) => { iterate(filters.notEqual, BasicOperator.NOT_EQUAL, (q, key, value) => {
const fnc = allOr ? "orWhereRaw" : "whereRaw" const fnc = allOr ? "orWhereRaw" : "whereRaw"
if (this.client === SqlClient.MS_SQL) { if (this.client === SqlClient.MS_SQL) {
return q[fnc]( return q[fnc](
@ -840,13 +883,13 @@ class InternalBuilder {
}) })
} }
if (filters.empty) { if (filters.empty) {
iterate(filters.empty, (q, key) => { iterate(filters.empty, BasicOperator.EMPTY, (q, key) => {
const fnc = allOr ? "orWhereNull" : "whereNull" const fnc = allOr ? "orWhereNull" : "whereNull"
return q[fnc](key) return q[fnc](key)
}) })
} }
if (filters.notEmpty) { if (filters.notEmpty) {
iterate(filters.notEmpty, (q, key) => { iterate(filters.notEmpty, BasicOperator.NOT_EMPTY, (q, key) => {
const fnc = allOr ? "orWhereNotNull" : "whereNotNull" const fnc = allOr ? "orWhereNotNull" : "whereNotNull"
return q[fnc](key) return q[fnc](key)
}) })
@ -1242,12 +1285,10 @@ class InternalBuilder {
}) })
: undefined : undefined
if (!throughTable) { if (!throughTable) {
// @ts-ignore
query = query.leftJoin(toTableWithSchema, function () { query = query.leftJoin(toTableWithSchema, function () {
for (let relationship of columns) { for (let relationship of columns) {
const from = relationship.from, const from = relationship.from,
to = relationship.to to = relationship.to
// @ts-ignore
this.orOn(`${fromAlias}.${from}`, "=", `${toAlias}.${to}`) this.orOn(`${fromAlias}.${from}`, "=", `${toAlias}.${to}`)
} }
}) })
@ -1258,7 +1299,6 @@ class InternalBuilder {
for (let relationship of columns) { for (let relationship of columns) {
const fromPrimary = relationship.fromPrimary const fromPrimary = relationship.fromPrimary
const from = relationship.from const from = relationship.from
// @ts-ignore
this.orOn( this.orOn(
`${fromAlias}.${fromPrimary}`, `${fromAlias}.${fromPrimary}`,
"=", "=",
@ -1270,7 +1310,6 @@ class InternalBuilder {
for (let relationship of columns) { for (let relationship of columns) {
const toPrimary = relationship.toPrimary const toPrimary = relationship.toPrimary
const to = relationship.to const to = relationship.to
// @ts-ignore
this.orOn(`${toAlias}.${toPrimary}`, `${throughAlias}.${to}`) this.orOn(`${toAlias}.${toPrimary}`, `${throughAlias}.${to}`)
} }
}) })

26
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -2384,7 +2384,10 @@ describe.each([
], ],
}, },
}).toContainExactly([ }).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, {
name: "foo",
productCat: [{ _id: productCatRows[0]._id }],
},
]) ])
} }
) )
@ -2462,7 +2465,7 @@ describe.each([
}).toContainExactly([ }).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
{ name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
// { name: "baz", productCat: undefined }, // TODO { name: "baz", productCat: undefined },
]) ])
}) })
@ -2484,7 +2487,10 @@ describe.each([
], ],
}, },
}).toContainExactly([ }).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, {
name: "foo",
productCat: [{ _id: productCatRows[0]._id }],
},
]) ])
} }
) )
@ -2508,9 +2514,15 @@ describe.each([
], ],
}, },
}).toContainExactly([ }).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, {
{ name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, name: "foo",
// { name: "baz", productCat: undefined }, // TODO productCat: [{ _id: productCatRows[0]._id }],
},
{
name: "bar",
productCat: [{ _id: productCatRows[1]._id }],
},
{ name: "baz", productCat: undefined },
]) ])
} }
) )
@ -2534,7 +2546,7 @@ describe.each([
}).toContainExactly([ }).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
{ name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
// { name: "baz", productCat: undefined }, // TODO { name: "baz", productCat: undefined },
]) ])
}) })
}) })

4
packages/server/src/integrations/tests/sqlAlias.spec.ts
View File

@ -78,7 +78,7 @@ describe("Captures of real examples", () => {
bindings: ["assembling", primaryLimit, relationshipLimit], bindings: ["assembling", primaryLimit, relationshipLimit],
sql: expect.stringContaining( sql: expect.stringContaining(
multiline( multiline(
`where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" and (COALESCE("b"."taskname" = $1, FALSE))` `where (exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" and (COALESCE("b"."taskname" = $1, FALSE)))`
) )
), ),
}) })
@ -145,7 +145,7 @@ describe("Captures of real examples", () => {
], ],
sql: expect.stringContaining( sql: expect.stringContaining(
multiline( multiline(
`where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2)) and exists (select 1 from "persons" as "c" where "c"."personid" = "a"."qaid" and ("c"."year" between $3 and $4)) and exists (select 1 from "products" as "b" inner join "products_tasks" as "d" on "b"."productid" = "d"."productid" where "d"."taskid" = "a"."taskid" and (COALESCE("b"."productname" = $5, FALSE))` `where (exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2))) and (exists (select 1 from "persons" as "c" where "c"."personid" = "a"."qaid" and ("c"."year" between $3 and $4))) and (exists (select 1 from "products" as "b" inner join "products_tasks" as "d" on "b"."productid" = "d"."productid" where "d"."taskid" = "a"."taskid" and (COALESCE("b"."productname" = $5, FALSE))))`
) )
), ),
}) })