From 76d0107d4da9ae180b3b9f030330cbe82a0d3a57 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 15 Oct 2024 10:10:15 +0200 Subject: [PATCH 01/22] Handle empty relationships --- packages/backend-core/src/sql/sql.ts | 81 +++++++++++++++++++++------- 1 file changed, 61 insertions(+), 20 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b415a6f1b7..b35b2b5ec8 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -23,12 +23,14 @@ import { InternalSearchFilterOperator, JsonFieldMetadata, JsonTypes, + LogicalOperator, Operation, prefixed, QueryJson, QueryOptions, RangeOperator, RelationshipsJson, + SearchFilterKey, SearchFilters, SortOrder, SqlClient, @@ -96,6 +98,22 @@ function isSqs(table: Table): boolean { ) } +const allowEmptyRelationships: Record = { + [BasicOperator.EQUAL]: false, + [BasicOperator.NOT_EQUAL]: true, + [BasicOperator.EMPTY]: false, + [BasicOperator.NOT_EMPTY]: true, + [BasicOperator.FUZZY]: false, + [BasicOperator.STRING]: false, + [RangeOperator.RANGE]: false, + [ArrayOperator.CONTAINS]: false, + [ArrayOperator.NOT_CONTAINS]: true, + [ArrayOperator.CONTAINS_ANY]: false, + [ArrayOperator.ONE_OF]: false, + [LogicalOperator.AND]: false, + [LogicalOperator.OR]: false, +} + class InternalBuilder { private readonly client: SqlClient private readonly query: QueryJson @@ -405,6 +423,7 @@ class InternalBuilder { addRelationshipForFilter( query: Knex.QueryBuilder, + allowEmptyRelationships: boolean, filterKey: string, whereCb: (query: Knex.QueryBuilder) => Knex.QueryBuilder ): Knex.QueryBuilder { @@ -426,9 +445,10 @@ class InternalBuilder { relationship.to && relationship.tableName ) { - let subQuery = mainKnex + const joinTable = mainKnex .select(mainKnex.raw(1)) .from({ [toAlias]: relatedTableName }) + let subQuery = joinTable.clone() const manyToMany = validateManyToMany(relationship) if (manyToMany) { const throughAlias = @@ -440,7 +460,6 @@ class InternalBuilder { subQuery = subQuery // add a join through the junction table .innerJoin(throughTable, function () { - // @ts-ignore this.on( `${toAlias}.${manyToMany.toPrimary}`, "=", @@ -460,18 +479,33 @@ class InternalBuilder { if (this.client === SqlClient.SQL_LITE) { subQuery = this.addJoinFieldCheck(subQuery, manyToMany) } + + query = query.whereExists(whereCb(subQuery)) + if (allowEmptyRelationships) { + query = query.orWhereNotExists( + joinTable.clone().innerJoin(throughTable, function () { + this.on( + `${fromAlias}.${manyToMany.fromPrimary}`, + "=", + `${throughAlias}.${manyToMany.from}` + ) + }) + ) + } } else { + const foreignKey = `${fromAlias}.${relationship.from}` // "join" to the main table, making sure the ID matches that of the main subQuery = subQuery.where( `${toAlias}.${relationship.to}`, "=", - mainKnex.raw( - this.quotedIdentifier(`${fromAlias}.${relationship.from}`) - ) + mainKnex.raw(this.quotedIdentifier(foreignKey)) ) + + query = query.whereExists(whereCb(subQuery)) + if (allowEmptyRelationships) { + query = query.orWhereNull(foreignKey) + } } - query = query.whereExists(whereCb(subQuery)) - break } } return query @@ -502,6 +536,7 @@ class InternalBuilder { } function iterate( structure: AnySearchFilter, + operation: SearchFilterKey, fn: ( query: Knex.QueryBuilder, key: string, @@ -558,9 +593,14 @@ class InternalBuilder { if (allOr) { query = query.or } - query = builder.addRelationshipForFilter(query, updatedKey, q => { - return handleRelationship(q, updatedKey, value) - }) + query = builder.addRelationshipForFilter( + query, + allowEmptyRelationships[operation], + updatedKey, + q => { + return handleRelationship(q, updatedKey, value) + } + ) } } } @@ -592,7 +632,7 @@ class InternalBuilder { return `[${value.join(",")}]` } if (this.client === SqlClient.POSTGRES) { - iterate(mode, (q, key, value) => { + iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => { const wrap = any ? "" : "'" const op = any ? "\\?| array" : "@>" const fieldNames = key.split(/\./g) @@ -610,7 +650,7 @@ class InternalBuilder { this.client === SqlClient.MARIADB ) { const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS" - iterate(mode, (q, key, value) => { + iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => { return q[rawFnc]( `${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray( value @@ -619,7 +659,7 @@ class InternalBuilder { }) } else { const andOr = mode === filters?.containsAny ? " OR " : " AND " - iterate(mode, (q, key, value) => { + iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => { let statement = "" const identifier = this.quotedIdentifier(key) for (let i in value) { @@ -673,6 +713,7 @@ class InternalBuilder { const fnc = allOr ? "orWhereIn" : "whereIn" iterate( filters.oneOf, + ArrayOperator.ONE_OF, (q, key: string, array) => { if (this.client === SqlClient.ORACLE) { key = this.convertClobs(key) @@ -697,7 +738,7 @@ class InternalBuilder { ) } if (filters.string) { - iterate(filters.string, (q, key, value) => { + iterate(filters.string, BasicOperator.STRING, (q, key, value) => { const fnc = allOr ? "orWhere" : "where" // postgres supports ilike, nothing else does if (this.client === SqlClient.POSTGRES) { @@ -712,10 +753,10 @@ class InternalBuilder { }) } if (filters.fuzzy) { - iterate(filters.fuzzy, like) + iterate(filters.fuzzy, BasicOperator.FUZZY, like) } if (filters.range) { - iterate(filters.range, (q, key, value) => { + iterate(filters.range, RangeOperator.RANGE, (q, key, value) => { const isEmptyObject = (val: any) => { return ( val && @@ -781,7 +822,7 @@ class InternalBuilder { }) } if (filters.equal) { - iterate(filters.equal, (q, key, value) => { + iterate(filters.equal, BasicOperator.EQUAL, (q, key, value) => { const fnc = allOr ? "orWhereRaw" : "whereRaw" if (this.client === SqlClient.MS_SQL) { return q[fnc]( @@ -801,7 +842,7 @@ class InternalBuilder { }) } if (filters.notEqual) { - iterate(filters.notEqual, (q, key, value) => { + iterate(filters.notEqual, BasicOperator.NOT_EQUAL, (q, key, value) => { const fnc = allOr ? "orWhereRaw" : "whereRaw" if (this.client === SqlClient.MS_SQL) { return q[fnc]( @@ -822,13 +863,13 @@ class InternalBuilder { }) } if (filters.empty) { - iterate(filters.empty, (q, key) => { + iterate(filters.empty, BasicOperator.EMPTY, (q, key) => { const fnc = allOr ? "orWhereNull" : "whereNull" return q[fnc](key) }) } if (filters.notEmpty) { - iterate(filters.notEmpty, (q, key) => { + iterate(filters.notEmpty, BasicOperator.NOT_EMPTY, (q, key) => { const fnc = allOr ? "orWhereNotNull" : "whereNotNull" return q[fnc](key) }) From 57da952f6944b5874fe20e4539ce33b5ac2c3055 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 15 Oct 2024 11:34:35 +0200 Subject: [PATCH 02/22] Fix "parenthesis" --- packages/backend-core/src/sql/sql.ts | 30 +++++++++++++--------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b35b2b5ec8..3e6032fc99 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -480,18 +480,20 @@ class InternalBuilder { subQuery = this.addJoinFieldCheck(subQuery, manyToMany) } - query = query.whereExists(whereCb(subQuery)) - if (allowEmptyRelationships) { - query = query.orWhereNotExists( - joinTable.clone().innerJoin(throughTable, function () { - this.on( - `${fromAlias}.${manyToMany.fromPrimary}`, - "=", - `${throughAlias}.${manyToMany.from}` - ) - }) - ) - } + query = query.where(q => { + q.whereExists(whereCb(subQuery)) + if (allowEmptyRelationships) { + q.orWhereNotExists( + joinTable.clone().innerJoin(throughTable, function () { + this.on( + `${fromAlias}.${manyToMany.fromPrimary}`, + "=", + `${throughAlias}.${manyToMany.from}` + ) + }) + ) + } + }) } else { const foreignKey = `${fromAlias}.${relationship.from}` // "join" to the main table, making sure the ID matches that of the main @@ -1265,12 +1267,10 @@ class InternalBuilder { }) : undefined if (!throughTable) { - // @ts-ignore query = query.leftJoin(toTableWithSchema, function () { for (let relationship of columns) { const from = relationship.from, to = relationship.to - // @ts-ignore this.orOn(`${fromAlias}.${from}`, "=", `${toAlias}.${to}`) } }) @@ -1281,7 +1281,6 @@ class InternalBuilder { for (let relationship of columns) { const fromPrimary = relationship.fromPrimary const from = relationship.from - // @ts-ignore this.orOn( `${fromAlias}.${fromPrimary}`, "=", @@ -1293,7 +1292,6 @@ class InternalBuilder { for (let relationship of columns) { const toPrimary = relationship.toPrimary const to = relationship.to - // @ts-ignore this.orOn(`${toAlias}.${toPrimary}`, `${throughAlias}.${to}`) } }) From 5782b20509e3f3ea209e692d15ce2446940d7bef Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 15 Oct 2024 11:36:29 +0200 Subject: [PATCH 03/22] Update test --- packages/server/src/api/routes/tests/search.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 3ab35c9294..23fb5a056f 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2458,7 +2458,7 @@ describe.each([ }).toContainExactly([ { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, - // { name: "baz", productCat: undefined }, // TODO + { name: "baz", productCat: undefined }, ]) }) @@ -2506,7 +2506,7 @@ describe.each([ }).toContainExactly([ { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, - // { name: "baz", productCat: undefined }, // TODO + { name: "baz", productCat: undefined }, ]) } ) @@ -2530,7 +2530,7 @@ describe.each([ }).toContainExactly([ { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, - // { name: "baz", productCat: undefined }, // TODO + { name: "baz", productCat: undefined }, ]) }) }) From 975945a23a0b9d30a2b448fc9af73ec75f1f56ae Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 15 Oct 2024 12:03:23 +0200 Subject: [PATCH 04/22] Improve tests --- .../src/api/routes/tests/search.spec.ts | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 23fb5a056f..798157b122 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2278,12 +2278,16 @@ describe.each([ // It also can't work for in-memory searching because the related table name // isn't available. !isInMemory && - describe("relations", () => { + describe.each([ + RelationshipType.ONE_TO_MANY, + RelationshipType.MANY_TO_ONE, + RelationshipType.MANY_TO_MANY, + ])("relations", relationshipType => { let productCategoryTable: Table, productCatRows: Row[] beforeAll(async () => { const { relatedTable, tableId } = await basicRelationshipTables( - RelationshipType.ONE_TO_MANY + relationshipType ) tableOrViewId = tableId productCategoryTable = relatedTable @@ -2380,7 +2384,10 @@ describe.each([ ], }, }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + { + name: "foo", + productCat: [{ _id: productCatRows[0]._id }], + }, ]) } ) @@ -2480,7 +2487,10 @@ describe.each([ ], }, }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + { + name: "foo", + productCat: [{ _id: productCatRows[0]._id }], + }, ]) } ) @@ -2504,8 +2514,14 @@ describe.each([ ], }, }).toContainExactly([ - { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, - { name: "bar", productCat: [{ _id: productCatRows[1]._id }] }, + { + name: "foo", + productCat: [{ _id: productCatRows[0]._id }], + }, + { + name: "bar", + productCat: [{ _id: productCatRows[1]._id }], + }, { name: "baz", productCat: undefined }, ]) } From 41405ea39db26b03fbde5664add1cb116cc74467 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 15 Oct 2024 14:15:45 +0200 Subject: [PATCH 05/22] Add extra rows --- .../src/api/routes/tests/search.spec.ts | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 3ab35c9294..ece84435bf 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2609,6 +2609,21 @@ describe.each([ related1: [relatedRows[2]._id!], related2: [relatedRows[3]._id!], }), + config.api.row.save(tableOrViewId, { + name: "test3", + related1: [], + related2: [], + }), + config.api.row.save(tableOrViewId, { + name: "test4", + related1: [relatedRows[1]._id!], + related2: [], + }), + config.api.row.save(tableOrViewId, { + name: "test5", + related1: [], + related2: [relatedRows[1]._id!], + }), ]) }) @@ -2626,6 +2641,17 @@ describe.each([ related1: [{ _id: relatedRows[2]._id }], related2: [{ _id: relatedRows[3]._id }], }, + { + name: "test3", + }, + { + name: "test4", + related1: [{ _id: relatedRows[1]._id }], + }, + { + name: "test5", + related2: [{ _id: relatedRows[1]._id! }], + }, ]) }) From 6688ccf1ab29ef958075d17ccc7df1b46d467a20 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 15 Oct 2024 15:50:52 +0200 Subject: [PATCH 06/22] Ensure we replace only on when starting with --- packages/server/src/sdk/app/rows/search/filters.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/filters.ts b/packages/server/src/sdk/app/rows/search/filters.ts index 4049fc5352..64656b1a37 100644 --- a/packages/server/src/sdk/app/rows/search/filters.ts +++ b/packages/server/src/sdk/app/rows/search/filters.ts @@ -53,8 +53,12 @@ export function updateFilterKeys( ) if (possibleKey && possibleKey.original !== possibleKey.updated) { // only replace the first, not replaceAll - filter[key.replace(possibleKey.original, possibleKey.updated)] = - filter[key] + filter[ + key.replace( + new RegExp(`^${possibleKey.original}\\.`), + `${possibleKey.updated}.` + ) + ] = filter[key] delete filter[key] } } From 225d062fccce09ba68e9c431dbfaa04fd6ff6b8b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 15 Oct 2024 16:30:10 +0200 Subject: [PATCH 07/22] Fix --- packages/server/src/sdk/app/rows/search/filters.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/filters.ts b/packages/server/src/sdk/app/rows/search/filters.ts index 64656b1a37..9fab045554 100644 --- a/packages/server/src/sdk/app/rows/search/filters.ts +++ b/packages/server/src/sdk/app/rows/search/filters.ts @@ -44,10 +44,11 @@ export function updateFilterKeys( if (!isPlainObject(filter)) { continue } - for (let [key, keyFilter] of Object.entries(filter)) { + for (const [key, keyFilter] of Object.entries(filter)) { if (keyFilter === "") { delete filter[key] } + const possibleKey = updates.find(({ original }) => key.match(makeFilterKeyRegex(original)) ) @@ -55,8 +56,8 @@ export function updateFilterKeys( // only replace the first, not replaceAll filter[ key.replace( - new RegExp(`^${possibleKey.original}\\.`), - `${possibleKey.updated}.` + new RegExp(`^(\\d+:)?${possibleKey.original}\\.`), + `$1${possibleKey.updated}.` ) ] = filter[key] delete filter[key] From 86e5718705df0c3fb8c3eaa3aa5577435280eb6f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 15 Oct 2024 17:40:03 +0200 Subject: [PATCH 08/22] Fix tests --- packages/server/src/integrations/tests/sqlAlias.spec.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 890c8c4663..ddc84ae1ce 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -78,8 +78,7 @@ describe("Captures of real examples", () => { bindings: ["assembling", primaryLimit, relationshipLimit], sql: expect.stringContaining( multiline( - `where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" - and (COALESCE("b"."taskname" = $1, FALSE))` + `where (exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" and (COALESCE("b"."taskname" = $1, FALSE)))` ) ), }) @@ -133,6 +132,8 @@ describe("Captures of real examples", () => { expect(query).toEqual({ bindings: [ + rangeValue.low, + rangeValue.high, rangeValue.low, rangeValue.high, equalValue, @@ -144,7 +145,7 @@ describe("Captures of real examples", () => { ], sql: expect.stringContaining( multiline( - `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2))` + `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2)) and exists (select 1 from "persons" as "c" where "c"."personid" = "a"."qaid" and ("c"."year" between $3 and $4)) and (exists (select 1 from "products" as "b" inner join "products_tasks" as "d" on "b"."productid" = "d"."productid" where "d"."taskid" = "a"."taskid" and (COALESCE("b"."productname" = $5, FALSE)))` ) ), }) From 8a6dbef24938d99f448e1fb3aa9e0be6224f036c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 15 Oct 2024 18:50:58 +0200 Subject: [PATCH 09/22] Fix sqs --- packages/backend-core/src/sql/sql.ts | 34 ++++++++++++++----- .../src/sdk/app/rows/search/internal/sqs.ts | 33 ++---------------- 2 files changed, 28 insertions(+), 39 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b415a6f1b7..83a4fd0d59 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -406,23 +406,26 @@ class InternalBuilder { addRelationshipForFilter( query: Knex.QueryBuilder, filterKey: string, - whereCb: (query: Knex.QueryBuilder) => Knex.QueryBuilder + whereCb: (filterKey: string, query: Knex.QueryBuilder) => Knex.QueryBuilder ): Knex.QueryBuilder { const mainKnex = this.knex const { relationships, endpoint, tableAliases: aliases } = this.query const tableName = endpoint.entityId const fromAlias = aliases?.[tableName] || tableName - const matches = (possibleTable: string) => - filterKey.startsWith(`${possibleTable}`) + const matches = (value: string) => filterKey.startsWith(`${value}`) if (!relationships) { return query } for (const relationship of relationships) { const relatedTableName = relationship.tableName const toAlias = aliases?.[relatedTableName] || relatedTableName + + const matchesTableName = matches(relatedTableName) || matches(toAlias) + const matchesRelationName = matches(relationship.column) + // this is the relationship which is being filtered if ( - (matches(relatedTableName) || matches(toAlias)) && + (matchesTableName || matchesRelationName) && relationship.to && relationship.tableName ) { @@ -430,6 +433,17 @@ class InternalBuilder { .select(mainKnex.raw(1)) .from({ [toAlias]: relatedTableName }) const manyToMany = validateManyToMany(relationship) + let updatedKey + + if (matchesRelationName) { + updatedKey = filterKey.replace( + new RegExp(`^${relationship.column}.`), + `${aliases![relationship.tableName]}.` + ) + } else { + updatedKey = filterKey + } + if (manyToMany) { const throughAlias = aliases?.[manyToMany.through] || relationship.through @@ -470,7 +484,7 @@ class InternalBuilder { ) ) } - query = query.whereExists(whereCb(subQuery)) + query = query.whereExists(whereCb(updatedKey, subQuery)) break } } @@ -558,9 +572,13 @@ class InternalBuilder { if (allOr) { query = query.or } - query = builder.addRelationshipForFilter(query, updatedKey, q => { - return handleRelationship(q, updatedKey, value) - }) + query = builder.addRelationshipForFilter( + query, + updatedKey, + (updatedKey, q) => { + return handleRelationship(q, updatedKey, value) + } + ) } } } diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index 916e20957b..e3bbca271e 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -39,11 +39,6 @@ import AliasTables from "../../sqlAlias" import { outputProcessing } from "../../../../../utilities/rowProcessor" import pick from "lodash/pick" import { processRowCountResponse } from "../../utils" -import { - getRelationshipColumns, - getTableIDList, - updateFilterKeys, -} from "../filters" import { dataFilters, helpers, @@ -133,31 +128,7 @@ async function buildInternalFieldList( return [...new Set(fieldList)] } -function cleanupFilters( - filters: SearchFilters, - table: Table, - allTables: Table[] -) { - // get a list of all relationship columns in the table for updating - const relationshipColumns = getRelationshipColumns(table) - // get table names to ID map for relationships - const tableNameToID = getTableIDList(allTables) - // all should be applied at once - filters = updateFilterKeys( - filters, - relationshipColumns - .map(({ name, definition }) => ({ - original: name, - updated: definition.tableId, - })) - .concat( - tableNameToID.map(({ name, id }) => ({ - original: name, - updated: id, - })) - ) - ) - +function cleanupFilters(filters: SearchFilters, allTables: Table[]) { // generate a map of all possible column names (these can be duplicated across tables // the map of them will always be the same const userColumnMap: Record = {} @@ -356,7 +327,7 @@ export async function search( const relationships = buildInternalRelationships(table, allTables) const searchFilters: SearchFilters = { - ...cleanupFilters(query, table, allTables), + ...cleanupFilters(query, allTables), documentType: DocumentType.ROW, } From b01564c934922c0acabfd5bea53861f511163fb5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 16 Oct 2024 10:21:17 +0200 Subject: [PATCH 10/22] Fix multiple relations to same table for external --- packages/backend-core/src/sql/sql.ts | 6 +- .../api/controllers/row/ExternalRequest.ts | 12 ---- packages/server/src/sdk/app/rows/index.ts | 2 - .../server/src/sdk/app/rows/search/filters.ts | 70 ------------------- 4 files changed, 3 insertions(+), 87 deletions(-) delete mode 100644 packages/server/src/sdk/app/rows/search/filters.ts diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 83a4fd0d59..c47eabca57 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -412,7 +412,8 @@ class InternalBuilder { const { relationships, endpoint, tableAliases: aliases } = this.query const tableName = endpoint.entityId const fromAlias = aliases?.[tableName] || tableName - const matches = (value: string) => filterKey.startsWith(`${value}`) + const matches = (value: string) => + filterKey.match(new RegExp(`^${value}\\.`)) if (!relationships) { return query } @@ -435,7 +436,7 @@ class InternalBuilder { const manyToMany = validateManyToMany(relationship) let updatedKey - if (matchesRelationName) { + if (!matchesTableName) { updatedKey = filterKey.replace( new RegExp(`^${relationship.column}.`), `${aliases![relationship.tableName]}.` @@ -485,7 +486,6 @@ class InternalBuilder { ) } query = query.whereExists(whereCb(updatedKey, subQuery)) - break } } return query diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 56522acb33..7ac13d3200 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -204,18 +204,6 @@ export class ExternalRequest { filters: SearchFilters, table: Table ): SearchFilters { - // replace any relationship columns initially, table names and relationship column names are acceptable - const relationshipColumns = sdk.rows.filters.getRelationshipColumns(table) - filters = sdk.rows.filters.updateFilterKeys( - filters, - relationshipColumns.map(({ name, definition }) => { - const { tableName } = breakExternalTableId(definition.tableId) - return { - original: name, - updated: tableName, - } - }) - ) const primary = table.primary // if passed in array need to copy for shifting etc let idCopy: undefined | string | any[] = cloneDeep(id) diff --git a/packages/server/src/sdk/app/rows/index.ts b/packages/server/src/sdk/app/rows/index.ts index fb077509a9..c117941419 100644 --- a/packages/server/src/sdk/app/rows/index.ts +++ b/packages/server/src/sdk/app/rows/index.ts @@ -3,14 +3,12 @@ import * as rows from "./rows" import * as search from "./search" import * as utils from "./utils" import * as external from "./external" -import * as filters from "./search/filters" import AliasTables from "./sqlAlias" export default { ...attachments, ...rows, ...search, - filters, utils, external, AliasTables, diff --git a/packages/server/src/sdk/app/rows/search/filters.ts b/packages/server/src/sdk/app/rows/search/filters.ts deleted file mode 100644 index 9fab045554..0000000000 --- a/packages/server/src/sdk/app/rows/search/filters.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { - FieldType, - RelationshipFieldMetadata, - SearchFilters, - Table, -} from "@budibase/types" -import { isPlainObject } from "lodash" -import { dataFilters } from "@budibase/shared-core" - -export function getRelationshipColumns(table: Table): { - name: string - definition: RelationshipFieldMetadata -}[] { - // performing this with a for loop rather than an array filter improves - // type guarding, as no casts are required - const linkEntries: [string, RelationshipFieldMetadata][] = [] - for (let entry of Object.entries(table.schema)) { - if (entry[1].type === FieldType.LINK) { - const linkColumn: RelationshipFieldMetadata = entry[1] - linkEntries.push([entry[0], linkColumn]) - } - } - return linkEntries.map(entry => ({ - name: entry[0], - definition: entry[1], - })) -} - -export function getTableIDList( - tables: Table[] -): { name: string; id: string }[] { - return tables - .filter(table => table.originalName && table._id) - .map(table => ({ id: table._id!, name: table.originalName! })) -} - -export function updateFilterKeys( - filters: SearchFilters, - updates: { original: string; updated: string }[] -): SearchFilters { - const makeFilterKeyRegex = (str: string) => - new RegExp(`^${str}\\.|:${str}\\.`) - for (let filter of Object.values(filters)) { - if (!isPlainObject(filter)) { - continue - } - for (const [key, keyFilter] of Object.entries(filter)) { - if (keyFilter === "") { - delete filter[key] - } - - const possibleKey = updates.find(({ original }) => - key.match(makeFilterKeyRegex(original)) - ) - if (possibleKey && possibleKey.original !== possibleKey.updated) { - // only replace the first, not replaceAll - filter[ - key.replace( - new RegExp(`^(\\d+:)?${possibleKey.original}\\.`), - `$1${possibleKey.updated}.` - ) - ] = filter[key] - delete filter[key] - } - } - } - return dataFilters.recurseLogicalOperators(filters, (f: SearchFilters) => { - return updateFilterKeys(f, updates) - }) -} From 2d460f19557c866b1087634d850229e91835747b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 16 Oct 2024 12:26:13 +0200 Subject: [PATCH 11/22] Fix supporting filter using table names --- .../server/src/api/controllers/row/index.ts | 54 ++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 680a7671d5..9122dc5ff3 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -15,13 +15,16 @@ import { ExportRowsResponse, FieldType, GetRowResponse, + isRelationshipField, PatchRowRequest, PatchRowResponse, Row, RowAttachment, RowSearchParams, + SearchFilters, SearchRowRequest, SearchRowResponse, + Table, UserCtx, ValidateResponse, } from "@budibase/types" @@ -33,6 +36,8 @@ import sdk from "../../../sdk" import * as exporters from "../view/exporters" import { Format } from "../view/exporters" import { apiFileReturn } from "../../../utilities/fileSystem" +import { dataFilters } from "@budibase/shared-core" +import { isPlainObject } from "lodash" export * as views from "./views" @@ -211,13 +216,16 @@ export async function search(ctx: Ctx) { await context.ensureSnippetContext(true) - const enrichedQuery = await utils.enrichSearchContext( + let enrichedQuery: SearchFilters = await utils.enrichSearchContext( { ...ctx.request.body.query }, { user: sdk.users.getUserContextBindings(ctx.user), } ) + const allTables = await sdk.tables.getAllTables() + enrichedQuery = replaceTableNamesInFilters(tableId, enrichedQuery, allTables) + const searchParams: RowSearchParams = { ...ctx.request.body, query: enrichedQuery, @@ -229,6 +237,50 @@ export async function search(ctx: Ctx) { ctx.body = await sdk.rows.search(searchParams) } +function replaceTableNamesInFilters( + tableId: string, + filters: SearchFilters, + allTables: Table[] +): SearchFilters { + for (const filter of Object.values(filters)) { + if (!isPlainObject(filter)) { + continue + } + for (const key of Object.keys(filter)) { + const matches = key.match(`^(?.+)\\.(?.+)`) + + const relation = matches?.groups?.["relation"] + const field = matches?.groups?.["field"] + + if (!relation || !field) { + continue + } + + const table = allTables.find(r => r._id === tableId)! + if (Object.values(table.schema).some(f => f.name === relation)) { + continue + } + + const matchedTable = allTables.find(t => t.name === relation) + const relationship = Object.values(table.schema).find( + f => isRelationshipField(f) && f.tableId === matchedTable?._id + ) + if (!relationship) { + continue + } + + const updatedField = `${relationship.name}.${field}` + if (updatedField && updatedField !== key) { + filter[updatedField] = filter[key] + delete filter[key] + } + } + } + return dataFilters.recurseLogicalOperators(filters, (f: SearchFilters) => { + return replaceTableNamesInFilters(tableId, f, allTables) + }) +} + export async function validate(ctx: Ctx) { const source = await utils.getSource(ctx) const table = await utils.getTableFromSource(source) From 83ffee31947284cf2b0bf7e7df2500ef26ce506c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 16 Oct 2024 12:33:19 +0200 Subject: [PATCH 12/22] Fix tests --- .../src/api/routes/tests/search.spec.ts | 25 +++---------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index ece84435bf..5d778ed6d2 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2614,16 +2614,6 @@ describe.each([ related1: [], related2: [], }), - config.api.row.save(tableOrViewId, { - name: "test4", - related1: [relatedRows[1]._id!], - related2: [], - }), - config.api.row.save(tableOrViewId, { - name: "test5", - related1: [], - related2: [relatedRows[1]._id!], - }), ]) }) @@ -2644,19 +2634,11 @@ describe.each([ { name: "test3", }, - { - name: "test4", - related1: [{ _id: relatedRows[1]._id }], - }, - { - name: "test5", - related2: [{ _id: relatedRows[1]._id! }], - }, ]) }) isSqs && - it("should be able to filter down to second row with equal", async () => { + it("should be able to filter via the first relation field with equal", async () => { await expectSearch({ query: { equal: { @@ -2672,11 +2654,11 @@ describe.each([ }) isSqs && - it("should be able to filter down to first row with not equal", async () => { + it("should be able to filter via the second relation field with not equal", async () => { await expectSearch({ query: { notEqual: { - ["1:related2.name"]: "bar", + ["1:related2.name"]: "foo", ["2:related2.name"]: "baz", ["3:related2.name"]: "boo", }, @@ -2684,7 +2666,6 @@ describe.each([ }).toContainExactly([ { name: "test", - related1: [{ _id: relatedRows[0]._id }], }, ]) }) From 7266fb3c1bd0cf7b9d3f344002e46602bd324677 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 16 Oct 2024 12:38:38 +0200 Subject: [PATCH 13/22] Run tests for all --- .../server/src/api/controllers/row/index.ts | 16 ++--- .../src/api/routes/tests/search.spec.ts | 59 +++++++++---------- 2 files changed, 37 insertions(+), 38 deletions(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 9122dc5ff3..51bdb12ca2 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -216,15 +216,15 @@ export async function search(ctx: Ctx) { await context.ensureSnippetContext(true) - let enrichedQuery: SearchFilters = await utils.enrichSearchContext( - { ...ctx.request.body.query }, - { - user: sdk.users.getUserContextBindings(ctx.user), - } - ) + let { query } = ctx.request.body + if (!isPlainObject(query)) { + const allTables = await sdk.tables.getAllTables() + query = replaceTableNamesInFilters(tableId, query, allTables) + } - const allTables = await sdk.tables.getAllTables() - enrichedQuery = replaceTableNamesInFilters(tableId, enrichedQuery, allTables) + let enrichedQuery: SearchFilters = await utils.enrichSearchContext(query, { + user: sdk.users.getUserContextBindings(ctx.user), + }) const searchParams: RowSearchParams = { ...ctx.request.body, diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5d778ed6d2..7132115d94 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2567,7 +2567,8 @@ describe.each([ expect(response.rows[0].productCat).toBeArrayOfSize(11) }) }) - ;(isSqs || isLucene) && + + isSql && describe("relations to same table", () => { let relatedTable: string, relatedRows: Row[] @@ -2637,38 +2638,36 @@ describe.each([ ]) }) - isSqs && - it("should be able to filter via the first relation field with equal", async () => { - await expectSearch({ - query: { - equal: { - ["related1.name"]: "baz", - }, + it("should be able to filter via the first relation field with equal", async () => { + await expectSearch({ + query: { + equal: { + ["related1.name"]: "baz", }, - }).toContainExactly([ - { - name: "test2", - related1: [{ _id: relatedRows[2]._id }], - }, - ]) - }) + }, + }).toContainExactly([ + { + name: "test2", + related1: [{ _id: relatedRows[2]._id }], + }, + ]) + }) - isSqs && - it("should be able to filter via the second relation field with not equal", async () => { - await expectSearch({ - query: { - notEqual: { - ["1:related2.name"]: "foo", - ["2:related2.name"]: "baz", - ["3:related2.name"]: "boo", - }, + it("should be able to filter via the second relation field with not equal", async () => { + await expectSearch({ + query: { + notEqual: { + ["1:related2.name"]: "foo", + ["2:related2.name"]: "baz", + ["3:related2.name"]: "boo", }, - }).toContainExactly([ - { - name: "test", - }, - ]) - }) + }, + }).toContainExactly([ + { + name: "test", + }, + ]) + }) }) isInternal && From 88d4ebc725f3a87cb3ad4aa2b34655cab82b1879 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 16 Oct 2024 12:44:25 +0200 Subject: [PATCH 14/22] Add more tests --- .../src/api/routes/tests/search.spec.ts | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 7132115d94..da0fd4fbba 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2612,8 +2612,8 @@ describe.each([ }), config.api.row.save(tableOrViewId, { name: "test3", - related1: [], - related2: [], + related1: [relatedRows[1]._id], + related2: [relatedRows[2]._id!], }), ]) }) @@ -2634,6 +2634,8 @@ describe.each([ }, { name: "test3", + related1: [{ _id: relatedRows[1]._id }], + related2: [{ _id: relatedRows[2]._id }], }, ]) }) @@ -2668,6 +2670,21 @@ describe.each([ }, ]) }) + + it("should be able to filter on both fields", async () => { + await expectSearch({ + query: { + notEqual: { + ["related1.name"]: "foo", + ["related2.name"]: "baz", + }, + }, + }).toContainExactly([ + { + name: "test2", + }, + ]) + }) }) isInternal && From 830ed0cda25c914cca5364a1c5de88b5c7258be6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 16 Oct 2024 12:48:07 +0200 Subject: [PATCH 15/22] Fix undefineds --- packages/server/src/api/controllers/row/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 51bdb12ca2..40b41f410f 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -217,7 +217,7 @@ export async function search(ctx: Ctx) { await context.ensureSnippetContext(true) let { query } = ctx.request.body - if (!isPlainObject(query)) { + if (query && !isPlainObject(query)) { const allTables = await sdk.tables.getAllTables() query = replaceTableNamesInFilters(tableId, query, allTables) } From ba80880ab65d76d938e9746f896513528c4119a4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 16 Oct 2024 13:02:53 +0200 Subject: [PATCH 16/22] Fix --- packages/server/src/api/controllers/row/index.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 40b41f410f..60775ce628 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -37,7 +37,6 @@ import * as exporters from "../view/exporters" import { Format } from "../view/exporters" import { apiFileReturn } from "../../../utilities/fileSystem" import { dataFilters } from "@budibase/shared-core" -import { isPlainObject } from "lodash" export * as views from "./views" @@ -217,7 +216,7 @@ export async function search(ctx: Ctx) { await context.ensureSnippetContext(true) let { query } = ctx.request.body - if (query && !isPlainObject(query)) { + if (query) { const allTables = await sdk.tables.getAllTables() query = replaceTableNamesInFilters(tableId, query, allTables) } @@ -243,9 +242,6 @@ function replaceTableNamesInFilters( allTables: Table[] ): SearchFilters { for (const filter of Object.values(filters)) { - if (!isPlainObject(filter)) { - continue - } for (const key of Object.keys(filter)) { const matches = key.match(`^(?.+)\\.(?.+)`) From 57ce8b7e855126c3f4bd905e7121137dd22d25ab Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 16 Oct 2024 13:17:07 +0200 Subject: [PATCH 17/22] Fix test --- packages/server/src/integrations/tests/sqlAlias.spec.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 890c8c4663..81013b328c 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -78,8 +78,7 @@ describe("Captures of real examples", () => { bindings: ["assembling", primaryLimit, relationshipLimit], sql: expect.stringContaining( multiline( - `where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" - and (COALESCE("b"."taskname" = $1, FALSE))` + `where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" and (COALESCE("b"."taskname" = $1, FALSE))` ) ), }) @@ -133,6 +132,8 @@ describe("Captures of real examples", () => { expect(query).toEqual({ bindings: [ + rangeValue.low, + rangeValue.high, rangeValue.low, rangeValue.high, equalValue, @@ -144,7 +145,7 @@ describe("Captures of real examples", () => { ], sql: expect.stringContaining( multiline( - `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2))` + `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2)) and exists (select 1 from "persons" as "c" where "c"."personid" = "a"."qaid" and ("c"."year" between $3 and $4)) and exists (select 1 from "products" as "b" inner join "products_tasks" as "d" on "b"."productid" = "d"."productid" where "d"."taskid" = "a"."taskid" and (COALESCE("b"."productname" = $5, FALSE))` ) ), }) From 5f1cd4eb9f562f3bf4e179fbf7a5caaa8d877977 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 16 Oct 2024 13:47:09 +0200 Subject: [PATCH 18/22] Test multiple relationship types --- .../server/src/api/routes/tests/search.spec.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index da0fd4fbba..0f5797f8de 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2278,12 +2278,16 @@ describe.each([ // It also can't work for in-memory searching because the related table name // isn't available. !isInMemory && - describe("relations", () => { + describe.each([ + RelationshipType.ONE_TO_MANY, + RelationshipType.MANY_TO_ONE, + RelationshipType.MANY_TO_MANY, + ])("relations (%s)", relationshipType => { let productCategoryTable: Table, productCatRows: Row[] beforeAll(async () => { const { relatedTable, tableId } = await basicRelationshipTables( - RelationshipType.ONE_TO_MANY + relationshipType ) tableOrViewId = tableId productCategoryTable = relatedTable @@ -2538,10 +2542,13 @@ describe.each([ }) isSql && - describe("big relations", () => { + describe.each([ + RelationshipType.MANY_TO_ONE, + RelationshipType.MANY_TO_MANY, + ])("big relations (%s)", relationshipType => { beforeAll(async () => { const { relatedTable, tableId } = await basicRelationshipTables( - RelationshipType.MANY_TO_ONE + relationshipType ) tableOrViewId = tableId const mainRow = await config.api.row.save(tableOrViewId, { From 3ea8e240e4716361850ddbb8c027d1fae968f531 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 16 Oct 2024 14:05:48 +0200 Subject: [PATCH 19/22] Fix one-to-many --- packages/backend-core/src/sql/sql.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b7b499b92c..662558415e 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -518,10 +518,12 @@ class InternalBuilder { mainKnex.raw(this.quotedIdentifier(foreignKey)) ) - query = query.whereExists(whereCb(updatedKey, subQuery)) - if (allowEmptyRelationships) { - query = query.orWhereNull(foreignKey) - } + query = query.where(q => { + q.whereExists(whereCb(updatedKey, subQuery)) + if (allowEmptyRelationships) { + q.orWhereNull(foreignKey) + } + }) } } } From 3fbe937bf43443647ce6709942bd32e3717045b2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 16 Oct 2024 14:23:58 +0200 Subject: [PATCH 20/22] Fix sqlalias test --- packages/server/src/integrations/tests/sqlAlias.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index ddc84ae1ce..739d3a4aee 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -145,7 +145,7 @@ describe("Captures of real examples", () => { ], sql: expect.stringContaining( multiline( - `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2)) and exists (select 1 from "persons" as "c" where "c"."personid" = "a"."qaid" and ("c"."year" between $3 and $4)) and (exists (select 1 from "products" as "b" inner join "products_tasks" as "d" on "b"."productid" = "d"."productid" where "d"."taskid" = "a"."taskid" and (COALESCE("b"."productname" = $5, FALSE)))` + `where (exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2))) and (exists (select 1 from "persons" as "c" where "c"."personid" = "a"."qaid" and ("c"."year" between $3 and $4))) and (exists (select 1 from "products" as "b" inner join "products_tasks" as "d" on "b"."productid" = "d"."productid" where "d"."taskid" = "a"."taskid" and (COALESCE("b"."productname" = $5, FALSE))))` ) ), }) From b6874f52f6d8e5228b57842a9844eba28f72e9e6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 16 Oct 2024 16:16:39 +0200 Subject: [PATCH 21/22] Fix many-to-one --- packages/backend-core/src/sql/sql.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 662558415e..949d7edf1b 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -510,18 +510,19 @@ class InternalBuilder { } }) } else { + const toKey = `${toAlias}.${relationship.to}` const foreignKey = `${fromAlias}.${relationship.from}` // "join" to the main table, making sure the ID matches that of the main subQuery = subQuery.where( - `${toAlias}.${relationship.to}`, + toKey, "=", mainKnex.raw(this.quotedIdentifier(foreignKey)) ) query = query.where(q => { - q.whereExists(whereCb(updatedKey, subQuery)) + q.whereExists(whereCb(updatedKey, subQuery.clone())) if (allowEmptyRelationships) { - q.orWhereNull(foreignKey) + q.orWhereNotExists(subQuery) } }) } From 42086f36a2194c0d2ce038d5e81ee36c27d4ca16 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 17 Oct 2024 07:30:11 +0000 Subject: [PATCH 22/22] Bump version to 2.33.2 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index caaa07683a..df50828b30 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.33.1", + "version": "2.33.2", "npmClient": "yarn", "packages": [ "packages/*",