From 120d5d953d2d404db9e0a5f57d599fc11e4a7abf Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 9 Aug 2024 10:16:07 +0200 Subject: [PATCH 01/30] Fetch only table fields and validate --- packages/server/src/sdk/app/rows/search.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 1ccd89639b..ea0b3bf5b6 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -73,6 +73,19 @@ export async function search( const table = await sdk.tables.getTable(options.tableId) options = searchInputMapping(table, options) + const visibleTableFields = Object.keys(table.schema).filter( + f => table.schema[f].visible !== false + ) + + if (options.fields) { + const tableFields = visibleTableFields.map(f => f.toLowerCase()) + options.fields = options.fields.filter(f => + tableFields.includes(f.toLowerCase()) + ) + } else { + options.fields = visibleTableFields + } + let result: SearchResponse if (isExternalTable) { span?.addTags({ searchType: "external" }) From 723dfee8c8a88f906d31c9cfac2f629c9c349402 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 9 Aug 2024 10:41:10 +0200 Subject: [PATCH 02/30] Add field mapping tests --- .../sdk/app/rows/search/tests/search.spec.ts | 83 ++++++++++++++++--- 1 file changed, 72 insertions(+), 11 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index 252b9a6556..339e7582fc 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -1,4 +1,4 @@ -import { Datasource, FieldType, Row, Table } from "@budibase/types" +import { Datasource, FieldType, Table } from "@budibase/types" import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" import { search } from "../../../../../sdk/app/rows/search" @@ -32,7 +32,6 @@ describe.each([ let envCleanup: (() => void) | undefined let datasource: Datasource | undefined let table: Table - let rows: Row[] beforeAll(async () => { await withCoreEnv({ SQS_SEARCH_ENABLE: isSqs ? "true" : "false" }, () => @@ -51,7 +50,9 @@ describe.each([ datasource: await dsProvider, }) } + }) + beforeEach(async () => { table = await config.api.table.save( tableForDatasource(datasource, { primary: ["id"], @@ -81,16 +82,13 @@ describe.each([ }) ) - rows = [] for (let i = 0; i < 10; i++) { - rows.push( - await config.api.row.save(table._id!, { - name: generator.first(), - surname: generator.last(), - age: generator.age(), - address: generator.address(), - }) - ) + await config.api.row.save(table._id!, { + name: generator.first(), + surname: generator.last(), + age: generator.age(), + address: generator.address(), + }) } }) @@ -138,4 +136,67 @@ describe.each([ ) }) }) + + it("does not allow accessing hidden fields", async () => { + await config.doInContext(config.appId, async () => { + await config.api.table.save({ + ...table, + schema: { + ...table.schema, + name: { + ...table.schema.name, + visible: true, + }, + age: { + ...table.schema.age, + visible: false, + }, + }, + }) + const result = await search({ + tableId: table._id!, + query: {}, + }) + expect(result.rows).toHaveLength(10) + for (const row of result.rows) { + const keys = Object.keys(row) + expect(keys).toContain("name") + expect(keys).toContain("surname") + expect(keys).toContain("address") + expect(keys).not.toContain("age") + } + }) + }) + + it("does not allow accessing hidden fields even if requested", async () => { + await config.doInContext(config.appId, async () => { + await config.api.table.save({ + ...table, + schema: { + ...table.schema, + name: { + ...table.schema.name, + visible: true, + }, + age: { + ...table.schema.age, + visible: false, + }, + }, + }) + const result = await search({ + tableId: table._id!, + query: {}, + fields: ["name", "age"], + }) + expect(result.rows).toHaveLength(10) + for (const row of result.rows) { + const keys = Object.keys(row) + expect(keys).toContain("name") + expect(keys).not.toContain("age") + expect(keys).not.toContain("surname") + expect(keys).not.toContain("address") + } + }) + }) }) From 3acc9bde45dd98a6904f27f96dafbdd7ae338c38 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 9 Aug 2024 10:43:33 +0200 Subject: [PATCH 03/30] Extra tests --- .../sdk/app/rows/search/tests/search.spec.ts | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index 339e7582fc..1219009381 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -199,4 +199,53 @@ describe.each([ } }) }) + + it("does not allow accessing non-mapped fields", async () => { + await config.doInContext(config.appId, async () => { + await config.api.table.save({ + ...table, + schema: { + name: table.schema.name, + surname: table.schema.surname, + }, + }) + const result = await search({ + tableId: table._id!, + query: {}, + }) + expect(result.rows).toHaveLength(10) + for (const row of result.rows) { + const keys = Object.keys(row) + expect(keys).toContain("name") + expect(keys).toContain("surname") + expect(keys).not.toContain("address") + expect(keys).not.toContain("age") + } + }) + }) + + it("does not allow accessing non-mapped fields even if requested", async () => { + await config.doInContext(config.appId, async () => { + await config.api.table.save({ + ...table, + schema: { + name: table.schema.name, + surname: table.schema.surname, + }, + }) + const result = await search({ + tableId: table._id!, + query: {}, + fields: ["name", "age"], + }) + expect(result.rows).toHaveLength(10) + for (const row of result.rows) { + const keys = Object.keys(row) + expect(keys).toContain("name") + expect(keys).not.toContain("age") + expect(keys).not.toContain("surname") + expect(keys).not.toContain("address") + } + }) + }) }) From 331e8eb7be6d4ecfc6d5cc24d732e6c32dc37903 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 9 Aug 2024 11:39:58 +0200 Subject: [PATCH 04/30] Add view tests --- .../src/api/routes/tests/viewV2.spec.ts | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index e18cffeaa8..9fec41ccac 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -1602,6 +1602,28 @@ describe.each([ }) expect(response.rows).toHaveLength(0) }) + + it("queries the row api passing the view fields only", async () => { + const searchSpy = jest.spyOn(sdk.rows, "search") + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + id: { visible: true }, + one: { visible: false }, + }, + }) + + await config.api.viewV2.search(view.id, { query: {} }) + expect(searchSpy).toHaveBeenCalledTimes(1) + + expect(searchSpy).toHaveBeenCalledWith( + expect.objectContaining({ + fields: ["id"], + }) + ) + }) }) describe("permissions", () => { From 0212f584d27c397d28fff555bb393beb58651827 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 9 Aug 2024 11:40:17 +0200 Subject: [PATCH 05/30] Add imports --- packages/server/src/api/routes/tests/viewV2.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 9fec41ccac..b714dcf5b6 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -30,6 +30,7 @@ import { withEnv as withCoreEnv, setEnv as setCoreEnv, } from "@budibase/backend-core" +import sdk from "../../../sdk" describe.each([ ["lucene", undefined], @@ -1623,7 +1624,7 @@ describe.each([ fields: ["id"], }) ) - }) + }) }) describe("permissions", () => { From 728b4d363d007ebe49848ae3a365a10139c2191e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 9 Aug 2024 11:47:16 +0200 Subject: [PATCH 06/30] Add removeInvalidFilters utils --- .../server/src/sdk/app/rows/queryUtils.ts | 45 +++++++++ .../src/sdk/app/rows/tests/queryUtils.spec.ts | 96 +++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 packages/server/src/sdk/app/rows/queryUtils.ts create mode 100644 packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts new file mode 100644 index 0000000000..84fe05a948 --- /dev/null +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -0,0 +1,45 @@ +import { isLogicalSearchOperator, SearchFilters } from "@budibase/types" +import { cloneDeep } from "lodash/fp" + +export const removeInvalidFilters = ( + filters: SearchFilters, + validFields: string[] +) => { + const result = cloneDeep(filters) + + validFields = validFields.map(f => f.toLowerCase()) + for (const filterKey of Object.keys(result) as (keyof SearchFilters)[]) { + if (typeof result[filterKey] !== "object") { + continue + } + if (isLogicalSearchOperator(filterKey)) { + const resultingConditions: SearchFilters[] = [] + for (const condition of result[filterKey].conditions) { + const resultingCondition = removeInvalidFilters(condition, validFields) + if (Object.keys(resultingCondition).length) { + resultingConditions.push(resultingCondition) + } + } + if (resultingConditions.length) { + result[filterKey].conditions = resultingConditions + } else { + delete result[filterKey] + } + continue + } + + const filter = result[filterKey] + for (const columnKey of Object.keys(filter)) { + if ( + !validFields.map(f => f.toLowerCase()).includes(columnKey.toLowerCase()) + ) { + delete filter[columnKey] + } + } + if (!Object.keys(filter).length) { + delete result[filterKey] + } + } + + return result +} diff --git a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts new file mode 100644 index 0000000000..b3852da4c6 --- /dev/null +++ b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts @@ -0,0 +1,96 @@ +import { SearchFilters } from "@budibase/types" +import { removeInvalidFilters } from "../queryUtils" + +describe("query utils", () => { + describe("removeInvalidFilters", () => { + const fullFilters: SearchFilters = { + equal: { one: "foo" }, + $or: { + conditions: [ + { + equal: { one: "foo2", two: "bar" }, + notEmpty: { one: null }, + $and: { + conditions: [ + { + equal: { three: "baz" }, + notEmpty: { forth: null }, + }, + ], + }, + }, + ], + }, + $and: { + conditions: [{ equal: { one: "foo2" }, notEmpty: { one: null } }], + }, + } + + it("can filter empty queries", () => { + const filters: SearchFilters = {} + const result = removeInvalidFilters(filters, []) + expect(result).toEqual({}) + }) + + it("does not trim any valid field", () => { + const result = removeInvalidFilters(fullFilters, [ + "one", + "two", + "three", + "forth", + ]) + expect(result).toEqual(fullFilters) + }) + + it("trims invalid field", () => { + const result = removeInvalidFilters(fullFilters, [ + "one", + "three", + "forth", + ]) + expect(result).toEqual({ + equal: { one: "foo" }, + $or: { + conditions: [ + { + equal: { one: "foo2" }, + notEmpty: { one: null }, + $and: { + conditions: [ + { + equal: { three: "baz" }, + notEmpty: { forth: null }, + }, + ], + }, + }, + ], + }, + $and: { + conditions: [{ equal: { one: "foo2" }, notEmpty: { one: null } }], + }, + }) + }) + + it("trims invalid field keeping a valid fields", () => { + const result = removeInvalidFilters(fullFilters, ["three", "forth"]) + const expected: SearchFilters = { + $or: { + conditions: [ + { + $and: { + conditions: [ + { + equal: { three: "baz" }, + notEmpty: { forth: null }, + }, + ], + }, + }, + ], + }, + } + expect(result).toEqual(expected) + }) + }) +}) From 42070dcd39841fab1737ab4a7989e14c7454431b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 9 Aug 2024 12:16:22 +0200 Subject: [PATCH 07/30] Add tests --- .../sdk/app/rows/search/tests/search.spec.ts | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index 1219009381..068c5f5f07 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -248,4 +248,37 @@ describe.each([ } }) }) + + !isLucene && + it.each([ + [["id", "name", "age"], 3], + [["name", "age"], 10], + ])( + "cannot query by non search fields", + async (queryFields, expectedRows) => { + await config.doInContext(config.appId, async () => { + const { rows } = await search({ + tableId: table._id!, + query: { + $or: { + conditions: [ + { + $and: { + conditions: [ + { range: { id: { low: 2, high: 4 } } }, + { range: { id: { low: 3, high: 5 } } }, + ], + }, + }, + { equal: { id: 7 } }, + ], + }, + }, + fields: queryFields, + }) + + expect(rows).toHaveLength(expectedRows) + }) + } + ) }) From 6fe628f804f246568df95d2170904845020593e2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 9 Aug 2024 12:31:54 +0200 Subject: [PATCH 08/30] Fix tests --- packages/server/src/sdk/app/rows/search/tests/search.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index 068c5f5f07..c3b5dc3c3b 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -229,6 +229,7 @@ describe.each([ await config.api.table.save({ ...table, schema: { + id: table.schema.id, name: table.schema.name, surname: table.schema.surname, }, From 5b13e53a3bbfb05fdc4ae0c77053d2d9c2f6a38f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 9 Aug 2024 12:32:18 +0200 Subject: [PATCH 09/30] Fix tests --- packages/server/src/sdk/app/rows/search/tests/search.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index c3b5dc3c3b..235598cc40 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -205,6 +205,7 @@ describe.each([ await config.api.table.save({ ...table, schema: { + id: table.schema.id, name: table.schema.name, surname: table.schema.surname, }, From 48486cdaa4aa8bdc664b8ee33ea4fabab4f74452 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 9 Aug 2024 13:14:16 +0200 Subject: [PATCH 10/30] Implement query --- packages/server/src/sdk/app/rows/search.ts | 6 ++++ .../sdk/app/rows/search/tests/search.spec.ts | 31 ++++++++++++++----- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index ea0b3bf5b6..e840c1ca51 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -14,6 +14,7 @@ import sdk from "../../index" import { searchInputMapping } from "./search/utils" import { db as dbCore } from "@budibase/backend-core" import tracer from "dd-trace" +import { removeInvalidFilters } from "./queryUtils" export { isValidFilter } from "../../../integrations/utils" @@ -86,6 +87,11 @@ export async function search( options.fields = visibleTableFields } + options.query = removeInvalidFilters(options.query, [ + "_id", + ...options.fields, + ]) + let result: SearchResponse if (isExternalTable) { span?.addTags({ searchType: "external" }) diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index 235598cc40..510aa209a8 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -1,4 +1,11 @@ -import { Datasource, FieldType, Table } from "@budibase/types" +import { + AutoColumnFieldMetadata, + AutoFieldSubType, + Datasource, + FieldType, + NumberFieldMetadata, + Table, +} from "@budibase/types" import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" import { search } from "../../../../../sdk/app/rows/search" @@ -53,15 +60,25 @@ describe.each([ }) beforeEach(async () => { + const idFieldSchema: NumberFieldMetadata | AutoColumnFieldMetadata = + isInternal + ? { + name: "id", + type: FieldType.AUTO, + subtype: AutoFieldSubType.AUTO_ID, + autocolumn: true, + } + : { + name: "id", + type: FieldType.NUMBER, + autocolumn: true, + } + table = await config.api.table.save( tableForDatasource(datasource, { primary: ["id"], schema: { - id: { - name: "id", - type: FieldType.NUMBER, - autocolumn: true, - }, + id: idFieldSchema, name: { name: "name", type: FieldType.STRING, @@ -256,7 +273,7 @@ describe.each([ [["id", "name", "age"], 3], [["name", "age"], 10], ])( - "cannot query by non search fields", + "cannot query by non search fields (fields: %s)", async (queryFields, expectedRows) => { await config.doInContext(config.appId, async () => { const { rows } = await search({ From 0bdb6396ed25e3f37639391ffb91a273fd0a155c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 9 Aug 2024 14:17:41 +0200 Subject: [PATCH 11/30] Remove unneeded test --- .../sdk/app/rows/search/tests/search.spec.ts | 51 ------------------- 1 file changed, 51 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts index 510aa209a8..db1104bcf6 100644 --- a/packages/server/src/sdk/app/rows/search/tests/search.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/search.spec.ts @@ -217,57 +217,6 @@ describe.each([ }) }) - it("does not allow accessing non-mapped fields", async () => { - await config.doInContext(config.appId, async () => { - await config.api.table.save({ - ...table, - schema: { - id: table.schema.id, - name: table.schema.name, - surname: table.schema.surname, - }, - }) - const result = await search({ - tableId: table._id!, - query: {}, - }) - expect(result.rows).toHaveLength(10) - for (const row of result.rows) { - const keys = Object.keys(row) - expect(keys).toContain("name") - expect(keys).toContain("surname") - expect(keys).not.toContain("address") - expect(keys).not.toContain("age") - } - }) - }) - - it("does not allow accessing non-mapped fields even if requested", async () => { - await config.doInContext(config.appId, async () => { - await config.api.table.save({ - ...table, - schema: { - id: table.schema.id, - name: table.schema.name, - surname: table.schema.surname, - }, - }) - const result = await search({ - tableId: table._id!, - query: {}, - fields: ["name", "age"], - }) - expect(result.rows).toHaveLength(10) - for (const row of result.rows) { - const keys = Object.keys(row) - expect(keys).toContain("name") - expect(keys).not.toContain("age") - expect(keys).not.toContain("surname") - expect(keys).not.toContain("address") - } - }) - }) - !isLucene && it.each([ [["id", "name", "age"], 3], From ac62fc0e292c0e1f2f4a0c1acafe94639e70f104 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 9 Aug 2024 14:29:19 +0200 Subject: [PATCH 12/30] Fix jest asserts --- packages/server/src/api/routes/tests/viewV2.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index b714dcf5b6..ad6652635f 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -121,6 +121,7 @@ describe.each([ }) beforeEach(() => { + jest.clearAllMocks() mocks.licenses.useCloudFree() }) @@ -1604,7 +1605,7 @@ describe.each([ expect(response.rows).toHaveLength(0) }) - it("queries the row api passing the view fields only", async () => { + it.only("queries the row api passing the view fields only", async () => { const searchSpy = jest.spyOn(sdk.rows, "search") const view = await config.api.viewV2.create({ From b1d78f129b7b870512713825a84742ffcafa0a8b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 9 Aug 2024 14:29:44 +0200 Subject: [PATCH 13/30] Remove jest only --- packages/server/src/api/routes/tests/viewV2.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index ad6652635f..da38112e2b 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -1605,7 +1605,7 @@ describe.each([ expect(response.rows).toHaveLength(0) }) - it.only("queries the row api passing the view fields only", async () => { + it("queries the row api passing the view fields only", async () => { const searchSpy = jest.spyOn(sdk.rows, "search") const view = await config.api.viewV2.create({ From ee5c4e8ed828c505726941825b7f4629e98602e9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 12 Aug 2024 10:56:06 +0200 Subject: [PATCH 14/30] Don't trim prefixed keys --- .../server/src/sdk/app/rows/queryUtils.ts | 8 ++- .../src/sdk/app/rows/tests/queryUtils.spec.ts | 53 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index 84fe05a948..9fa72e84d7 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -1,3 +1,4 @@ +import { db } from "@budibase/backend-core" import { isLogicalSearchOperator, SearchFilters } from "@budibase/types" import { cloneDeep } from "lodash/fp" @@ -31,7 +32,12 @@ export const removeInvalidFilters = ( const filter = result[filterKey] for (const columnKey of Object.keys(filter)) { if ( - !validFields.map(f => f.toLowerCase()).includes(columnKey.toLowerCase()) + !validFields + .map(f => f.toLowerCase()) + .includes(columnKey.toLowerCase()) && + !validFields + .map(f => f.toLowerCase()) + .includes(db.removeKeyNumbering(columnKey).toLowerCase()) ) { delete filter[columnKey] } diff --git a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts index b3852da4c6..f249a8f91e 100644 --- a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts @@ -92,5 +92,58 @@ describe("query utils", () => { } expect(result).toEqual(expected) }) + + it("keeps filter key numering", () => { + const prefixedFilters: SearchFilters = { + equal: { "1:one": "foo" }, + $or: { + conditions: [ + { + equal: { "2:one": "foo2", "3:two": "bar" }, + notEmpty: { "4:one": null }, + $and: { + conditions: [ + { + equal: { "5:three": "baz", two: "bar2" }, + notEmpty: { forth: null }, + }, + ], + }, + }, + ], + }, + $and: { + conditions: [{ equal: { "6:one": "foo2" }, notEmpty: { one: null } }], + }, + } + + const result = removeInvalidFilters(prefixedFilters, [ + "one", + "three", + "forth", + ]) + expect(result).toEqual({ + equal: { "1:one": "foo" }, + $or: { + conditions: [ + { + equal: { "2:one": "foo2" }, + notEmpty: { "4:one": null }, + $and: { + conditions: [ + { + equal: { "5:three": "baz" }, + notEmpty: { forth: null }, + }, + ], + }, + }, + ], + }, + $and: { + conditions: [{ equal: { "6:one": "foo2" }, notEmpty: { one: null } }], + }, + }) + }) }) }) From 59a164f4df4c25e06120a61f786208b4d25e0049 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 12 Aug 2024 12:57:07 +0200 Subject: [PATCH 15/30] Test relationships --- .../server/src/sdk/app/rows/queryUtils.ts | 12 +++---- .../src/sdk/app/rows/tests/queryUtils.spec.ts | 33 +++++++++++++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index 9fa72e84d7..20e6af4d67 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -31,14 +31,10 @@ export const removeInvalidFilters = ( const filter = result[filterKey] for (const columnKey of Object.keys(filter)) { - if ( - !validFields - .map(f => f.toLowerCase()) - .includes(columnKey.toLowerCase()) && - !validFields - .map(f => f.toLowerCase()) - .includes(db.removeKeyNumbering(columnKey).toLowerCase()) - ) { + const possibleKeys = [columnKey, db.removeKeyNumbering(columnKey)].map( + c => c.toLowerCase() + ) + if (!validFields.some(f => possibleKeys.includes(f.toLowerCase()))) { delete filter[columnKey] } } diff --git a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts index f249a8f91e..4970e83685 100644 --- a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts @@ -145,5 +145,38 @@ describe("query utils", () => { }, }) }) + + it("handles relationship filters", () => { + const prefixedFilters: SearchFilters = { + $or: { + conditions: [ + { equal: { "1:other.one": "foo" } }, + { + equal: { + "2:other.one": "foo2", + "3:other.two": "bar", + "4:other.three": "baz", + }, + }, + { equal: { "another.three": "baz2" } }, + ], + }, + } + + const result = removeInvalidFilters(prefixedFilters, [ + "other.one", + "other.two", + "another.three", + ]) + expect(result).toEqual({ + $or: { + conditions: [ + { equal: { "1:other.one": "foo" } }, + { equal: { "2:other.one": "foo2", "3:other.two": "bar" } }, + { equal: { "another.three": "baz2" } }, + ], + }, + }) + }) }) }) From 08b0b6af197d0f35392d2655f39909d67ebc4805 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 12 Aug 2024 13:10:38 +0200 Subject: [PATCH 16/30] Fix search relationships --- packages/server/src/sdk/app/rows/search.ts | 56 ++++++++++++++++++++-- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index e840c1ca51..a3c695c61c 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -1,9 +1,12 @@ import { EmptyFilterOption, + FieldType, Row, RowSearchParams, SearchResponse, SortOrder, + Table, + TableSchema, } from "@budibase/types" import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./search/internal" @@ -87,10 +90,10 @@ export async function search( options.fields = visibleTableFields } - options.query = removeInvalidFilters(options.query, [ - "_id", - ...options.fields, - ]) + options.query = removeInvalidFilters( + options.query, + await getQueriableFields(options.fields, table.schema) + ) let result: SearchResponse if (isExternalTable) { @@ -134,3 +137,48 @@ export async function fetchView( ): Promise { return pickApi(tableId).fetchView(viewName, params) } + +async function getQueriableFields( + fields: string[], + schema: TableSchema +): Promise { + const handledTables: Record = {} + const extractTableFields = async ( + fromField: string, + tableId: string + ): Promise => { + const result = [] + if (handledTables[tableId]) { + return [] + } + const table = await sdk.tables.getTable(tableId) + handledTables[tableId] = table + + for (const field of Object.keys(table.schema)) { + const formattedColumn = `${fromField}.${field}` + const subSchema = table.schema[field] + if (subSchema.type === FieldType.LINK) { + result.push( + ...(await extractTableFields(formattedColumn, subSchema.tableId)) + ) + } else { + result.push(formattedColumn) + } + } + return result + } + + const result = [ + "_id", // Querying by _id is always allowed, even if it's never part of the schema + ] + + for (const field of fields) { + if (schema[field].type === FieldType.LINK) { + result.push(...(await extractTableFields(field, schema[field].tableId))) + } else { + result.push(field) + } + } + + return result +} From 1a88d9f89bfc3fa5a10ecea4c235616fae281fb4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 12 Aug 2024 14:21:05 +0200 Subject: [PATCH 17/30] Don't clean relationship query --- packages/server/src/sdk/app/rows/search.ts | 47 +++++++++++----------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index a3c695c61c..f4082a16fd 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -6,7 +6,6 @@ import { SearchResponse, SortOrder, Table, - TableSchema, } from "@budibase/types" import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./search/internal" @@ -92,7 +91,7 @@ export async function search( options.query = removeInvalidFilters( options.query, - await getQueriableFields(options.fields, table.schema) + await getQueriableFields(options.fields, table) ) let result: SearchResponse @@ -140,29 +139,35 @@ export async function fetchView( async function getQueriableFields( fields: string[], - schema: TableSchema + table: Table ): Promise { - const handledTables: Record = {} + const handledTables = new Set([table._id!]) const extractTableFields = async ( - fromField: string, - tableId: string + table: Table, + allowedFields: string[] ): Promise => { const result = [] - if (handledTables[tableId]) { - return [] - } - const table = await sdk.tables.getTable(tableId) - handledTables[tableId] = table - - for (const field of Object.keys(table.schema)) { - const formattedColumn = `${fromField}.${field}` + for (const field of Object.keys(table.schema).filter(f => + allowedFields.includes(f) + )) { const subSchema = table.schema[field] if (subSchema.type === FieldType.LINK) { - result.push( - ...(await extractTableFields(formattedColumn, subSchema.tableId)) + if (handledTables.has(`${table._id}_${subSchema.tableId}`)) { + // avoid circular loops + continue + } + handledTables.add(`${subSchema.tableId}_${table._id}`) + const relatedTable = await sdk.tables.getTable(subSchema.tableId) + const relatedFields = await extractTableFields( + relatedTable, + Object.keys(relatedTable.schema) ) + + result.push(...relatedFields.map(f => `${subSchema.name}.${f}`)) + // should be able to filter by relationship using table name + result.push(...relatedFields.map(f => `${relatedTable.name}.${f}`)) } else { - result.push(formattedColumn) + result.push(field) } } return result @@ -172,13 +177,7 @@ async function getQueriableFields( "_id", // Querying by _id is always allowed, even if it's never part of the schema ] - for (const field of fields) { - if (schema[field].type === FieldType.LINK) { - result.push(...(await extractTableFields(field, schema[field].tableId))) - } else { - result.push(field) - } - } + result.push(...(await extractTableFields(table, fields))) return result } From 385c5f6e99e6a68e0d656baa6f30996dadde13fc Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 14 Aug 2024 12:24:49 +0200 Subject: [PATCH 18/30] Trim unwanted columns in outputProcessing --- .../src/api/controllers/row/utils/sqlUtils.ts | 8 ++-- .../src/utilities/rowProcessor/index.ts | 48 +++++++++++++------ 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index 32124fa79d..7beadabd89 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -192,10 +192,10 @@ export function buildSqlFieldList( function extractRealFields(table: Table, existing: string[] = []) { return Object.entries(table.schema) .filter( - column => - column[1].type !== FieldType.LINK && - column[1].type !== FieldType.FORMULA && - !existing.find((field: string) => field === column[0]) + ([columnName, column]) => + column.type !== FieldType.LINK && + column.type !== FieldType.FORMULA && + !existing.find((field: string) => field === columnName) ) .map(column => `${table.name}.${column[0]}`) } diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 62a3b2dd74..38cb34a395 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -19,6 +19,7 @@ import { User, } from "@budibase/types" import { cloneDeep } from "lodash/fp" +import { pick } from "lodash" import { processInputBBReference, processInputBBReferences, @@ -26,7 +27,11 @@ import { processOutputBBReferences, } from "./bbReferenceProcessor" import { isExternalTableID } from "../../integrations/utils" -import { helpers } from "@budibase/shared-core" +import { + helpers, + PROTECTED_EXTERNAL_COLUMNS, + PROTECTED_INTERNAL_COLUMNS, +} from "@budibase/shared-core" import { processString } from "@budibase/string-templates" export * from "./utils" @@ -53,9 +58,9 @@ export async function processAutoColumn( row: Row, opts?: AutoColumnProcessingOpts ) { - let noUser = !userId - let isUserTable = table._id === InternalTables.USER_METADATA - let now = new Date().toISOString() + const noUser = !userId + const isUserTable = table._id === InternalTables.USER_METADATA + const now = new Date().toISOString() // if a row doesn't have a revision then it doesn't exist yet const creating = !row._rev // check its not user table, or whether any of the processing options have been disabled @@ -111,7 +116,7 @@ async function processDefaultValues(table: Table, row: Row) { ctx.user = user } - for (let [key, schema] of Object.entries(table.schema)) { + for (const [key, schema] of Object.entries(table.schema)) { if ("default" in schema && schema.default != null && row[key] == null) { const processed = await processString(schema.default, ctx) @@ -165,10 +170,10 @@ export async function inputProcessing( row: Row, opts?: AutoColumnProcessingOpts ) { - let clonedRow = cloneDeep(row) + const clonedRow = cloneDeep(row) const dontCleanseKeys = ["type", "_id", "_rev", "tableId"] - for (let [key, value] of Object.entries(clonedRow)) { + for (const [key, value] of Object.entries(clonedRow)) { const field = table.schema[key] // cleanse fields that aren't in the schema if (!field) { @@ -268,13 +273,13 @@ export async function outputProcessing( } // process complex types: attachments, bb references... - for (let [property, column] of Object.entries(table.schema)) { + for (const [property, column] of Object.entries(table.schema)) { if ( column.type === FieldType.ATTACHMENTS || column.type === FieldType.ATTACHMENT_SINGLE || column.type === FieldType.SIGNATURE_SINGLE ) { - for (let row of enriched) { + for (const row of enriched) { if (row[property] == null) { continue } @@ -299,7 +304,7 @@ export async function outputProcessing( !opts.skipBBReferences && column.type == FieldType.BB_REFERENCE ) { - for (let row of enriched) { + for (const row of enriched) { row[property] = await processOutputBBReferences( row[property], column.subtype @@ -309,14 +314,14 @@ export async function outputProcessing( !opts.skipBBReferences && column.type == FieldType.BB_REFERENCE_SINGLE ) { - for (let row of enriched) { + for (const row of enriched) { row[property] = await processOutputBBReference( row[property], column.subtype ) } } else if (column.type === FieldType.DATETIME && column.timeOnly) { - for (let row of enriched) { + for (const row of enriched) { if (row[property] instanceof Date) { const hours = row[property].getUTCHours().toString().padStart(2, "0") const minutes = row[property] @@ -343,14 +348,27 @@ export async function outputProcessing( )) as Row[] } // remove null properties to match internal API - if (isExternalTableID(table._id!)) { - for (let row of enriched) { - for (let key of Object.keys(row)) { + const isExternal = isExternalTableID(table._id!) + if (isExternal) { + for (const row of enriched) { + for (const key of Object.keys(row)) { if (row[key] === null) { delete row[key] } } } } + + const protectedColumns = isExternal + ? PROTECTED_EXTERNAL_COLUMNS + : PROTECTED_INTERNAL_COLUMNS + + const tableFields = Object.keys(table.schema).filter( + f => table.schema[f].visible !== false + ) + enriched = enriched.map((r: Row) => + pick(r, [...tableFields, ...protectedColumns]) + ) + return (wasArray ? enriched : enriched[0]) as T } From ce8730f176dfaa90d0d403ad75eb4e9d63b4af51 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 14 Aug 2024 12:30:01 +0200 Subject: [PATCH 19/30] Clean code --- packages/server/src/sdk/app/rows/search.ts | 24 ++++++++-------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index f4082a16fd..c96c0738d2 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -76,23 +76,17 @@ export async function search( const table = await sdk.tables.getTable(options.tableId) options = searchInputMapping(table, options) - const visibleTableFields = Object.keys(table.schema).filter( - f => table.schema[f].visible !== false - ) - - if (options.fields) { - const tableFields = visibleTableFields.map(f => f.toLowerCase()) - options.fields = options.fields.filter(f => - tableFields.includes(f.toLowerCase()) + if (options.query) { + const tableFields = Object.keys(table.schema).filter( + f => table.schema[f].visible !== false ) - } else { - options.fields = visibleTableFields - } - options.query = removeInvalidFilters( - options.query, - await getQueriableFields(options.fields, table) - ) + const queriableFields = await getQueriableFields( + options.fields?.filter(f => tableFields.includes(f)) ?? tableFields, + table + ) + options.query = removeInvalidFilters(options.query, queriableFields) + } let result: SearchResponse if (isExternalTable) { From da87c0233b1d9e4d1efc218f746ed7c9db1eeb15 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 14 Aug 2024 12:51:46 +0200 Subject: [PATCH 20/30] Don't trim usertable --- .../src/utilities/rowProcessor/index.ts | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 38cb34a395..59b3ea7c16 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -33,6 +33,7 @@ import { PROTECTED_INTERNAL_COLUMNS, } from "@budibase/shared-core" import { processString } from "@budibase/string-templates" +import { isUserMetadataTable } from "../../api/controllers/row/utils" export * from "./utils" export * from "./attachments" @@ -359,16 +360,18 @@ export async function outputProcessing( } } - const protectedColumns = isExternal - ? PROTECTED_EXTERNAL_COLUMNS - : PROTECTED_INTERNAL_COLUMNS + if (!isUserMetadataTable(table._id!)) { + const protectedColumns = isExternal + ? PROTECTED_EXTERNAL_COLUMNS + : PROTECTED_INTERNAL_COLUMNS - const tableFields = Object.keys(table.schema).filter( - f => table.schema[f].visible !== false - ) - enriched = enriched.map((r: Row) => - pick(r, [...tableFields, ...protectedColumns]) - ) + const tableFields = Object.keys(table.schema).filter( + f => table.schema[f].visible !== false + ) + enriched = enriched.map((r: Row) => + pick(r, [...tableFields, ...protectedColumns]) + ) + } return (wasArray ? enriched : enriched[0]) as T } From f53500450ae2a97e93bf93f03a2d2b436b2bcd66 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 14 Aug 2024 13:00:13 +0200 Subject: [PATCH 21/30] Fix wrong test --- packages/server/src/api/routes/tests/row.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 9351976d66..95e714fbd6 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1664,7 +1664,7 @@ describe.each([ isInternal && describe("attachments and signatures", () => { const coreAttachmentEnrichment = async ( - schema: any, + schema: TableSchema, field: string, attachmentCfg: string | string[] ) => { @@ -1691,7 +1691,7 @@ describe.each([ await withEnv({ SELF_HOSTED: "true" }, async () => { return context.doInAppContext(config.getAppId(), async () => { - const enriched: Row[] = await outputProcessing(table, [row]) + const enriched: Row[] = await outputProcessing(testTable, [row]) const [targetRow] = enriched const attachmentEntries = Array.isArray(targetRow[field]) ? targetRow[field] From 09938ae442bf5b94c0d7a5510df1023bd038d830 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 14 Aug 2024 13:13:15 +0200 Subject: [PATCH 22/30] Don't change key order on trimming --- packages/server/src/utilities/rowProcessor/index.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 59b3ea7c16..4b2fd83882 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -19,7 +19,6 @@ import { User, } from "@budibase/types" import { cloneDeep } from "lodash/fp" -import { pick } from "lodash" import { processInputBBReference, processInputBBReferences, @@ -368,9 +367,16 @@ export async function outputProcessing( const tableFields = Object.keys(table.schema).filter( f => table.schema[f].visible !== false ) - enriched = enriched.map((r: Row) => - pick(r, [...tableFields, ...protectedColumns]) + const fields = [...tableFields, ...protectedColumns].map(f => + f.toLowerCase() ) + for (const row of enriched) { + for (const key of Object.keys(row)) { + if (!fields.includes(key.toLowerCase())) { + delete row[key] + } + } + } } return (wasArray ? enriched : enriched[0]) as T From 084a481821f509a5b865248b5a05ad2372ee460e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 14 Aug 2024 14:17:32 +0200 Subject: [PATCH 23/30] Typo --- packages/server/src/sdk/app/rows/search.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index c96c0738d2..1bf25ff926 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -81,7 +81,7 @@ export async function search( f => table.schema[f].visible !== false ) - const queriableFields = await getQueriableFields( + const queriableFields = await getQueryableFields( options.fields?.filter(f => tableFields.includes(f)) ?? tableFields, table ) @@ -131,7 +131,7 @@ export async function fetchView( return pickApi(tableId).fetchView(viewName, params) } -async function getQueriableFields( +async function getQueryableFields( fields: string[], table: Table ): Promise { From eb2d71e9808a200f514e935cc082a1f3939e88d7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 14 Aug 2024 14:35:20 +0200 Subject: [PATCH 24/30] Move getQueryableFields to utils --- .../server/src/sdk/app/rows/queryUtils.ts | 53 ++++++++++++++++++- packages/server/src/sdk/app/rows/search.ts | 49 +---------------- 2 files changed, 53 insertions(+), 49 deletions(-) diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index 20e6af4d67..02e77a77b7 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -1,6 +1,12 @@ import { db } from "@budibase/backend-core" -import { isLogicalSearchOperator, SearchFilters } from "@budibase/types" +import { + FieldType, + isLogicalSearchOperator, + SearchFilters, + Table, +} from "@budibase/types" import { cloneDeep } from "lodash/fp" +import sdk from "../../../sdk" export const removeInvalidFilters = ( filters: SearchFilters, @@ -45,3 +51,48 @@ export const removeInvalidFilters = ( return result } + +export const getQueryableFields = async ( + fields: string[], + table: Table +): Promise => { + const handledTables = new Set([table._id!]) + const extractTableFields = async ( + table: Table, + allowedFields: string[] + ): Promise => { + const result = [] + for (const field of Object.keys(table.schema).filter(f => + allowedFields.includes(f) + )) { + const subSchema = table.schema[field] + if (subSchema.type === FieldType.LINK) { + if (handledTables.has(`${table._id}_${subSchema.tableId}`)) { + // avoid circular loops + continue + } + handledTables.add(`${table._id}_${subSchema.tableId}`) + const relatedTable = await sdk.tables.getTable(subSchema.tableId) + const relatedFields = await extractTableFields( + relatedTable, + Object.keys(relatedTable.schema) + ) + + result.push(...relatedFields.map(f => `${subSchema.name}.${f}`)) + // should be able to filter by relationship using table name + result.push(...relatedFields.map(f => `${relatedTable.name}.${f}`)) + } else { + result.push(field) + } + } + return result + } + + const result = [ + "_id", // Querying by _id is always allowed, even if it's never part of the schema + ] + + result.push(...(await extractTableFields(table, fields))) + + return result +} diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 1bf25ff926..6a4286814d 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -1,11 +1,9 @@ import { EmptyFilterOption, - FieldType, Row, RowSearchParams, SearchResponse, SortOrder, - Table, } from "@budibase/types" import { isExternalTableID } from "../../../integrations/utils" import * as internal from "./search/internal" @@ -16,7 +14,7 @@ import sdk from "../../index" import { searchInputMapping } from "./search/utils" import { db as dbCore } from "@budibase/backend-core" import tracer from "dd-trace" -import { removeInvalidFilters } from "./queryUtils" +import { getQueryableFields, removeInvalidFilters } from "./queryUtils" export { isValidFilter } from "../../../integrations/utils" @@ -130,48 +128,3 @@ export async function fetchView( ): Promise { return pickApi(tableId).fetchView(viewName, params) } - -async function getQueryableFields( - fields: string[], - table: Table -): Promise { - const handledTables = new Set([table._id!]) - const extractTableFields = async ( - table: Table, - allowedFields: string[] - ): Promise => { - const result = [] - for (const field of Object.keys(table.schema).filter(f => - allowedFields.includes(f) - )) { - const subSchema = table.schema[field] - if (subSchema.type === FieldType.LINK) { - if (handledTables.has(`${table._id}_${subSchema.tableId}`)) { - // avoid circular loops - continue - } - handledTables.add(`${subSchema.tableId}_${table._id}`) - const relatedTable = await sdk.tables.getTable(subSchema.tableId) - const relatedFields = await extractTableFields( - relatedTable, - Object.keys(relatedTable.schema) - ) - - result.push(...relatedFields.map(f => `${subSchema.name}.${f}`)) - // should be able to filter by relationship using table name - result.push(...relatedFields.map(f => `${relatedTable.name}.${f}`)) - } else { - result.push(field) - } - } - return result - } - - const result = [ - "_id", // Querying by _id is always allowed, even if it's never part of the schema - ] - - result.push(...(await extractTableFields(table, fields))) - - return result -} From f0a89764f21839c33c0cdc84ae876c6463298701 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 14 Aug 2024 14:51:17 +0200 Subject: [PATCH 25/30] Add basic test --- .../src/sdk/app/rows/tests/queryUtils.spec.ts | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts index 4970e83685..a88ceace90 100644 --- a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts @@ -1,5 +1,6 @@ -import { SearchFilters } from "@budibase/types" -import { removeInvalidFilters } from "../queryUtils" +import { FieldType, SearchFilters, Table } from "@budibase/types" +import { getQueryableFields, removeInvalidFilters } from "../queryUtils" +import { structures } from "../../../../api/routes/tests/utilities" describe("query utils", () => { describe("removeInvalidFilters", () => { @@ -179,4 +180,19 @@ describe("query utils", () => { }) }) }) + + describe("getQueryableFields", () => { + it("allows querying by table schema fields and _id", async () => { + const table: Table = { + ...structures.basicTable(), + schema: { + name: { name: "name", type: FieldType.STRING }, + age: { name: "age", type: FieldType.NUMBER }, + }, + } + + const result = await getQueryableFields(["name", "age"], table) + expect(result).toEqual(["_id", "name", "age"]) + }) + }) }) From b744ec3c3552c07987834319e3491399ddd32d3f Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 14 Aug 2024 15:02:30 +0200 Subject: [PATCH 26/30] Add extra tests --- .../server/src/sdk/app/rows/queryUtils.ts | 4 +- .../src/sdk/app/rows/tests/queryUtils.spec.ts | 67 ++++++++++++++++++- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index 02e77a77b7..5eb2b2d1c8 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -62,8 +62,8 @@ export const getQueryableFields = async ( allowedFields: string[] ): Promise => { const result = [] - for (const field of Object.keys(table.schema).filter(f => - allowedFields.includes(f) + for (const field of Object.keys(table.schema).filter( + f => allowedFields.includes(f) && table.schema[f].visible !== false )) { const subSchema = table.schema[field] if (subSchema.type === FieldType.LINK) { diff --git a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts index a88ceace90..9a824069d5 100644 --- a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts @@ -1,6 +1,12 @@ -import { FieldType, SearchFilters, Table } from "@budibase/types" +import { + FieldType, + RelationshipType, + SearchFilters, + Table, +} from "@budibase/types" import { getQueryableFields, removeInvalidFilters } from "../queryUtils" import { structures } from "../../../../api/routes/tests/utilities" +import TestConfiguration from "../../../../tests/utilities/TestConfiguration" describe("query utils", () => { describe("removeInvalidFilters", () => { @@ -182,7 +188,13 @@ describe("query utils", () => { }) describe("getQueryableFields", () => { - it("allows querying by table schema fields and _id", async () => { + const config = new TestConfiguration() + + beforeAll(async () => { + await config.init() + }) + + it("returns table schema fields and _id", async () => { const table: Table = { ...structures.basicTable(), schema: { @@ -194,5 +206,56 @@ describe("query utils", () => { const result = await getQueryableFields(["name", "age"], table) expect(result).toEqual(["_id", "name", "age"]) }) + + it("does not return hidden fields", async () => { + const table: Table = { + ...structures.basicTable(), + schema: { + name: { name: "name", type: FieldType.STRING }, + age: { name: "age", type: FieldType.NUMBER, visible: false }, + }, + } + + const result = await getQueryableFields(["name", "age"], table) + expect(result).toEqual(["_id", "name"]) + }) + + it("includes relationship fields", async () => { + const aux: Table = await config.api.table.save({ + ...structures.basicTable(), + name: "auxTable", + schema: { + title: { name: "title", type: FieldType.STRING }, + name: { name: "name", type: FieldType.STRING }, + }, + }) + + const table: Table = { + ...structures.basicTable(), + schema: { + name: { name: "name", type: FieldType.STRING }, + age: { name: "age", type: FieldType.NUMBER, visible: false }, + aux: { + name: "aux", + type: FieldType.LINK, + tableId: aux._id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "table", + }, + }, + } + + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(["name", "age", "aux"], table) + }) + expect(result).toEqual([ + "_id", + "name", + "aux.title", + "aux.name", + "auxTable.title", + "auxTable.name", + ]) + }) }) }) From d8462ba96168a840dd34c6f620bc0fb992f6669b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 14 Aug 2024 18:16:46 +0200 Subject: [PATCH 27/30] Fix tests --- .../server/src/sdk/app/rows/queryUtils.ts | 10 +- .../src/sdk/app/rows/tests/queryUtils.spec.ts | 129 +++++++++++++++++- 2 files changed, 130 insertions(+), 9 deletions(-) diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index 5eb2b2d1c8..dba33d7694 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -78,9 +78,13 @@ export const getQueryableFields = async ( Object.keys(relatedTable.schema) ) - result.push(...relatedFields.map(f => `${subSchema.name}.${f}`)) - // should be able to filter by relationship using table name - result.push(...relatedFields.map(f => `${relatedTable.name}.${f}`)) + result.push( + ...relatedFields.flatMap(f => [ + `${subSchema.name}.${f}`, + // should be able to filter by relationship using table name + `${relatedTable.name}.${f}`, + ]) + ) } else { result.push(field) } diff --git a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts index 9a824069d5..e422ef75a7 100644 --- a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts @@ -203,11 +203,11 @@ describe("query utils", () => { }, } - const result = await getQueryableFields(["name", "age"], table) + const result = await getQueryableFields(Object.keys(table.schema), table) expect(result).toEqual(["_id", "name", "age"]) }) - it("does not return hidden fields", async () => { + it("excludes hidden fields", async () => { const table: Table = { ...structures.basicTable(), schema: { @@ -216,7 +216,7 @@ describe("query utils", () => { }, } - const result = await getQueryableFields(["name", "age"], table) + const result = await getQueryableFields(Object.keys(table.schema), table) expect(result).toEqual(["_id", "name"]) }) @@ -234,7 +234,6 @@ describe("query utils", () => { ...structures.basicTable(), schema: { name: { name: "name", type: FieldType.STRING }, - age: { name: "age", type: FieldType.NUMBER, visible: false }, aux: { name: "aux", type: FieldType.LINK, @@ -246,16 +245,134 @@ describe("query utils", () => { } const result = await config.doInContext(config.appId, () => { - return getQueryableFields(["name", "age", "aux"], table) + return getQueryableFields(Object.keys(table.schema), table) }) expect(result).toEqual([ "_id", "name", "aux.title", - "aux.name", "auxTable.title", + "aux.name", "auxTable.name", ]) }) + + it("excludes hidden relationship fields", async () => { + const aux: Table = await config.api.table.save({ + ...structures.basicTable(), + name: "auxTable", + schema: { + title: { name: "title", type: FieldType.STRING, visible: false }, + name: { name: "name", type: FieldType.STRING, visible: true }, + }, + }) + + const table: Table = { + ...structures.basicTable(), + schema: { + name: { name: "name", type: FieldType.STRING }, + aux: { + name: "aux", + type: FieldType.LINK, + tableId: aux._id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "table", + }, + }, + } + + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(table.schema), table) + }) + expect(result).toEqual(["_id", "name", "aux.name", "auxTable.name"]) + }) + + it("excludes all relationship fields if hidden", async () => { + const aux: Table = await config.api.table.save({ + ...structures.basicTable(), + name: "auxTable", + schema: { + title: { name: "title", type: FieldType.STRING, visible: false }, + name: { name: "name", type: FieldType.STRING, visible: true }, + }, + }) + + const table: Table = { + ...structures.basicTable(), + schema: { + name: { name: "name", type: FieldType.STRING }, + aux: { + name: "aux", + type: FieldType.LINK, + tableId: aux._id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "table", + visible: false, + }, + }, + } + + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(table.schema), table) + }) + expect(result).toEqual(["_id", "name"]) + }) + + it("includes nested relationship fields", async () => { + const aux1: Table = await config.api.table.save({ + ...structures.basicTable(), + name: "aux1Table", + schema: { + name: { name: "name", type: FieldType.STRING }, + }, + }) + const aux2: Table = await config.api.table.save({ + ...structures.basicTable(), + name: "aux2Table", + schema: { + title: { name: "title", type: FieldType.STRING }, + aux1: { + name: "aux1", + type: FieldType.LINK, + tableId: aux1._id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "aux2", + }, + }, + }) + + const table: Table = { + ...structures.basicTable(), + schema: { + name: { name: "name", type: FieldType.STRING }, + aux: { + name: "aux", + type: FieldType.LINK, + tableId: aux2._id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "table", + }, + }, + } + + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(table.schema), table) + }) + expect(result).toEqual([ + "_id", + "name", + // Aux primitive props + "aux.title", + "aux2Table.title", + + // Aux deep 1 primitive props + "aux.aux1.name", + "aux2Table.aux1.name", + + // Aux deep 2 primitive props + "aux.aux1Table.name", + "aux2Table.aux1Table.name", + ]) + }) }) }) From bbf7142bd79043ebc642cd05b428007d725c5dee Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 16 Aug 2024 09:36:31 +0200 Subject: [PATCH 28/30] Handle relationships properly --- .../server/src/sdk/app/rows/queryUtils.ts | 12 +- .../src/sdk/app/rows/tests/queryUtils.spec.ts | 139 ++++++++++-------- 2 files changed, 86 insertions(+), 65 deletions(-) diff --git a/packages/server/src/sdk/app/rows/queryUtils.ts b/packages/server/src/sdk/app/rows/queryUtils.ts index dba33d7694..b033c6dd17 100644 --- a/packages/server/src/sdk/app/rows/queryUtils.ts +++ b/packages/server/src/sdk/app/rows/queryUtils.ts @@ -56,10 +56,10 @@ export const getQueryableFields = async ( fields: string[], table: Table ): Promise => { - const handledTables = new Set([table._id!]) const extractTableFields = async ( table: Table, - allowedFields: string[] + allowedFields: string[], + fromTables: string[] ): Promise => { const result = [] for (const field of Object.keys(table.schema).filter( @@ -67,15 +67,15 @@ export const getQueryableFields = async ( )) { const subSchema = table.schema[field] if (subSchema.type === FieldType.LINK) { - if (handledTables.has(`${table._id}_${subSchema.tableId}`)) { + if (fromTables.includes(subSchema.tableId)) { // avoid circular loops continue } - handledTables.add(`${table._id}_${subSchema.tableId}`) const relatedTable = await sdk.tables.getTable(subSchema.tableId) const relatedFields = await extractTableFields( relatedTable, - Object.keys(relatedTable.schema) + Object.keys(relatedTable.schema), + [...fromTables, subSchema.tableId] ) result.push( @@ -96,7 +96,7 @@ export const getQueryableFields = async ( "_id", // Querying by _id is always allowed, even if it's never part of the schema ] - result.push(...(await extractTableFields(table, fields))) + result.push(...(await extractTableFields(table, fields, [table._id!]))) return result } diff --git a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts index e422ef75a7..57ca93e578 100644 --- a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts @@ -195,26 +195,26 @@ describe("query utils", () => { }) it("returns table schema fields and _id", async () => { - const table: Table = { + const table: Table = await config.api.table.save({ ...structures.basicTable(), schema: { name: { name: "name", type: FieldType.STRING }, age: { name: "age", type: FieldType.NUMBER }, }, - } + }) const result = await getQueryableFields(Object.keys(table.schema), table) expect(result).toEqual(["_id", "name", "age"]) }) it("excludes hidden fields", async () => { - const table: Table = { + const table: Table = await config.api.table.save({ ...structures.basicTable(), schema: { name: { name: "name", type: FieldType.STRING }, age: { name: "age", type: FieldType.NUMBER, visible: false }, }, - } + }) const result = await getQueryableFields(Object.keys(table.schema), table) expect(result).toEqual(["_id", "name"]) @@ -230,7 +230,7 @@ describe("query utils", () => { }, }) - const table: Table = { + const table: Table = await config.api.table.save({ ...structures.basicTable(), schema: { name: { name: "name", type: FieldType.STRING }, @@ -242,7 +242,7 @@ describe("query utils", () => { fieldName: "table", }, }, - } + }) const result = await config.doInContext(config.appId, () => { return getQueryableFields(Object.keys(table.schema), table) @@ -267,7 +267,7 @@ describe("query utils", () => { }, }) - const table: Table = { + const table: Table = await config.api.table.save({ ...structures.basicTable(), schema: { name: { name: "name", type: FieldType.STRING }, @@ -279,7 +279,7 @@ describe("query utils", () => { fieldName: "table", }, }, - } + }) const result = await config.doInContext(config.appId, () => { return getQueryableFields(Object.keys(table.schema), table) @@ -297,7 +297,7 @@ describe("query utils", () => { }, }) - const table: Table = { + const table: Table = await config.api.table.save({ ...structures.basicTable(), schema: { name: { name: "name", type: FieldType.STRING }, @@ -310,7 +310,7 @@ describe("query utils", () => { visible: false, }, }, - } + }) const result = await config.doInContext(config.appId, () => { return getQueryableFields(Object.keys(table.schema), table) @@ -318,61 +318,82 @@ describe("query utils", () => { expect(result).toEqual(["_id", "name"]) }) - it("includes nested relationship fields", async () => { - const aux1: Table = await config.api.table.save({ - ...structures.basicTable(), - name: "aux1Table", - schema: { - name: { name: "name", type: FieldType.STRING }, - }, - }) - const aux2: Table = await config.api.table.save({ - ...structures.basicTable(), - name: "aux2Table", - schema: { - title: { name: "title", type: FieldType.STRING }, - aux1: { - name: "aux1", - type: FieldType.LINK, - tableId: aux1._id!, - relationshipType: RelationshipType.ONE_TO_MANY, - fieldName: "aux2", + describe("nested relationship", () => { + let table: Table, aux1: Table, aux2: Table + + beforeAll(async () => { + aux1 = await config.api.table.save({ + ...structures.basicTable(), + name: "aux1Table", + schema: { + name: { name: "name", type: FieldType.STRING }, }, - }, - }) - - const table: Table = { - ...structures.basicTable(), - schema: { - name: { name: "name", type: FieldType.STRING }, - aux: { - name: "aux", - type: FieldType.LINK, - tableId: aux2._id!, - relationshipType: RelationshipType.ONE_TO_MANY, - fieldName: "table", + }) + aux2 = await config.api.table.save({ + ...structures.basicTable(), + name: "aux2Table", + schema: { + title: { name: "title", type: FieldType.STRING }, + aux1_1: { + name: "aux1_1", + type: FieldType.LINK, + tableId: aux1._id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "aux2_1", + }, }, - }, - } + }) - const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(table.schema), table) + table = await config.api.table.save({ + ...structures.basicTable(), + schema: { + name: { name: "name", type: FieldType.STRING }, + aux1: { + name: "aux1", + type: FieldType.LINK, + tableId: aux1._id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "table", + }, + aux2: { + name: "aux2", + type: FieldType.LINK, + tableId: aux2._id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "table", + }, + }, + }) }) - expect(result).toEqual([ - "_id", - "name", - // Aux primitive props - "aux.title", - "aux2Table.title", - // Aux deep 1 primitive props - "aux.aux1.name", - "aux2Table.aux1.name", + it("includes nested relationship fields from main table", async () => { + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(table.schema), table) + }) + expect(result).toEqual([ + "_id", + "name", + // deep 1 aux1 primitive props + "aux1.name", + "aux1Table.name", - // Aux deep 2 primitive props - "aux.aux1Table.name", - "aux2Table.aux1Table.name", - ]) + // deep 2 aux1 primitive props + "aux1.aux2_1.title", + "aux1Table.aux2_1.title", + "aux1.aux2Table.title", + "aux1Table.aux2Table.title", + + // deep 1 aux2 primitive props + "aux2.title", + "aux2Table.title", + + // deep 2 aux2 primitive props + "aux2.aux1_1.name", + "aux2Table.aux1_1.name", + "aux2.aux1Table.name", + "aux2Table.aux1Table.name", + ]) + }) }) }) }) From 27c618d263a8f416cac281f64fe1a0c81d7186f8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 16 Aug 2024 09:57:13 +0200 Subject: [PATCH 29/30] Extra tests --- .../src/sdk/app/rows/tests/queryUtils.spec.ts | 114 +++++++++++++++++- 1 file changed, 108 insertions(+), 6 deletions(-) diff --git a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts index 57ca93e578..cdc29b8b32 100644 --- a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts @@ -322,14 +322,14 @@ describe("query utils", () => { let table: Table, aux1: Table, aux2: Table beforeAll(async () => { - aux1 = await config.api.table.save({ + const { _id: aux1Id } = await config.api.table.save({ ...structures.basicTable(), name: "aux1Table", schema: { name: { name: "name", type: FieldType.STRING }, }, }) - aux2 = await config.api.table.save({ + const { _id: aux2Id } = await config.api.table.save({ ...structures.basicTable(), name: "aux2Table", schema: { @@ -337,33 +337,45 @@ describe("query utils", () => { aux1_1: { name: "aux1_1", type: FieldType.LINK, - tableId: aux1._id!, + tableId: aux1Id!, relationshipType: RelationshipType.ONE_TO_MANY, fieldName: "aux2_1", }, + aux1_2: { + name: "aux1_2", + type: FieldType.LINK, + tableId: aux1Id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "aux2_2", + }, }, }) - table = await config.api.table.save({ + const { _id: tableId } = await config.api.table.save({ ...structures.basicTable(), schema: { name: { name: "name", type: FieldType.STRING }, aux1: { name: "aux1", type: FieldType.LINK, - tableId: aux1._id!, + tableId: aux1Id!, relationshipType: RelationshipType.ONE_TO_MANY, fieldName: "table", }, aux2: { name: "aux2", type: FieldType.LINK, - tableId: aux2._id!, + tableId: aux2Id!, relationshipType: RelationshipType.ONE_TO_MANY, fieldName: "table", }, }, }) + + // We need to refech them to get the updated foreign keys + aux1 = await config.api.table.get(aux1Id!) + aux2 = await config.api.table.get(aux2Id!) + table = await config.api.table.get(tableId!) }) it("includes nested relationship fields from main table", async () => { @@ -383,6 +395,12 @@ describe("query utils", () => { "aux1.aux2Table.title", "aux1Table.aux2Table.title", + // deep 2 aux2 primitive props + "aux1.aux2_2.title", + "aux1Table.aux2_2.title", + "aux1.aux2Table.title", + "aux1Table.aux2Table.title", + // deep 1 aux2 primitive props "aux2.title", "aux2Table.title", @@ -392,6 +410,90 @@ describe("query utils", () => { "aux2Table.aux1_1.name", "aux2.aux1Table.name", "aux2Table.aux1Table.name", + "aux2.aux1_2.name", + "aux2Table.aux1_2.name", + "aux2.aux1Table.name", + "aux2Table.aux1Table.name", + ]) + }) + + it("includes nested relationship fields from aux 1 table", async () => { + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(aux1.schema), aux1) + }) + expect(result).toEqual([ + "_id", + "name", + + // deep 1 aux2_1 primitive props + "aux2_1.title", + "aux2Table.title", + + // deep 2 aux2_1 primitive props + "aux2_1.table.name", + "aux2Table.table.name", + "aux2_1.TestTable.name", + "aux2Table.TestTable.name", + + // deep 1 aux2_2 primitive props + "aux2_2.title", + "aux2Table.title", + + // deep 2 aux2_2 primitive props + "aux2_2.table.name", + "aux2Table.table.name", + "aux2_2.TestTable.name", + "aux2Table.TestTable.name", + + // deep 1 table primitive props + "table.name", + "TestTable.name", + + // deep 2 table primitive props + "table.aux2.title", + "TestTable.aux2.title", + "table.aux2Table.title", + "TestTable.aux2Table.title", + ]) + }) + + it("includes nested relationship fields from aux 2 table", async () => { + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(aux2.schema), aux2) + }) + expect(result).toEqual([ + "_id", + "title", + + // deep 1 aux1_1 primitive props + "aux1_1.name", + "aux1Table.name", + + // deep 2 aux1_1 primitive props + "aux1_1.table.name", + "aux1Table.table.name", + "aux1_1.TestTable.name", + "aux1Table.TestTable.name", + + // deep 1 aux1_2 primitive props + "aux1_2.name", + "aux1Table.name", + + // deep 2 aux1_2 primitive props + "aux1_2.table.name", + "aux1Table.table.name", + "aux1_2.TestTable.name", + "aux1Table.TestTable.name", + + // deep 1 table primitive props + "table.name", + "TestTable.name", + + // deep 2 table primitive props + "table.aux1.name", + "TestTable.aux1.name", + "table.aux1Table.name", + "TestTable.aux1Table.name", ]) }) }) From 07fe8c6c1fafeb7c6eeed79abad8fab80d07a65e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 16 Aug 2024 10:03:15 +0200 Subject: [PATCH 30/30] Many to many tests --- .../src/sdk/app/rows/tests/queryUtils.spec.ts | 382 ++++++++++-------- 1 file changed, 222 insertions(+), 160 deletions(-) diff --git a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts index cdc29b8b32..c156b2d5ba 100644 --- a/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts @@ -319,182 +319,244 @@ describe("query utils", () => { }) describe("nested relationship", () => { - let table: Table, aux1: Table, aux2: Table + describe("one-to-many", () => { + let table: Table, aux1: Table, aux2: Table - beforeAll(async () => { - const { _id: aux1Id } = await config.api.table.save({ - ...structures.basicTable(), - name: "aux1Table", - schema: { - name: { name: "name", type: FieldType.STRING }, - }, - }) - const { _id: aux2Id } = await config.api.table.save({ - ...structures.basicTable(), - name: "aux2Table", - schema: { - title: { name: "title", type: FieldType.STRING }, - aux1_1: { - name: "aux1_1", - type: FieldType.LINK, - tableId: aux1Id!, - relationshipType: RelationshipType.ONE_TO_MANY, - fieldName: "aux2_1", + beforeAll(async () => { + const { _id: aux1Id } = await config.api.table.save({ + ...structures.basicTable(), + name: "aux1Table", + schema: { + name: { name: "name", type: FieldType.STRING }, }, - aux1_2: { - name: "aux1_2", - type: FieldType.LINK, - tableId: aux1Id!, - relationshipType: RelationshipType.ONE_TO_MANY, - fieldName: "aux2_2", + }) + const { _id: aux2Id } = await config.api.table.save({ + ...structures.basicTable(), + name: "aux2Table", + schema: { + title: { name: "title", type: FieldType.STRING }, + aux1_1: { + name: "aux1_1", + type: FieldType.LINK, + tableId: aux1Id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "aux2_1", + }, + aux1_2: { + name: "aux1_2", + type: FieldType.LINK, + tableId: aux1Id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "aux2_2", + }, }, - }, + }) + + const { _id: tableId } = await config.api.table.save({ + ...structures.basicTable(), + schema: { + name: { name: "name", type: FieldType.STRING }, + aux1: { + name: "aux1", + type: FieldType.LINK, + tableId: aux1Id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "table", + }, + aux2: { + name: "aux2", + type: FieldType.LINK, + tableId: aux2Id!, + relationshipType: RelationshipType.ONE_TO_MANY, + fieldName: "table", + }, + }, + }) + + // We need to refech them to get the updated foreign keys + aux1 = await config.api.table.get(aux1Id!) + aux2 = await config.api.table.get(aux2Id!) + table = await config.api.table.get(tableId!) }) - const { _id: tableId } = await config.api.table.save({ - ...structures.basicTable(), - schema: { - name: { name: "name", type: FieldType.STRING }, - aux1: { - name: "aux1", - type: FieldType.LINK, - tableId: aux1Id!, - relationshipType: RelationshipType.ONE_TO_MANY, - fieldName: "table", - }, - aux2: { - name: "aux2", - type: FieldType.LINK, - tableId: aux2Id!, - relationshipType: RelationshipType.ONE_TO_MANY, - fieldName: "table", - }, - }, + it("includes nested relationship fields from main table", async () => { + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(table.schema), table) + }) + expect(result).toEqual([ + "_id", + "name", + // deep 1 aux1 primitive props + "aux1.name", + "aux1Table.name", + + // deep 2 aux1 primitive props + "aux1.aux2_1.title", + "aux1Table.aux2_1.title", + "aux1.aux2Table.title", + "aux1Table.aux2Table.title", + + // deep 2 aux2 primitive props + "aux1.aux2_2.title", + "aux1Table.aux2_2.title", + "aux1.aux2Table.title", + "aux1Table.aux2Table.title", + + // deep 1 aux2 primitive props + "aux2.title", + "aux2Table.title", + + // deep 2 aux2 primitive props + "aux2.aux1_1.name", + "aux2Table.aux1_1.name", + "aux2.aux1Table.name", + "aux2Table.aux1Table.name", + "aux2.aux1_2.name", + "aux2Table.aux1_2.name", + "aux2.aux1Table.name", + "aux2Table.aux1Table.name", + ]) }) - // We need to refech them to get the updated foreign keys - aux1 = await config.api.table.get(aux1Id!) - aux2 = await config.api.table.get(aux2Id!) - table = await config.api.table.get(tableId!) + it("includes nested relationship fields from aux 1 table", async () => { + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(aux1.schema), aux1) + }) + expect(result).toEqual([ + "_id", + "name", + + // deep 1 aux2_1 primitive props + "aux2_1.title", + "aux2Table.title", + + // deep 2 aux2_1 primitive props + "aux2_1.table.name", + "aux2Table.table.name", + "aux2_1.TestTable.name", + "aux2Table.TestTable.name", + + // deep 1 aux2_2 primitive props + "aux2_2.title", + "aux2Table.title", + + // deep 2 aux2_2 primitive props + "aux2_2.table.name", + "aux2Table.table.name", + "aux2_2.TestTable.name", + "aux2Table.TestTable.name", + + // deep 1 table primitive props + "table.name", + "TestTable.name", + + // deep 2 table primitive props + "table.aux2.title", + "TestTable.aux2.title", + "table.aux2Table.title", + "TestTable.aux2Table.title", + ]) + }) + + it("includes nested relationship fields from aux 2 table", async () => { + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(aux2.schema), aux2) + }) + expect(result).toEqual([ + "_id", + "title", + + // deep 1 aux1_1 primitive props + "aux1_1.name", + "aux1Table.name", + + // deep 2 aux1_1 primitive props + "aux1_1.table.name", + "aux1Table.table.name", + "aux1_1.TestTable.name", + "aux1Table.TestTable.name", + + // deep 1 aux1_2 primitive props + "aux1_2.name", + "aux1Table.name", + + // deep 2 aux1_2 primitive props + "aux1_2.table.name", + "aux1Table.table.name", + "aux1_2.TestTable.name", + "aux1Table.TestTable.name", + + // deep 1 table primitive props + "table.name", + "TestTable.name", + + // deep 2 table primitive props + "table.aux1.name", + "TestTable.aux1.name", + "table.aux1Table.name", + "TestTable.aux1Table.name", + ]) + }) }) - it("includes nested relationship fields from main table", async () => { - const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(table.schema), table) + describe("many-to-many", () => { + let table: Table, aux: Table + + beforeAll(async () => { + const { _id: auxId } = await config.api.table.save({ + ...structures.basicTable(), + name: "auxTable", + schema: { + title: { name: "title", type: FieldType.STRING }, + }, + }) + + const { _id: tableId } = await config.api.table.save({ + ...structures.basicTable(), + schema: { + name: { name: "name", type: FieldType.STRING }, + aux: { + name: "aux", + type: FieldType.LINK, + tableId: auxId!, + relationshipType: RelationshipType.MANY_TO_MANY, + fieldName: "table", + }, + }, + }) + + // We need to refech them to get the updated foreign keys + aux = await config.api.table.get(auxId!) + table = await config.api.table.get(tableId!) }) - expect(result).toEqual([ - "_id", - "name", - // deep 1 aux1 primitive props - "aux1.name", - "aux1Table.name", - // deep 2 aux1 primitive props - "aux1.aux2_1.title", - "aux1Table.aux2_1.title", - "aux1.aux2Table.title", - "aux1Table.aux2Table.title", + it("includes nested relationship fields from main table", async () => { + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(table.schema), table) + }) + expect(result).toEqual([ + "_id", + "name", - // deep 2 aux2 primitive props - "aux1.aux2_2.title", - "aux1Table.aux2_2.title", - "aux1.aux2Table.title", - "aux1Table.aux2Table.title", - - // deep 1 aux2 primitive props - "aux2.title", - "aux2Table.title", - - // deep 2 aux2 primitive props - "aux2.aux1_1.name", - "aux2Table.aux1_1.name", - "aux2.aux1Table.name", - "aux2Table.aux1Table.name", - "aux2.aux1_2.name", - "aux2Table.aux1_2.name", - "aux2.aux1Table.name", - "aux2Table.aux1Table.name", - ]) - }) - - it("includes nested relationship fields from aux 1 table", async () => { - const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(aux1.schema), aux1) + // deep 1 aux primitive props + "aux.title", + "auxTable.title", + ]) }) - expect(result).toEqual([ - "_id", - "name", - // deep 1 aux2_1 primitive props - "aux2_1.title", - "aux2Table.title", + it("includes nested relationship fields from aux table", async () => { + const result = await config.doInContext(config.appId, () => { + return getQueryableFields(Object.keys(aux.schema), aux) + }) + expect(result).toEqual([ + "_id", + "title", - // deep 2 aux2_1 primitive props - "aux2_1.table.name", - "aux2Table.table.name", - "aux2_1.TestTable.name", - "aux2Table.TestTable.name", - - // deep 1 aux2_2 primitive props - "aux2_2.title", - "aux2Table.title", - - // deep 2 aux2_2 primitive props - "aux2_2.table.name", - "aux2Table.table.name", - "aux2_2.TestTable.name", - "aux2Table.TestTable.name", - - // deep 1 table primitive props - "table.name", - "TestTable.name", - - // deep 2 table primitive props - "table.aux2.title", - "TestTable.aux2.title", - "table.aux2Table.title", - "TestTable.aux2Table.title", - ]) - }) - - it("includes nested relationship fields from aux 2 table", async () => { - const result = await config.doInContext(config.appId, () => { - return getQueryableFields(Object.keys(aux2.schema), aux2) + // deep 1 dependency primitive props + "table.name", + "TestTable.name", + ]) }) - expect(result).toEqual([ - "_id", - "title", - - // deep 1 aux1_1 primitive props - "aux1_1.name", - "aux1Table.name", - - // deep 2 aux1_1 primitive props - "aux1_1.table.name", - "aux1Table.table.name", - "aux1_1.TestTable.name", - "aux1Table.TestTable.name", - - // deep 1 aux1_2 primitive props - "aux1_2.name", - "aux1Table.name", - - // deep 2 aux1_2 primitive props - "aux1_2.table.name", - "aux1Table.table.name", - "aux1_2.TestTable.name", - "aux1Table.TestTable.name", - - // deep 1 table primitive props - "table.name", - "TestTable.name", - - // deep 2 table primitive props - "table.aux1.name", - "TestTable.aux1.name", - "table.aux1Table.name", - "TestTable.aux1Table.name", - ]) }) }) })