From 33453646758ddba07a6d2b6a53f24a70bbb5055c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 20 Jun 2024 14:36:08 +0100 Subject: [PATCH 01/27] Updating test case - not exactly sure what it was testing before, but now it definitely confirms paginated results are stable. --- .../src/api/routes/tests/search.spec.ts | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index be66253090..c5c18997e0 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1548,19 +1548,32 @@ describe.each([ // be stable or pagination will break. We don't want the user to need // to specify an order for pagination to work. it("is stable without a sort specified", async () => { - let { rows } = await config.api.row.search(table._id!, { - tableId: table._id!, - query: {}, - }) + let { rows: fullRowList } = await config.api.row.search( + table._id!, + { + tableId: table._id!, + query: {}, + } + ) - for (let i = 0; i < 10; i++) { + // repeat the search many times to check the first row is always the same + let bookmark: string | number | undefined, + hasNextPage: boolean | undefined = true, + rowCount: number = 0 + do { const response = await config.api.row.search(table._id!, { tableId: table._id!, limit: 1, + paginate: true, query: {}, + bookmark, }) - expect(response.rows).toEqual(rows) - } + bookmark = response.bookmark + hasNextPage = response.hasNextPage + expect(response.rows.length).toEqual(1) + const foundRow = response.rows[0] + expect(foundRow).toEqual(fullRowList[rowCount++]) + } while (hasNextPage) }) }) From 093b06ed7d4dc824a9ea6711cda89fe2bd452f3f Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 20 Jun 2024 14:51:25 +0100 Subject: [PATCH 02/27] updating account portal SHA --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 247f56d455..b2d217f824 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 247f56d455abbd64da17d865275ed978f577549f +Subproject commit b2d217f8246c4b8597ae0bfc5ac1f578e4d2aef5 From b6b05e08b182fefc36c2ff3da533c61fe930a47d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 20 Jun 2024 15:52:56 +0100 Subject: [PATCH 03/27] Removing SQS from view test to check. --- .../src/api/routes/tests/viewV2.spec.ts | 1 - yarn.lock | 34 ++++++++----------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 99ff4f8db7..4efd20e66b 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -28,7 +28,6 @@ import { db, roles } from "@budibase/backend-core" describe.each([ ["lucene", undefined], - ["sqs", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], diff --git a/yarn.lock b/yarn.lock index 3606f068b1..9914c334df 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10296,7 +10296,7 @@ engine.io-parser@~5.0.3: resolved "https://registry.yarnpkg.com/engine.io-parser/-/engine.io-parser-5.0.6.tgz#7811244af173e157295dec9b2718dfe42a64ef45" integrity sha512-tjuoZDMAdEhVnSFleYPCtdL2GXwVTGtNjoeJd9IhIG3C1xs9uwxqRNEu5WpnDZCaozwVlK/nuQhpodhXSIMaxw== -engine.io@~6.4.1: +engine.io@~6.4.2: version "6.4.2" resolved "https://registry.yarnpkg.com/engine.io/-/engine.io-6.4.2.tgz#ffeaf68f69b1364b0286badddf15ff633476473f" integrity sha512-FKn/3oMiJjrOEOeUub2WCox6JhxBXq/Zn3fZOMCBxKnNYtsdKjxhl7yR3fZhM9PV+rdE75SU5SYMc+2PGzo+Tg== @@ -20160,17 +20160,25 @@ socket.io-parser@~4.2.1: "@socket.io/component-emitter" "~3.1.0" debug "~4.3.1" -socket.io@4.6.1: - version "4.6.1" - resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-4.6.1.tgz#62ec117e5fce0692fa50498da9347cfb52c3bc70" - integrity sha512-KMcaAi4l/8+xEjkRICl6ak8ySoxsYG+gG6/XfRCPJPQ/haCRIJBTL4wIl8YCsmtaBovcAXGLOShyVWQ/FG8GZA== +socket.io-parser@~4.2.4: + version "4.2.4" + resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.2.4.tgz#c806966cf7270601e47469ddeec30fbdfda44c83" + integrity sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew== + dependencies: + "@socket.io/component-emitter" "~3.1.0" + debug "~4.3.1" + +socket.io@4.6.2: + version "4.6.2" + resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-4.6.2.tgz#d597db077d4df9cbbdfaa7a9ed8ccc3d49439786" + integrity sha512-Vp+lSks5k0dewYTfwgPT9UeGGd+ht7sCpB7p0e83VgO4X/AHYWhXITMrNk/pg8syY2bpx23ptClCQuHhqi2BgQ== dependencies: accepts "~1.3.4" base64id "~2.0.0" debug "~4.3.2" - engine.io "~6.4.1" + engine.io "~6.4.2" socket.io-adapter "~2.5.2" - socket.io-parser "~4.2.1" + socket.io-parser "~4.2.4" socks-proxy-agent@^7.0.0: version "7.0.0" @@ -21102,18 +21110,6 @@ tar@6.1.11: mkdirp "^1.0.3" yallist "^4.0.0" -tar@6.1.15: - version "6.1.15" - resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.15.tgz#c9738b0b98845a3b344d334b8fa3041aaba53a69" - integrity sha512-/zKt9UyngnxIT/EAGYuxaMYgOIJiP81ab9ZfkILq4oNLPFX50qyYmu7jRj9qeXoxmJHjGlbH0+cm2uy1WCs10A== - dependencies: - chownr "^2.0.0" - fs-minipass "^2.0.0" - minipass "^5.0.0" - minizlib "^2.1.1" - mkdirp "^1.0.3" - yallist "^4.0.0" - tar@6.2.1, tar@^6.1.11, tar@^6.1.2: version "6.2.1" resolved "https://registry.yarnpkg.com/tar/-/tar-6.2.1.tgz#717549c541bc3c2af15751bea94b1dd068d4b03a" From 66ef0cb79afe0c28662bebd62dad36f9a3a3e6ff Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 20 Jun 2024 16:05:03 +0100 Subject: [PATCH 04/27] Adding back SQS - wasn't causing a problem. --- packages/server/src/api/routes/tests/viewV2.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 4efd20e66b..99ff4f8db7 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -28,6 +28,7 @@ import { db, roles } from "@budibase/backend-core" describe.each([ ["lucene", undefined], + ["sqs", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], From 295961edb15c1811060bb86972aeb789a49cdf5b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 20 Jun 2024 16:18:32 +0100 Subject: [PATCH 05/27] Attempting without promise.all in external. --- packages/server/src/sdk/app/rows/search/external.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 9fc3487f62..bfb1ccc442 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -81,15 +81,11 @@ export async function search( paginate: paginateObj as PaginationJson, includeSqlRelationships: IncludeRelationship.INCLUDE, } - const queries: Promise[] = [] - queries.push(handleRequest(Operation.READ, tableId, parameters)) + let rows = await handleRequest(Operation.READ, tableId, parameters) + let totalRows: number | undefined if (countRows) { - queries.push(handleRequest(Operation.COUNT, tableId, parameters)) + totalRows = await handleRequest(Operation.COUNT, tableId, parameters) } - const responses = await Promise.all(queries) - let rows = responses[0] as Row[] - const totalRows = - responses.length > 1 ? (responses[1] as number) : undefined let hasNextPage = false // remove the extra row if it's there From df56371ab6e589ad9310f99ea792ede2368ef8e1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 20 Jun 2024 16:36:18 +0100 Subject: [PATCH 06/27] Reverting change to promises. --- packages/server/src/sdk/app/rows/search/external.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index bfb1ccc442..9fc3487f62 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -81,11 +81,15 @@ export async function search( paginate: paginateObj as PaginationJson, includeSqlRelationships: IncludeRelationship.INCLUDE, } - let rows = await handleRequest(Operation.READ, tableId, parameters) - let totalRows: number | undefined + const queries: Promise[] = [] + queries.push(handleRequest(Operation.READ, tableId, parameters)) if (countRows) { - totalRows = await handleRequest(Operation.COUNT, tableId, parameters) + queries.push(handleRequest(Operation.COUNT, tableId, parameters)) } + const responses = await Promise.all(queries) + let rows = responses[0] as Row[] + const totalRows = + responses.length > 1 ? (responses[1] as number) : undefined let hasNextPage = false // remove the extra row if it's there From 4ac74b1e9a76dc46dfb0abbee80cdc74f0bed9ac Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 20 Jun 2024 17:08:03 +0100 Subject: [PATCH 07/27] bump account portal --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index b2d217f824..b41b427134 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit b2d217f8246c4b8597ae0bfc5ac1f578e4d2aef5 +Subproject commit b41b4271348ff0b526bfc1a9c41dff1169f7bfd9 From 86bae92ada951adb57f417628a3486d0219f353a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 20 Jun 2024 17:13:42 +0100 Subject: [PATCH 08/27] Refactoring search test to make it easier to find promises which aren't handled. --- .../src/api/routes/tests/search.spec.ts | 997 ++++++++++-------- 1 file changed, 584 insertions(+), 413 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index c5c18997e0..cff966ab89 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -274,55 +274,63 @@ describe.each([ }) describe("equal", () => { - it("successfully finds true row", () => - expectQuery({ equal: { isTrue: true } }).toMatchExactly([ + it("successfully finds true row", async () => { + await expectQuery({ equal: { isTrue: true } }).toMatchExactly([ { isTrue: true }, - ])) + ]) + }) - it("successfully finds false row", () => - expectQuery({ equal: { isTrue: false } }).toMatchExactly([ + it("successfully finds false row", async () => { + await expectQuery({ equal: { isTrue: false } }).toMatchExactly([ { isTrue: false }, - ])) + ]) + }) }) describe("notEqual", () => { - it("successfully finds false row", () => - expectQuery({ notEqual: { isTrue: true } }).toContainExactly([ + it("successfully finds false row", async () => { + await expectQuery({ notEqual: { isTrue: true } }).toContainExactly([ { isTrue: false }, - ])) + ]) + }) - it("successfully finds true row", () => - expectQuery({ notEqual: { isTrue: false } }).toContainExactly([ + it("successfully finds true row", async () => { + await expectQuery({ notEqual: { isTrue: false } }).toContainExactly([ { isTrue: true }, - ])) + ]) + }) }) describe("oneOf", () => { - it("successfully finds true row", () => - expectQuery({ oneOf: { isTrue: [true] } }).toContainExactly([ + it("successfully finds true row", async () => { + await expectQuery({ oneOf: { isTrue: [true] } }).toContainExactly([ { isTrue: true }, - ])) + ]) + }) - it("successfully finds false row", () => - expectQuery({ oneOf: { isTrue: [false] } }).toContainExactly([ + it("successfully finds false row", async () => { + await expectQuery({ oneOf: { isTrue: [false] } }).toContainExactly([ { isTrue: false }, - ])) + ]) + }) }) describe("sort", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "isTrue", sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ isTrue: false }, { isTrue: true }])) + }).toMatchExactly([{ isTrue: false }, { isTrue: true }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "isTrue", sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ isTrue: true }, { isTrue: false }])) + }).toMatchExactly([{ isTrue: true }, { isTrue: false }]) + }) }) }) @@ -676,191 +684,230 @@ describe.each([ }) describe("misc", () => { - it("should return all if no query is passed", () => - expectSearch({} as RowSearchParams).toContainExactly([ + it("should return all if no query is passed", async () => { + await expectSearch({} as RowSearchParams).toContainExactly([ { name: "foo" }, { name: "bar" }, - ])) + ]) + }) - it("should return all if empty query is passed", () => - expectQuery({}).toContainExactly([{ name: "foo" }, { name: "bar" }])) + it("should return all if empty query is passed", async () => { + await expectQuery({}).toContainExactly([ + { name: "foo" }, + { name: "bar" }, + ]) + }) - it("should return all if onEmptyFilter is RETURN_ALL", () => - expectQuery({ + it("should return all if onEmptyFilter is RETURN_ALL", async () => { + await expectQuery({ onEmptyFilter: EmptyFilterOption.RETURN_ALL, - }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) - it("should return nothing if onEmptyFilter is RETURN_NONE", () => - expectQuery({ + it("should return nothing if onEmptyFilter is RETURN_NONE", async () => { + await expectQuery({ onEmptyFilter: EmptyFilterOption.RETURN_NONE, - }).toFindNothing()) + }).toFindNothing() + }) - it("should respect limit", () => - expectSearch({ limit: 1, paginate: true, query: {} }).toHaveLength(1)) + it("should respect limit", async () => { + await expectSearch({ + limit: 1, + paginate: true, + query: {}, + }).toHaveLength(1) + }) }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { name: "foo" } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ equal: { name: "foo" } }).toContainExactly([ { name: "foo" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { name: "none" } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { name: "none" } }).toFindNothing() + }) - it("works as an or condition", () => - expectQuery({ + it("works as an or condition", async () => { + await expectQuery({ allOr: true, equal: { name: "foo" }, oneOf: { name: ["bar"] }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) - it("can have multiple values for same column", () => - expectQuery({ + it("can have multiple values for same column", async () => { + await expectQuery({ allOr: true, equal: { "1:name": "foo", "2:name": "bar" }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) }) describe("notEqual", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { name: "foo" } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { name: "foo" } }).toContainExactly([ { name: "bar" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notEqual: { name: "bar" } }).toContainExactly([ + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { name: "bar" } }).toContainExactly([ { name: "foo" }, - ])) + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { name: ["foo"] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { name: ["foo"] } }).toContainExactly([ { name: "foo" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { name: ["none"] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { name: ["none"] } }).toFindNothing() + }) }) describe("fuzzy", () => { - it("successfully finds a row", () => - expectQuery({ fuzzy: { name: "oo" } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ fuzzy: { name: "oo" } }).toContainExactly([ { name: "foo" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ fuzzy: { name: "none" } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ fuzzy: { name: "none" } }).toFindNothing() + }) }) describe("string", () => { - it("successfully finds a row", () => - expectQuery({ string: { name: "fo" } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ string: { name: "fo" } }).toContainExactly([ { name: "foo" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ string: { name: "none" } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ string: { name: "none" } }).toFindNothing() + }) - it("is case-insensitive", () => - expectQuery({ string: { name: "FO" } }).toContainExactly([ + it("is case-insensitive", async () => { + await expectQuery({ string: { name: "FO" } }).toContainExactly([ { name: "foo" }, - ])) + ]) + }) }) describe("range", () => { - it("successfully finds multiple rows", () => - expectQuery({ + it("successfully finds multiple rows", async () => { + await expectQuery({ range: { name: { low: "a", high: "z" } }, - }).toContainExactly([{ name: "bar" }, { name: "foo" }])) + }).toContainExactly([{ name: "bar" }, { name: "foo" }]) + }) - it("successfully finds a row with a high bound", () => - expectQuery({ + it("successfully finds a row with a high bound", async () => { + await expectQuery({ range: { name: { low: "a", high: "c" } }, - }).toContainExactly([{ name: "bar" }])) + }).toContainExactly([{ name: "bar" }]) + }) - it("successfully finds a row with a low bound", () => - expectQuery({ + it("successfully finds a row with a low bound", async () => { + await expectQuery({ range: { name: { low: "f", high: "z" } }, - }).toContainExactly([{ name: "foo" }])) + }).toContainExactly([{ name: "foo" }]) + }) - it("successfully finds no rows", () => - expectQuery({ + it("successfully finds no rows", async () => { + await expectQuery({ range: { name: { low: "g", high: "h" } }, - }).toFindNothing()) + }).toFindNothing() + }) !isLucene && - it("ignores low if it's an empty object", () => - expectQuery({ + it("ignores low if it's an empty object", async () => { + await expectQuery({ // @ts-ignore range: { name: { low: {}, high: "z" } }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) !isLucene && - it("ignores high if it's an empty object", () => - expectQuery({ + it("ignores high if it's an empty object", async () => { + await expectQuery({ // @ts-ignore range: { name: { low: "a", high: {} } }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) }) describe("empty", () => { - it("finds no empty rows", () => - expectQuery({ empty: { name: null } }).toFindNothing()) + it("finds no empty rows", async () => { + await expectQuery({ empty: { name: null } }).toFindNothing() + }) - it("should not be affected by when filter empty behaviour", () => - expectQuery({ + it("should not be affected by when filter empty behaviour", async () => { + await expectQuery({ empty: { name: null }, onEmptyFilter: EmptyFilterOption.RETURN_ALL, - }).toFindNothing()) + }).toFindNothing() + }) }) describe("notEmpty", () => { - it("finds all non-empty rows", () => - expectQuery({ notEmpty: { name: null } }).toContainExactly([ + it("finds all non-empty rows", async () => { + await expectQuery({ notEmpty: { name: null } }).toContainExactly([ { name: "foo" }, { name: "bar" }, - ])) + ]) + }) - it("should not be affected by when filter empty behaviour", () => - expectQuery({ + it("should not be affected by when filter empty behaviour", async () => { + await expectQuery({ notEmpty: { name: null }, onEmptyFilter: EmptyFilterOption.RETURN_NONE, - }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) }) describe("sort", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "name", sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ name: "bar" }, { name: "foo" }])) + }).toMatchExactly([{ name: "bar" }, { name: "foo" }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "name", sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ name: "foo" }, { name: "bar" }])) + }).toMatchExactly([{ name: "foo" }, { name: "bar" }]) + }) describe("sortType STRING", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "name", sortType: SortType.STRING, sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ name: "bar" }, { name: "foo" }])) + }).toMatchExactly([{ name: "bar" }, { name: "foo" }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "name", sortType: SortType.STRING, sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ name: "foo" }, { name: "bar" }])) + }).toMatchExactly([{ name: "foo" }, { name: "bar" }]) + }) }) }) }) @@ -874,97 +921,119 @@ describe.each([ }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { age: 1 } }).toContainExactly([{ age: 1 }])) + it("successfully finds a row", async () => { + await expectQuery({ equal: { age: 1 } }).toContainExactly([{ age: 1 }]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { age: 2 } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { age: 2 } }).toFindNothing() + }) }) describe("notEqual", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { age: 1 } }).toContainExactly([{ age: 10 }])) + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { age: 1 } }).toContainExactly([ + { age: 10 }, + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notEqual: { age: 10 } }).toContainExactly([{ age: 1 }])) + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { age: 10 } }).toContainExactly([ + { age: 1 }, + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { age: [1] } }).toContainExactly([{ age: 1 }])) + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { age: [1] } }).toContainExactly([ + { age: 1 }, + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { age: [2] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { age: [2] } }).toFindNothing() + }) }) describe("range", () => { - it("successfully finds a row", () => - expectQuery({ + it("successfully finds a row", async () => { + await expectQuery({ range: { age: { low: 1, high: 5 } }, - }).toContainExactly([{ age: 1 }])) + }).toContainExactly([{ age: 1 }]) + }) - it("successfully finds multiple rows", () => - expectQuery({ + it("successfully finds multiple rows", async () => { + await expectQuery({ range: { age: { low: 1, high: 10 } }, - }).toContainExactly([{ age: 1 }, { age: 10 }])) + }).toContainExactly([{ age: 1 }, { age: 10 }]) + }) - it("successfully finds a row with a high bound", () => - expectQuery({ + it("successfully finds a row with a high bound", async () => { + await expectQuery({ range: { age: { low: 5, high: 10 } }, - }).toContainExactly([{ age: 10 }])) + }).toContainExactly([{ age: 10 }]) + }) - it("successfully finds no rows", () => - expectQuery({ + it("successfully finds no rows", async () => { + await expectQuery({ range: { age: { low: 5, high: 9 } }, - }).toFindNothing()) + }).toFindNothing() + }) // We never implemented half-open ranges in Lucene. !isLucene && - it("can search using just a low value", () => - expectQuery({ + it("can search using just a low value", async () => { + await expectQuery({ range: { age: { low: 5 } }, - }).toContainExactly([{ age: 10 }])) + }).toContainExactly([{ age: 10 }]) + }) // We never implemented half-open ranges in Lucene. !isLucene && - it("can search using just a high value", () => - expectQuery({ + it("can search using just a high value", async () => { + await expectQuery({ range: { age: { high: 5 } }, - }).toContainExactly([{ age: 1 }])) + }).toContainExactly([{ age: 1 }]) + }) }) describe("sort", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "age", sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ age: 1 }, { age: 10 }])) + }).toMatchExactly([{ age: 1 }, { age: 10 }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "age", sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ age: 10 }, { age: 1 }])) + }).toMatchExactly([{ age: 10 }, { age: 1 }]) + }) }) describe("sortType NUMBER", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "age", sortType: SortType.NUMBER, sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ age: 1 }, { age: 10 }])) + }).toMatchExactly([{ age: 1 }, { age: 10 }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "age", sortType: SortType.NUMBER, sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ age: 10 }, { age: 1 }])) + }).toMatchExactly([{ age: 10 }, { age: 1 }]) + }) }) }) @@ -984,104 +1053,120 @@ describe.each([ }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { dob: JAN_1ST } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ equal: { dob: JAN_1ST } }).toContainExactly([ { dob: JAN_1ST }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { dob: JAN_2ND } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { dob: JAN_2ND } }).toFindNothing() + }) }) describe("notEqual", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { dob: JAN_1ST } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { dob: JAN_1ST } }).toContainExactly([ { dob: JAN_10TH }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notEqual: { dob: JAN_10TH } }).toContainExactly([ + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { dob: JAN_10TH } }).toContainExactly([ { dob: JAN_1ST }, - ])) + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { dob: [JAN_1ST] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { dob: [JAN_1ST] } }).toContainExactly([ { dob: JAN_1ST }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { dob: [JAN_2ND] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { dob: [JAN_2ND] } }).toFindNothing() + }) }) describe("range", () => { - it("successfully finds a row", () => - expectQuery({ + it("successfully finds a row", async () => { + await expectQuery({ range: { dob: { low: JAN_1ST, high: JAN_5TH } }, - }).toContainExactly([{ dob: JAN_1ST }])) + }).toContainExactly([{ dob: JAN_1ST }]) + }) - it("successfully finds multiple rows", () => - expectQuery({ + it("successfully finds multiple rows", async () => { + await expectQuery({ range: { dob: { low: JAN_1ST, high: JAN_10TH } }, - }).toContainExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }])) + }).toContainExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }]) + }) - it("successfully finds a row with a high bound", () => - expectQuery({ + it("successfully finds a row with a high bound", async () => { + await expectQuery({ range: { dob: { low: JAN_5TH, high: JAN_10TH } }, - }).toContainExactly([{ dob: JAN_10TH }])) + }).toContainExactly([{ dob: JAN_10TH }]) + }) - it("successfully finds no rows", () => - expectQuery({ + it("successfully finds no rows", async () => { + await expectQuery({ range: { dob: { low: JAN_5TH, high: JAN_9TH } }, - }).toFindNothing()) + }).toFindNothing() + }) // We never implemented half-open ranges in Lucene. !isLucene && - it("can search using just a low value", () => - expectQuery({ + it("can search using just a low value", async () => { + await expectQuery({ range: { dob: { low: JAN_5TH } }, - }).toContainExactly([{ dob: JAN_10TH }])) + }).toContainExactly([{ dob: JAN_10TH }]) + }) // We never implemented half-open ranges in Lucene. !isLucene && - it("can search using just a high value", () => - expectQuery({ + it("can search using just a high value", async () => { + await expectQuery({ range: { dob: { high: JAN_5TH } }, - }).toContainExactly([{ dob: JAN_1ST }])) + }).toContainExactly([{ dob: JAN_1ST }]) + }) }) describe("sort", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "dob", sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }])) + }).toMatchExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "dob", sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ dob: JAN_10TH }, { dob: JAN_1ST }])) + }).toMatchExactly([{ dob: JAN_10TH }, { dob: JAN_1ST }]) + }) describe("sortType STRING", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "dob", sortType: SortType.STRING, sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }])) + }).toMatchExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "dob", sortType: SortType.STRING, sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ dob: JAN_10TH }, { dob: JAN_1ST }])) + }).toMatchExactly([{ dob: JAN_10TH }, { dob: JAN_1ST }]) + }) }) }) }) @@ -1115,72 +1200,85 @@ describe.each([ }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { time: T_1000 } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ equal: { time: T_1000 } }).toContainExactly([ { time: "10:00:00" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { time: UNEXISTING_TIME } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ + equal: { time: UNEXISTING_TIME }, + }).toFindNothing() + }) }) describe("notEqual", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { time: T_1000 } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { time: T_1000 } }).toContainExactly([ { timeid: NULL_TIME__ID }, { time: "10:45:00" }, { time: "12:00:00" }, { time: "15:30:00" }, { time: "00:00:00" }, - ])) + ]) + }) - it("return all when requesting non-existing", () => - expectQuery({ notEqual: { time: UNEXISTING_TIME } }).toContainExactly( - [ - { timeid: NULL_TIME__ID }, - { time: "10:00:00" }, - { time: "10:45:00" }, - { time: "12:00:00" }, - { time: "15:30:00" }, - { time: "00:00:00" }, - ] - )) + it("return all when requesting non-existing", async () => { + await expectQuery({ + notEqual: { time: UNEXISTING_TIME }, + }).toContainExactly([ + { timeid: NULL_TIME__ID }, + { time: "10:00:00" }, + { time: "10:45:00" }, + { time: "12:00:00" }, + { time: "15:30:00" }, + { time: "00:00:00" }, + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { time: [T_1000] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { time: [T_1000] } }).toContainExactly([ { time: "10:00:00" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { time: [UNEXISTING_TIME] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ + oneOf: { time: [UNEXISTING_TIME] }, + }).toFindNothing() + }) }) describe("range", () => { - it("successfully finds a row", () => - expectQuery({ + it("successfully finds a row", async () => { + await expectQuery({ range: { time: { low: T_1045, high: T_1045 } }, - }).toContainExactly([{ time: "10:45:00" }])) + }).toContainExactly([{ time: "10:45:00" }]) + }) - it("successfully finds multiple rows", () => - expectQuery({ + it("successfully finds multiple rows", async () => { + await expectQuery({ range: { time: { low: T_1045, high: T_1530 } }, }).toContainExactly([ { time: "10:45:00" }, { time: "12:00:00" }, { time: "15:30:00" }, - ])) + ]) + }) - it("successfully finds no rows", () => - expectQuery({ + it("successfully finds no rows", async () => { + await expectQuery({ range: { time: { low: UNEXISTING_TIME, high: UNEXISTING_TIME } }, - }).toFindNothing()) + }).toFindNothing() + }) }) describe("sort", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "time", sortOrder: SortOrder.ASCENDING, @@ -1191,10 +1289,11 @@ describe.each([ { time: "10:45:00" }, { time: "12:00:00" }, { time: "15:30:00" }, - ])) + ]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "time", sortOrder: SortOrder.DESCENDING, @@ -1205,11 +1304,12 @@ describe.each([ { time: "10:00:00" }, { time: "00:00:00" }, { timeid: NULL_TIME__ID }, - ])) + ]) + }) describe("sortType STRING", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "time", sortType: SortType.STRING, @@ -1221,10 +1321,11 @@ describe.each([ { time: "10:45:00" }, { time: "12:00:00" }, { time: "15:30:00" }, - ])) + ]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "time", sortType: SortType.STRING, @@ -1236,7 +1337,8 @@ describe.each([ { time: "10:00:00" }, { time: "00:00:00" }, { timeid: NULL_TIME__ID }, - ])) + ]) + }) }) }) }) @@ -1254,66 +1356,78 @@ describe.each([ }) describe("contains", () => { - it("successfully finds a row", () => - expectQuery({ contains: { numbers: ["one"] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ contains: { numbers: ["one"] } }).toContainExactly([ { numbers: ["one", "two"] }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ contains: { numbers: ["none"] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ contains: { numbers: ["none"] } }).toFindNothing() + }) - it("fails to find row containing all", () => - expectQuery({ + it("fails to find row containing all", async () => { + await expectQuery({ contains: { numbers: ["one", "two", "three"] }, - }).toFindNothing()) + }).toFindNothing() + }) - it("finds all with empty list", () => - expectQuery({ contains: { numbers: [] } }).toContainExactly([ + it("finds all with empty list", async () => { + await expectQuery({ contains: { numbers: [] } }).toContainExactly([ { numbers: ["one", "two"] }, { numbers: ["three"] }, - ])) + ]) + }) }) describe("notContains", () => { - it("successfully finds a row", () => - expectQuery({ notContains: { numbers: ["one"] } }).toContainExactly([ - { numbers: ["three"] }, - ])) + it("successfully finds a row", async () => { + await expectQuery({ + notContains: { numbers: ["one"] }, + }).toContainExactly([{ numbers: ["three"] }]) + }) - it("fails to find nonexistent row", () => - expectQuery({ + it("fails to find nonexistent row", async () => { + await expectQuery({ notContains: { numbers: ["one", "two", "three"] }, }).toContainExactly([ { numbers: ["one", "two"] }, { numbers: ["three"] }, - ])) + ]) + }) // Not sure if this is correct behaviour but changing it would be a // breaking change. - it("finds all with empty list", () => - expectQuery({ notContains: { numbers: [] } }).toContainExactly([ + it("finds all with empty list", async () => { + await expectQuery({ notContains: { numbers: [] } }).toContainExactly([ { numbers: ["one", "two"] }, { numbers: ["three"] }, - ])) + ]) + }) }) describe("containsAny", () => { - it("successfully finds rows", () => - expectQuery({ + it("successfully finds rows", async () => { + await expectQuery({ containsAny: { numbers: ["one", "two", "three"] }, }).toContainExactly([ { numbers: ["one", "two"] }, { numbers: ["three"] }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ containsAny: { numbers: ["none"] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ + containsAny: { numbers: ["none"] }, + }).toFindNothing() + }) - it("finds all with empty list", () => - expectQuery({ containsAny: { numbers: [] } }).toContainExactly([ + it("finds all with empty list", async () => { + await expectQuery({ containsAny: { numbers: [] } }).toContainExactly([ { numbers: ["one", "two"] }, { numbers: ["three"] }, - ])) + ]) + }) }) }) @@ -1332,48 +1446,56 @@ describe.each([ }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { num: SMALL } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ equal: { num: SMALL } }).toContainExactly([ { num: SMALL }, - ])) + ]) + }) - it("successfully finds a big value", () => - expectQuery({ equal: { num: BIG } }).toContainExactly([{ num: BIG }])) + it("successfully finds a big value", async () => { + await expectQuery({ equal: { num: BIG } }).toContainExactly([ + { num: BIG }, + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { num: "2" } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { num: "2" } }).toFindNothing() + }) }) describe("notEqual", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { num: SMALL } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { num: SMALL } }).toContainExactly([ { num: MEDIUM }, { num: BIG }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notEqual: { num: 10 } }).toContainExactly([ + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { num: 10 } }).toContainExactly([ { num: SMALL }, { num: MEDIUM }, { num: BIG }, - ])) + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { num: [SMALL] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { num: [SMALL] } }).toContainExactly([ { num: SMALL }, - ])) + ]) + }) - it("successfully finds all rows", () => - expectQuery({ oneOf: { num: [SMALL, MEDIUM, BIG] } }).toContainExactly([ - { num: SMALL }, - { num: MEDIUM }, - { num: BIG }, - ])) + it("successfully finds all rows", async () => { + await expectQuery({ + oneOf: { num: [SMALL, MEDIUM, BIG] }, + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }, { num: BIG }]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { num: [2] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { num: [2] } }).toFindNothing() + }) }) // Range searches against bigints don't seem to work at all in Lucene, and I @@ -1381,35 +1503,41 @@ describe.each([ // we've decided not to spend time on it. !isLucene && describe("range", () => { - it("successfully finds a row", () => - expectQuery({ + it("successfully finds a row", async () => { + await expectQuery({ range: { num: { low: SMALL, high: "5" } }, - }).toContainExactly([{ num: SMALL }])) + }).toContainExactly([{ num: SMALL }]) + }) - it("successfully finds multiple rows", () => - expectQuery({ + it("successfully finds multiple rows", async () => { + await expectQuery({ range: { num: { low: SMALL, high: MEDIUM } }, - }).toContainExactly([{ num: SMALL }, { num: MEDIUM }])) + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]) + }) - it("successfully finds a row with a high bound", () => - expectQuery({ + it("successfully finds a row with a high bound", async () => { + await expectQuery({ range: { num: { low: MEDIUM, high: BIG } }, - }).toContainExactly([{ num: MEDIUM }, { num: BIG }])) + }).toContainExactly([{ num: MEDIUM }, { num: BIG }]) + }) - it("successfully finds no rows", () => - expectQuery({ + it("successfully finds no rows", async () => { + await expectQuery({ range: { num: { low: "5", high: "5" } }, - }).toFindNothing()) + }).toFindNothing() + }) - it("can search using just a low value", () => - expectQuery({ + it("can search using just a low value", async () => { + await expectQuery({ range: { num: { low: MEDIUM } }, - }).toContainExactly([{ num: MEDIUM }, { num: BIG }])) + }).toContainExactly([{ num: MEDIUM }, { num: BIG }]) + }) - it("can search using just a high value", () => - expectQuery({ + it("can search using just a high value", async () => { + await expectQuery({ range: { num: { high: MEDIUM } }, - }).toContainExactly([{ num: SMALL }, { num: MEDIUM }])) + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]) + }) }) }) @@ -1428,16 +1556,20 @@ describe.each([ }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { auto: 1 } }).toContainExactly([{ auto: 1 }])) + it("successfully finds a row", async () => { + await expectQuery({ equal: { auto: 1 } }).toContainExactly([ + { auto: 1 }, + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { auto: 0 } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { auto: 0 } }).toFindNothing() + }) }) describe("not equal", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { auto: 1 } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { auto: 1 } }).toContainExactly([ { auto: 2 }, { auto: 3 }, { auto: 4 }, @@ -1447,10 +1579,11 @@ describe.each([ { auto: 8 }, { auto: 9 }, { auto: 10 }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notEqual: { auto: 0 } }).toContainExactly([ + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { auto: 0 } }).toContainExactly([ { auto: 1 }, { auto: 2 }, { auto: 3 }, @@ -1461,55 +1594,66 @@ describe.each([ { auto: 8 }, { auto: 9 }, { auto: 10 }, - ])) + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { auto: [1] } }).toContainExactly([{ auto: 1 }])) + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { auto: [1] } }).toContainExactly([ + { auto: 1 }, + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { auto: [0] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { auto: [0] } }).toFindNothing() + }) }) describe("range", () => { - it("successfully finds a row", () => - expectQuery({ + it("successfully finds a row", async () => { + await expectQuery({ range: { auto: { low: 1, high: 1 } }, - }).toContainExactly([{ auto: 1 }])) + }).toContainExactly([{ auto: 1 }]) + }) - it("successfully finds multiple rows", () => - expectQuery({ + it("successfully finds multiple rows", async () => { + await expectQuery({ range: { auto: { low: 1, high: 2 } }, - }).toContainExactly([{ auto: 1 }, { auto: 2 }])) + }).toContainExactly([{ auto: 1 }, { auto: 2 }]) + }) - it("successfully finds a row with a high bound", () => - expectQuery({ + it("successfully finds a row with a high bound", async () => { + await expectQuery({ range: { auto: { low: 2, high: 2 } }, - }).toContainExactly([{ auto: 2 }])) + }).toContainExactly([{ auto: 2 }]) + }) - it("successfully finds no rows", () => - expectQuery({ + it("successfully finds no rows", async () => { + await expectQuery({ range: { auto: { low: 0, high: 0 } }, - }).toFindNothing()) + }).toFindNothing() + }) isSqs && - it("can search using just a low value", () => - expectQuery({ + it("can search using just a low value", async () => { + await expectQuery({ range: { auto: { low: 9 } }, - }).toContainExactly([{ auto: 9 }, { auto: 10 }])) + }).toContainExactly([{ auto: 9 }, { auto: 10 }]) + }) isSqs && - it("can search using just a high value", () => - expectQuery({ + it("can search using just a high value", async () => { + await expectQuery({ range: { auto: { high: 2 } }, - }).toContainExactly([{ auto: 1 }, { auto: 2 }])) + }).toContainExactly([{ auto: 1 }, { auto: 2 }]) + }) }) isSqs && describe("sort", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "auto", sortOrder: SortOrder.ASCENDING, @@ -1524,10 +1668,11 @@ describe.each([ { auto: 8 }, { auto: 9 }, { auto: 10 }, - ])) + ]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "auto", sortOrder: SortOrder.DESCENDING, @@ -1542,7 +1687,8 @@ describe.each([ { auto: 3 }, { auto: 2 }, { auto: 1 }, - ])) + ]) + }) // This is important for pagination. The order of results must always // be stable or pagination will break. We don't want the user to need @@ -1615,13 +1761,15 @@ describe.each([ await createRows([{ "1:name": "bar" }, { "1:name": "foo" }]) }) - it("successfully finds a row", () => - expectQuery({ equal: { "1:1:name": "bar" } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ equal: { "1:1:name": "bar" } }).toContainExactly([ { "1:name": "bar" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { "1:1:name": "none" } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { "1:1:name": "none" } }).toFindNothing() + }) }) describe("user", () => { @@ -1648,51 +1796,59 @@ describe.each([ }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { user: user1._id } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ equal: { user: user1._id } }).toContainExactly([ { user: { _id: user1._id } }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { user: "us_none" } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { user: "us_none" } }).toFindNothing() + }) }) describe("notEqual", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { user: user1._id } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { user: user1._id } }).toContainExactly([ { user: { _id: user2._id } }, {}, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([ + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([ { user: { _id: user1._id } }, { user: { _id: user2._id } }, {}, - ])) + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { user: [user1._id] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { user: [user1._id] } }).toContainExactly([ { user: { _id: user1._id } }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { user: ["us_none"] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { user: ["us_none"] } }).toFindNothing() + }) }) describe("empty", () => { - it("finds empty rows", () => - expectQuery({ empty: { user: null } }).toContainExactly([{}])) + it("finds empty rows", async () => { + await expectQuery({ empty: { user: null } }).toContainExactly([{}]) + }) }) describe("notEmpty", () => { - it("finds non-empty rows", () => - expectQuery({ notEmpty: { user: null } }).toContainExactly([ + it("finds non-empty rows", async () => { + await expectQuery({ notEmpty: { user: null } }).toContainExactly([ { user: { _id: user1._id } }, { user: { _id: user2._id } }, - ])) + ]) + }) }) }) @@ -1726,58 +1882,71 @@ describe.each([ }) describe("contains", () => { - it("successfully finds a row", () => - expectQuery({ contains: { users: [user1._id] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ + contains: { users: [user1._id] }, + }).toContainExactly([ { users: [{ _id: user1._id }] }, { users: [{ _id: user1._id }, { _id: user2._id }] }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ contains: { users: ["us_none"] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ contains: { users: ["us_none"] } }).toFindNothing() + }) }) describe("notContains", () => { - it("successfully finds a row", () => - expectQuery({ notContains: { users: [user1._id] } }).toContainExactly([ - { users: [{ _id: user2._id }] }, - {}, - ])) + it("successfully finds a row", async () => { + await expectQuery({ + notContains: { users: [user1._id] }, + }).toContainExactly([{ users: [{ _id: user2._id }] }, {}]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notContains: { users: ["us_none"] } }).toContainExactly([ + it("fails to find nonexistent row", async () => { + await expectQuery({ + notContains: { users: ["us_none"] }, + }).toContainExactly([ { users: [{ _id: user1._id }] }, { users: [{ _id: user2._id }] }, { users: [{ _id: user1._id }, { _id: user2._id }] }, {}, - ])) + ]) + }) }) describe("containsAny", () => { - it("successfully finds rows", () => - expectQuery({ + it("successfully finds rows", async () => { + await expectQuery({ containsAny: { users: [user1._id, user2._id] }, }).toContainExactly([ { users: [{ _id: user1._id }] }, { users: [{ _id: user2._id }] }, { users: [{ _id: user1._id }, { _id: user2._id }] }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ containsAny: { users: ["us_none"] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ + containsAny: { users: ["us_none"] }, + }).toFindNothing() + }) }) describe("multi-column equals", () => { - it("successfully finds a row", () => - expectQuery({ + it("successfully finds a row", async () => { + await expectQuery({ equal: { number: 1 }, contains: { users: [user1._id] }, - }).toContainExactly([{ users: [{ _id: user1._id }], number: 1 }])) + }).toContainExactly([{ users: [{ _id: user1._id }], number: 1 }]) + }) - it("fails to find nonexistent row", () => - expectQuery({ + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { number: 2 }, contains: { users: [user1._id] }, - }).toFindNothing()) + }).toFindNothing() + }) }) }) @@ -1827,12 +1996,13 @@ describe.each([ rows = await config.api.row.fetch(table._id!) }) - it("can search through relations", () => - expectQuery({ + it("can search through relations", async () => { + await expectQuery({ equal: { [`${otherTable.name}.one`]: "foo" }, }).toContainExactly([ { two: "foo", other: [{ _id: otherRows[0]._id }] }, - ])) + ]) + }) }) // lucene can't count the total rows @@ -1848,18 +2018,19 @@ describe.each([ await createRows([{ name: "a" }, { name: "b" }]) }) - it("should be able to count rows when option set", () => - expectSearch({ + it("should be able to count rows when option set", async () => { + await expectSearch({ countRows: true, query: { notEmpty: { name: true, }, }, - }).toMatch({ totalRows: 2, rows: expect.any(Array) })) + }).toMatch({ totalRows: 2, rows: expect.any(Array) }) + }) - it("shouldn't count rows when option is not set", () => { - expectSearch({ + it("shouldn't count rows when option is not set", async () => { + await expectSearch({ countRows: false, query: { notEmpty: { From 524cf6100b2aaa7cb73aab64c3908f35bcf74309 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 20 Jun 2024 17:57:15 +0100 Subject: [PATCH 09/27] bumping account portal prod tag --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index b41b427134..537b3a6947 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit b41b4271348ff0b526bfc1a9c41dff1169f7bfd9 +Subproject commit 537b3a6947d4aaebbd225b968342745a76f87c40 From 96be6e85e3cf9be07ed4ecb66b8d45403739a4df Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 20 Jun 2024 18:05:48 +0100 Subject: [PATCH 10/27] update account portal sha --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 537b3a6947..b600cca314 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 537b3a6947d4aaebbd225b968342745a76f87c40 +Subproject commit b600cca314a5cc9971e44d46047d1a0019b46b08 From def3b0260e394ceac61718154e7c39600e44810d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 20 Jun 2024 18:48:22 +0100 Subject: [PATCH 11/27] Disallowing prohibited columns consistently, no matter the case, and backend validation for this as well. --- packages/backend-core/src/db/constants.ts | 19 +++++----------- .../DataTable/modals/CreateEditColumn.svelte | 17 ++++++++++---- .../src/sdk/app/tables/internal/index.ts | 12 +++++++++- packages/shared-core/src/constants/index.ts | 1 + packages/shared-core/src/constants/rows.ts | 14 ++++++++++++ packages/shared-core/src/table.ts | 22 ++++++++++++++++++- 6 files changed, 65 insertions(+), 20 deletions(-) create mode 100644 packages/shared-core/src/constants/rows.ts diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts index bfa7595d62..69c98fe569 100644 --- a/packages/backend-core/src/db/constants.ts +++ b/packages/backend-core/src/db/constants.ts @@ -1,14 +1,5 @@ -export const CONSTANT_INTERNAL_ROW_COLS = [ - "_id", - "_rev", - "type", - "createdAt", - "updatedAt", - "tableId", -] as const - -export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const - -export function isInternalColumnName(name: string): boolean { - return (CONSTANT_INTERNAL_ROW_COLS as readonly string[]).includes(name) -} +export { + CONSTANT_INTERNAL_ROW_COLS, + CONSTANT_EXTERNAL_ROW_COLS, + isInternalColumnName, +} from "@budibase/shared-core" diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 17ecd8f844..29d418c1f6 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -17,6 +17,8 @@ SWITCHABLE_TYPES, ValidColumnNameRegex, helpers, + CONSTANT_INTERNAL_ROW_COLS, + CONSTANT_EXTERNAL_ROW_COLS, } from "@budibase/shared-core" import { createEventDispatcher, getContext, onMount } from "svelte" import { cloneDeep } from "lodash/fp" @@ -487,20 +489,27 @@ }) } const newError = {} + const prohibited = externalTable + ? CONSTANT_EXTERNAL_ROW_COLS + : CONSTANT_INTERNAL_ROW_COLS if (!externalTable && fieldInfo.name?.startsWith("_")) { newError.name = `Column name cannot start with an underscore.` } else if (fieldInfo.name && !fieldInfo.name.match(ValidColumnNameRegex)) { newError.name = `Illegal character; must be alpha-numeric.` - } else if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) { - newError.name = `${PROHIBITED_COLUMN_NAMES.join( + } else if ( + prohibited.some( + name => fieldInfo?.name?.toLowerCase() === name.toLowerCase() + ) + ) { + newError.name = `${prohibited.join( ", " - )} are not allowed as column names` + )} are not allowed as column names - case insensitive.` } else if (inUse($tables.selected, fieldInfo.name, originalName)) { newError.name = `Column name already in use.` } if (fieldInfo.type === FieldType.AUTO && !fieldInfo.subtype) { - newError.subtype = `Auto Column requires a type` + newError.subtype = `Auto Column requires a type.` } if (fieldInfo.fieldName && fieldInfo.tableId) { diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts index ea40d2bfe9..9178b2cea3 100644 --- a/packages/server/src/sdk/app/tables/internal/index.ts +++ b/packages/server/src/sdk/app/tables/internal/index.ts @@ -16,7 +16,8 @@ import { EventType, updateLinks } from "../../../../db/linkedRows" import { cloneDeep } from "lodash/fp" import isEqual from "lodash/isEqual" import { runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula" -import { context } from "@budibase/backend-core" +import { context, db as dbCore } from "@budibase/backend-core" +import { findDuplicateInternalColumns } from "@budibase/shared-core" import { getTable } from "../getters" import { checkAutoColumns } from "./utils" import * as viewsSdk from "../../views" @@ -44,6 +45,15 @@ export async function save( if (hasTypeChanged(table, oldTable)) { throw new Error("A column type has changed.") } + + // check for case sensitivity - we don't want to allow duplicated columns + const duplicateColumn = findDuplicateInternalColumns(table) + if (duplicateColumn) { + throw new Error( + `Column "${duplicateColumn}" is duplicated - make sure there are no duplicate columns names, this is case insensitive.` + ) + } + // check that subtypes have been maintained table = checkAutoColumns(table, oldTable) diff --git a/packages/shared-core/src/constants/index.ts b/packages/shared-core/src/constants/index.ts index afb7e659e1..084bc8fe9e 100644 --- a/packages/shared-core/src/constants/index.ts +++ b/packages/shared-core/src/constants/index.ts @@ -1,5 +1,6 @@ export * from "./api" export * from "./fields" +export * from "./rows" export const OperatorOptions = { Equals: { diff --git a/packages/shared-core/src/constants/rows.ts b/packages/shared-core/src/constants/rows.ts new file mode 100644 index 0000000000..bfa7595d62 --- /dev/null +++ b/packages/shared-core/src/constants/rows.ts @@ -0,0 +1,14 @@ +export const CONSTANT_INTERNAL_ROW_COLS = [ + "_id", + "_rev", + "type", + "createdAt", + "updatedAt", + "tableId", +] as const + +export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const + +export function isInternalColumnName(name: string): boolean { + return (CONSTANT_INTERNAL_ROW_COLS as readonly string[]).includes(name) +} diff --git a/packages/shared-core/src/table.ts b/packages/shared-core/src/table.ts index 7706b78037..1e40b38c05 100644 --- a/packages/shared-core/src/table.ts +++ b/packages/shared-core/src/table.ts @@ -1,4 +1,5 @@ -import { FieldType } from "@budibase/types" +import { FieldType, Table } from "@budibase/types" +import { CONSTANT_INTERNAL_ROW_COLS } from "./constants" const allowDisplayColumnByType: Record = { [FieldType.STRING]: true, @@ -51,3 +52,22 @@ export function canBeDisplayColumn(type: FieldType): boolean { export function canBeSortColumn(type: FieldType): boolean { return !!allowSortColumnByType[type] } + +export function findDuplicateInternalColumns(table: Table): string | undefined { + // get the column names + const columnNames = Object.keys(table.schema) + .concat(CONSTANT_INTERNAL_ROW_COLS) + .map(colName => colName.toLowerCase()) + // there are duplicates + const set = new Set(columnNames) + let foundDuplicate: string | undefined + if (set.size !== columnNames.length) { + for (let key of set.keys()) { + const count = columnNames.filter(name => name === key).length + if (count > 1) { + foundDuplicate = key + } + } + } + return foundDuplicate +} From ae68c561f4750bd0901f349837f7695b871c1436 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 20 Jun 2024 18:51:04 +0100 Subject: [PATCH 12/27] Test case. --- .../server/src/api/routes/tests/table.spec.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index f23e0de6db..4c3d8db1af 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -276,6 +276,31 @@ describe.each([ }) }) + it("shouldn't allow duplicate column names", async () => { + const saveTableRequest: SaveTableRequest = { + ...basicTable(), + } + saveTableRequest.schema["Type"] = { type: FieldType.STRING, name: "Type" } + await config.api.table.save(saveTableRequest, { + status: 400, + body: { + message: + 'Column "type" is duplicated - make sure there are no duplicate columns names, this is case insensitive.', + }, + }) + saveTableRequest.schema = { + foo: { type: FieldType.STRING, name: "foo" }, + FOO: { type: FieldType.STRING, name: "FOO" }, + } + await config.api.table.save(saveTableRequest, { + status: 400, + body: { + message: + 'Column "foo" is duplicated - make sure there are no duplicate columns names, this is case insensitive.', + }, + }) + }) + it("should add a new column for an internal DB table", async () => { const saveTableRequest: SaveTableRequest = { ...basicTable(), From fead1f436a5f30c4c28974d2b282866bbf0be70f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 20 Jun 2024 18:53:01 +0100 Subject: [PATCH 13/27] test case is only for internal. --- .../server/src/api/routes/tests/table.spec.ts | 50 ++++++++++--------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index 4c3d8db1af..a84fd923bb 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -276,30 +276,34 @@ describe.each([ }) }) - it("shouldn't allow duplicate column names", async () => { - const saveTableRequest: SaveTableRequest = { - ...basicTable(), - } - saveTableRequest.schema["Type"] = { type: FieldType.STRING, name: "Type" } - await config.api.table.save(saveTableRequest, { - status: 400, - body: { - message: - 'Column "type" is duplicated - make sure there are no duplicate columns names, this is case insensitive.', - }, + isInternal && + it("shouldn't allow duplicate column names", async () => { + const saveTableRequest: SaveTableRequest = { + ...basicTable(), + } + saveTableRequest.schema["Type"] = { + type: FieldType.STRING, + name: "Type", + } + await config.api.table.save(saveTableRequest, { + status: 400, + body: { + message: + 'Column "type" is duplicated - make sure there are no duplicate columns names, this is case insensitive.', + }, + }) + saveTableRequest.schema = { + foo: { type: FieldType.STRING, name: "foo" }, + FOO: { type: FieldType.STRING, name: "FOO" }, + } + await config.api.table.save(saveTableRequest, { + status: 400, + body: { + message: + 'Column "foo" is duplicated - make sure there are no duplicate columns names, this is case insensitive.', + }, + }) }) - saveTableRequest.schema = { - foo: { type: FieldType.STRING, name: "foo" }, - FOO: { type: FieldType.STRING, name: "FOO" }, - } - await config.api.table.save(saveTableRequest, { - status: 400, - body: { - message: - 'Column "foo" is duplicated - make sure there are no duplicate columns names, this is case insensitive.', - }, - }) - }) it("should add a new column for an internal DB table", async () => { const saveTableRequest: SaveTableRequest = { From b4910043c67fe3bcd9c162684a46bee5bcaaa707 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 21 Jun 2024 11:27:47 +0100 Subject: [PATCH 14/27] Addressing PR comments. --- .../backend/DataTable/modals/CreateEditColumn.svelte | 1 - packages/server/src/api/routes/tests/table.spec.ts | 11 +++++------ packages/server/src/sdk/app/tables/internal/index.ts | 8 +++++--- packages/shared-core/src/table.ts | 8 ++++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 29d418c1f6..d79eedd194 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -54,7 +54,6 @@ const DATE_TYPE = FieldType.DATETIME const dispatch = createEventDispatcher() - const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"] const { dispatch: gridDispatch, rows } = getContext("grid") export let field diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index a84fd923bb..e75e5e23e7 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -289,18 +289,17 @@ describe.each([ status: 400, body: { message: - 'Column "type" is duplicated - make sure there are no duplicate columns names, this is case insensitive.', + 'Column(s) "type" are duplicated - check for other columns with these name (case in-sensitive)', }, }) - saveTableRequest.schema = { - foo: { type: FieldType.STRING, name: "foo" }, - FOO: { type: FieldType.STRING, name: "FOO" }, - } + saveTableRequest.schema.foo = { type: FieldType.STRING, name: "foo" } + saveTableRequest.schema.FOO = { type: FieldType.STRING, name: "FOO" } + await config.api.table.save(saveTableRequest, { status: 400, body: { message: - 'Column "foo" is duplicated - make sure there are no duplicate columns names, this is case insensitive.', + 'Column(s) "type, foo" are duplicated - check for other columns with these name (case in-sensitive)', }, }) }) diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts index 9178b2cea3..fc32708708 100644 --- a/packages/server/src/sdk/app/tables/internal/index.ts +++ b/packages/server/src/sdk/app/tables/internal/index.ts @@ -16,7 +16,7 @@ import { EventType, updateLinks } from "../../../../db/linkedRows" import { cloneDeep } from "lodash/fp" import isEqual from "lodash/isEqual" import { runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula" -import { context, db as dbCore } from "@budibase/backend-core" +import { context } from "@budibase/backend-core" import { findDuplicateInternalColumns } from "@budibase/shared-core" import { getTable } from "../getters" import { checkAutoColumns } from "./utils" @@ -48,9 +48,11 @@ export async function save( // check for case sensitivity - we don't want to allow duplicated columns const duplicateColumn = findDuplicateInternalColumns(table) - if (duplicateColumn) { + if (duplicateColumn.length) { throw new Error( - `Column "${duplicateColumn}" is duplicated - make sure there are no duplicate columns names, this is case insensitive.` + `Column(s) "${duplicateColumn.join( + ", " + )}" are duplicated - check for other columns with these name (case in-sensitive)` ) } diff --git a/packages/shared-core/src/table.ts b/packages/shared-core/src/table.ts index 1e40b38c05..4b578a2aef 100644 --- a/packages/shared-core/src/table.ts +++ b/packages/shared-core/src/table.ts @@ -53,21 +53,21 @@ export function canBeSortColumn(type: FieldType): boolean { return !!allowSortColumnByType[type] } -export function findDuplicateInternalColumns(table: Table): string | undefined { +export function findDuplicateInternalColumns(table: Table): string[] { // get the column names const columnNames = Object.keys(table.schema) .concat(CONSTANT_INTERNAL_ROW_COLS) .map(colName => colName.toLowerCase()) // there are duplicates const set = new Set(columnNames) - let foundDuplicate: string | undefined + let duplicates: string[] = [] if (set.size !== columnNames.length) { for (let key of set.keys()) { const count = columnNames.filter(name => name === key).length if (count > 1) { - foundDuplicate = key + duplicates.push(key) } } } - return foundDuplicate + return duplicates } From 9a375d6716f1004ea53061c969c5e370bb100bbe Mon Sep 17 00:00:00 2001 From: Conor Webb <126772285+ConorWebb96@users.noreply.github.com> Date: Fri, 21 Jun 2024 11:57:46 +0100 Subject: [PATCH 15/27] Add the option to change the confirm and cancel button texts within the confirmation modals (#13966) * Add button parameters to actions * Add button states to confirmation store * Set text of buttons if values are exist * Pass stored values through to the modal * Add missing duplicate text map * Fix lint issues --------- Co-authored-by: melohagan <101575380+melohagan@users.noreply.github.com> --- .../ButtonActionEditor/actions/DeleteRow.svelte | 6 ++++++ .../ButtonActionEditor/actions/DuplicateRow.svelte | 6 ++++++ .../ButtonActionEditor/actions/ExecuteQuery.svelte | 12 ++++++++++++ .../ButtonActionEditor/actions/SaveRow.svelte | 6 ++++++ .../components/overlay/ConfirmationDisplay.svelte | 2 ++ packages/client/src/stores/confirmation.js | 13 ++++++++++++- packages/client/src/utils/buttonActions.js | 10 +++++++++- 7 files changed, 53 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DeleteRow.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DeleteRow.svelte index 5b7844ce53..fd3521d597 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DeleteRow.svelte +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DeleteRow.svelte @@ -53,6 +53,12 @@ placeholder="Are you sure you want to delete?" bind:value={parameters.confirmText} /> + + + + + + {/if} diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DuplicateRow.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DuplicateRow.svelte index 3b4a7c2d38..b6cdd663fd 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DuplicateRow.svelte +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DuplicateRow.svelte @@ -83,6 +83,12 @@ placeholder="Are you sure you want to duplicate this row?" bind:value={parameters.confirmText} /> + + + + + + {/if} diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExecuteQuery.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExecuteQuery.svelte index 54295d8b0f..43797f6369 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExecuteQuery.svelte +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExecuteQuery.svelte @@ -74,6 +74,18 @@ placeholder="Are you sure you want to execute this query?" bind:value={parameters.confirmText} /> + + + + {/if} {#if query?.parameters?.length > 0} diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte index d834e9aac9..aed2618778 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte @@ -80,6 +80,12 @@ placeholder="Are you sure you want to save this row?" bind:value={parameters.confirmText} /> + + + + + + {/if} diff --git a/packages/client/src/components/overlay/ConfirmationDisplay.svelte b/packages/client/src/components/overlay/ConfirmationDisplay.svelte index e7a1046191..b96af502df 100644 --- a/packages/client/src/components/overlay/ConfirmationDisplay.svelte +++ b/packages/client/src/components/overlay/ConfirmationDisplay.svelte @@ -8,6 +8,8 @@ {$confirmationStore.text} diff --git a/packages/client/src/stores/confirmation.js b/packages/client/src/stores/confirmation.js index bb9a54386f..3fbf3d5deb 100644 --- a/packages/client/src/stores/confirmation.js +++ b/packages/client/src/stores/confirmation.js @@ -4,6 +4,8 @@ const initialState = { showConfirmation: false, title: null, text: null, + confirmButtonText: null, + cancelButtonText: null, onConfirm: null, onCancel: null, } @@ -11,11 +13,20 @@ const initialState = { const createConfirmationStore = () => { const store = writable(initialState) - const showConfirmation = (title, text, onConfirm, onCancel) => { + const showConfirmation = ( + title, + text, + onConfirm, + onCancel, + confirmButtonText, + cancelButtonText + ) => { store.set({ showConfirmation: true, title, text, + confirmButtonText, + cancelButtonText, onConfirm, onCancel, }) diff --git a/packages/client/src/utils/buttonActions.js b/packages/client/src/utils/buttonActions.js index bd220b8e85..8f0cb575a7 100644 --- a/packages/client/src/utils/buttonActions.js +++ b/packages/client/src/utils/buttonActions.js @@ -522,6 +522,7 @@ const confirmTextMap = { ["Execute Query"]: "Are you sure you want to execute this query?", ["Trigger Automation"]: "Are you sure you want to trigger this automation?", ["Prompt User"]: "Are you sure you want to continue?", + ["Duplicate Row"]: "Are you sure you want to duplicate this row?", } /** @@ -582,6 +583,11 @@ export const enrichButtonActions = (actions, context) => { const defaultTitleText = action["##eventHandlerType"] const customTitleText = action.parameters?.customTitleText || defaultTitleText + const cancelButtonText = + action.parameters?.cancelButtonText || "Cancel" + const confirmButtonText = + action.parameters?.confirmButtonText || "Confirm" + confirmationStore.actions.showConfirmation( customTitleText, confirmText, @@ -612,7 +618,9 @@ export const enrichButtonActions = (actions, context) => { }, () => { resolve(false) - } + }, + confirmButtonText, + cancelButtonText ) }) } From a850ce6996ad14db04a9cf372c2ce0041a45abc0 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 21 Jun 2024 12:39:18 +0000 Subject: [PATCH 16/27] Bump version to 2.29.1 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 9d04750a0d..0efaf75283 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.29.0", + "version": "2.29.1", "npmClient": "yarn", "packages": [ "packages/*", From e5c40c7ecdf00f654cbe133317a119f2ee1808d0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 21 Jun 2024 16:58:27 +0100 Subject: [PATCH 17/27] Moving some stuff around inside ExternalRequests to make it easier to access parts of the full context. --- .../api/controllers/row/ExternalRequest.ts | 238 ++++++++++-------- 1 file changed, 138 insertions(+), 100 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 619a1e9548..3209416544 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -72,92 +72,6 @@ export type ExternalRequestReturnType = ? number : { row: Row; table: Table } -function buildFilters( - id: string | undefined | string[], - filters: SearchFilters, - table: Table -) { - const primary = table.primary - // if passed in array need to copy for shifting etc - let idCopy: undefined | string | any[] = cloneDeep(id) - if (filters) { - // need to map over the filters and make sure the _id field isn't present - let prefix = 1 - for (let operator of Object.values(filters)) { - for (let field of Object.keys(operator || {})) { - if (dbCore.removeKeyNumbering(field) === "_id") { - if (primary) { - const parts = breakRowIdField(operator[field]) - for (let field of primary) { - operator[`${prefix}:${field}`] = parts.shift() - } - prefix++ - } - // make sure this field doesn't exist on any filter - delete operator[field] - } - } - } - } - // there is no id, just use the user provided filters - if (!idCopy || !table) { - return filters - } - // if used as URL parameter it will have been joined - if (!Array.isArray(idCopy)) { - idCopy = breakRowIdField(idCopy) - } - const equal: any = {} - if (primary && idCopy) { - for (let field of primary) { - // work through the ID and get the parts - equal[field] = idCopy.shift() - } - } - return { - equal, - } -} - -async function removeManyToManyRelationships( - rowId: string, - table: Table, - colName: string -) { - const tableId = table._id! - const filters = buildFilters(rowId, {}, table) - // safety check, if there are no filters on deletion bad things happen - if (Object.keys(filters).length !== 0) { - return getDatasourceAndQuery({ - endpoint: getEndpoint(tableId, Operation.DELETE), - body: { [colName]: null }, - filters, - meta: { - table, - }, - }) - } else { - return [] - } -} - -async function removeOneToManyRelationships(rowId: string, table: Table) { - const tableId = table._id! - const filters = buildFilters(rowId, {}, table) - // safety check, if there are no filters on deletion bad things happen - if (Object.keys(filters).length !== 0) { - return getDatasourceAndQuery({ - endpoint: getEndpoint(tableId, Operation.UPDATE), - filters, - meta: { - table, - }, - }) - } else { - return [] - } -} - /** * This function checks the incoming parameters to make sure all the inputs are * valid based on on the table schema. The main thing this is looking for is when a @@ -240,6 +154,7 @@ export class ExternalRequest { private readonly tableId: string private datasource?: Datasource private tables: { [key: string]: Table } = {} + private tableList: Table[] constructor(operation: T, tableId: string, datasource?: Datasource) { this.operation = operation @@ -248,6 +163,117 @@ export class ExternalRequest { if (datasource && datasource.entities) { this.tables = datasource.entities } + this.tableList = Object.values(this.tables) + } + + private prepareFilters( + id: string | undefined | string[], + filters: SearchFilters, + table: Table + ) { + const tables = this.tableList + // replace any relationship columns initially, table names and relationship column names are acceptable + const relationshipColumns = sdk.rows.filters.getRelationshipColumns(table) + filters = sdk.rows.filters.updateFilterKeys( + filters, + relationshipColumns + .map(({ name, definition }) => { + const { tableName } = breakExternalTableId(definition.tableId) + return { + original: name, + updated: tableName!, + } + }) + // don't update table names - include this for context incase a column would be replaced + .concat( + tables.map(table => { + const tableName = table.originalName || table.name + return { + original: tableName, + updated: tableName, + } + }) + ) + ) + const primary = table.primary + // if passed in array need to copy for shifting etc + let idCopy: undefined | string | any[] = cloneDeep(id) + if (filters) { + // need to map over the filters and make sure the _id field isn't present + let prefix = 1 + for (let operator of Object.values(filters)) { + for (let field of Object.keys(operator || {})) { + if (dbCore.removeKeyNumbering(field) === "_id") { + if (primary) { + const parts = breakRowIdField(operator[field]) + for (let field of primary) { + operator[`${prefix}:${field}`] = parts.shift() + } + prefix++ + } + // make sure this field doesn't exist on any filter + delete operator[field] + } + } + } + } + // there is no id, just use the user provided filters + if (!idCopy || !table) { + return filters + } + // if used as URL parameter it will have been joined + if (!Array.isArray(idCopy)) { + idCopy = breakRowIdField(idCopy) + } + const equal: any = {} + if (primary && idCopy) { + for (let field of primary) { + // work through the ID and get the parts + equal[field] = idCopy.shift() + } + } + return { + equal, + } + } + + private async removeManyToManyRelationships( + rowId: string, + table: Table, + colName: string + ) { + const tableId = table._id! + const filters = this.prepareFilters(rowId, {}, table) + // safety check, if there are no filters on deletion bad things happen + if (Object.keys(filters).length !== 0) { + return getDatasourceAndQuery({ + endpoint: getEndpoint(tableId, Operation.DELETE), + body: { [colName]: null }, + filters, + meta: { + table, + }, + }) + } else { + return [] + } + } + + private async removeOneToManyRelationships(rowId: string, table: Table) { + const tableId = table._id! + const filters = this.prepareFilters(rowId, {}, table) + // safety check, if there are no filters on deletion bad things happen + if (Object.keys(filters).length !== 0) { + return getDatasourceAndQuery({ + endpoint: getEndpoint(tableId, Operation.UPDATE), + filters, + meta: { + table, + }, + }) + } else { + return [] + } } getTable(tableId: string | undefined): Table | undefined { @@ -260,10 +286,22 @@ export class ExternalRequest { } } + // seeds the object with table and datasource information + async retrieveMetadata(datasourceId: string) { + if (!this.datasource) { + this.datasource = await sdk.datasources.get(datasourceId) + if (!this.datasource || !this.datasource.entities) { + throw "No tables found, fetch tables before query." + } + this.tables = this.datasource.entities + this.tableList = Object.values(this.tables) + } + } + async getRow(table: Table, rowId: string): Promise { const response = await getDatasourceAndQuery({ endpoint: getEndpoint(table._id!, Operation.READ), - filters: buildFilters(rowId, {}, table), + filters: this.prepareFilters(rowId, {}, table), meta: { table, }, @@ -514,7 +552,7 @@ export class ExternalRequest { endpoint: getEndpoint(tableId, operation), // if we're doing many relationships then we're writing, only one response body, - filters: buildFilters(id, {}, linkTable), + filters: this.prepareFilters(id, {}, linkTable), meta: { table: linkTable, }, @@ -538,8 +576,8 @@ export class ExternalRequest { for (let row of rows) { const rowId = generateIdForRow(row, table) const promise: Promise = isMany - ? removeManyToManyRelationships(rowId, table, colName) - : removeOneToManyRelationships(rowId, table) + ? this.removeManyToManyRelationships(rowId, table, colName) + : this.removeOneToManyRelationships(rowId, table) if (promise) { promises.push(promise) } @@ -562,12 +600,12 @@ export class ExternalRequest { rows.map(row => { const rowId = generateIdForRow(row, table) return isMany - ? removeManyToManyRelationships( + ? this.removeManyToManyRelationships( rowId, table, relationshipColumn.fieldName ) - : removeOneToManyRelationships(rowId, table) + : this.removeOneToManyRelationships(rowId, table) }) ) } @@ -580,14 +618,10 @@ export class ExternalRequest { throw "Unable to run without a table name" } if (!this.datasource) { - this.datasource = await sdk.datasources.get(datasourceId!) - if (!this.datasource || !this.datasource.entities) { - throw "No tables found, fetch tables before query." - } - this.tables = this.datasource.entities + await this.retrieveMetadata(datasourceId!) } const table = this.tables[tableName] - let isSql = isSQL(this.datasource) + let isSql = isSQL(this.datasource!) if (!table) { throw `Unable to process query, table "${tableName}" not defined.` } @@ -612,7 +646,7 @@ export class ExternalRequest { break } } - filters = buildFilters(id, filters || {}, table) + filters = this.prepareFilters(id, filters || {}, table) const relationships = buildExternalRelationships(table, this.tables) const incRelationships = @@ -660,7 +694,11 @@ export class ExternalRequest { body: row || rows, // pass an id filter into extra, purely for mysql/returning extra: { - idFilter: buildFilters(id || generateIdForRow(row, table), {}, table), + idFilter: this.prepareFilters( + id || generateIdForRow(row, table), + {}, + table + ), }, meta: { table, From 6812c2107696c05d35e87367088f3639f71c952a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 21 Jun 2024 16:58:40 +0100 Subject: [PATCH 18/27] Updating test cases. --- .../src/api/routes/tests/search.spec.ts | 71 +++++++++++-------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index cff966ab89..c92738479f 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -76,9 +76,9 @@ describe.each([ } }) - async function createTable(schema: TableSchema) { + async function createTable(schema: TableSchema, name?: string) { return await config.api.table.save( - tableForDatasource(datasource, { schema }) + tableForDatasource(datasource, { schema, name }) ) } @@ -1956,51 +1956,62 @@ describe.each([ // isn't available. !isInMemory && describe("relations", () => { - let otherTable: Table - let otherRows: Row[] + let productCategoryTable: Table, productCatRows: Row[] beforeAll(async () => { - otherTable = await createTable({ - one: { name: "one", type: FieldType.STRING }, - }) - table = await createTable({ - two: { name: "two", type: FieldType.STRING }, - other: { - type: FieldType.LINK, - relationshipType: RelationshipType.ONE_TO_MANY, - name: "other", - fieldName: "other", - tableId: otherTable._id!, - constraints: { - type: "array", + productCategoryTable = await createTable( + { + name: { name: "name", type: FieldType.STRING }, + }, + "productCategory" + ) + table = await createTable( + { + name: { name: "name", type: FieldType.STRING }, + productCat: { + type: FieldType.LINK, + relationshipType: RelationshipType.ONE_TO_MANY, + name: "productCat", + fieldName: "product", + tableId: productCategoryTable._id!, + constraints: { + type: "array", + }, }, }, - }) + "product" + ) - otherRows = await Promise.all([ - config.api.row.save(otherTable._id!, { one: "foo" }), - config.api.row.save(otherTable._id!, { one: "bar" }), + productCatRows = await Promise.all([ + config.api.row.save(productCategoryTable._id!, { name: "foo" }), + config.api.row.save(productCategoryTable._id!, { name: "bar" }), ]) await Promise.all([ config.api.row.save(table._id!, { - two: "foo", - other: [otherRows[0]._id], + name: "foo", + productCat: [productCatRows[0]._id], }), config.api.row.save(table._id!, { - two: "bar", - other: [otherRows[1]._id], + name: "bar", + productCat: [productCatRows[1]._id], }), ]) - - rows = await config.api.row.fetch(table._id!) }) - it("can search through relations", async () => { + it("should be able to filter by relationship using column name", async () => { await expectQuery({ - equal: { [`${otherTable.name}.one`]: "foo" }, + equal: { ["productCat.name"]: "foo" }, }).toContainExactly([ - { two: "foo", other: [{ _id: otherRows[0]._id }] }, + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) + + it("should be able to filter by relationship using table name", async () => { + await expectQuery({ + equal: { ["productCategory.name"]: "foo" }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, ]) }) }) From 28d0d627ce971e029018f2b77e9abaae2bdbccf3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 21 Jun 2024 17:00:12 +0100 Subject: [PATCH 19/27] Getting functions in place which make it easy to update pats of a filter list by their keys - getting this to work for SQS and external. --- packages/server/src/sdk/app/rows/index.ts | 2 + .../server/src/sdk/app/rows/search/filters.ts | 56 +++++++++++++++++ .../server/src/sdk/app/rows/search/sqs.ts | 60 ++++++++++--------- 3 files changed, 89 insertions(+), 29 deletions(-) create mode 100644 packages/server/src/sdk/app/rows/search/filters.ts diff --git a/packages/server/src/sdk/app/rows/index.ts b/packages/server/src/sdk/app/rows/index.ts index c117941419..fb077509a9 100644 --- a/packages/server/src/sdk/app/rows/index.ts +++ b/packages/server/src/sdk/app/rows/index.ts @@ -3,12 +3,14 @@ import * as rows from "./rows" import * as search from "./search" import * as utils from "./utils" import * as external from "./external" +import * as filters from "./search/filters" import AliasTables from "./sqlAlias" export default { ...attachments, ...rows, ...search, + filters, utils, external, AliasTables, diff --git a/packages/server/src/sdk/app/rows/search/filters.ts b/packages/server/src/sdk/app/rows/search/filters.ts new file mode 100644 index 0000000000..3f8facc78e --- /dev/null +++ b/packages/server/src/sdk/app/rows/search/filters.ts @@ -0,0 +1,56 @@ +import { + FieldType, + RelationshipFieldMetadata, + SearchFilters, + Table, +} from "@budibase/types" + +export function getRelationshipColumns(table: Table): { + name: string + definition: RelationshipFieldMetadata +}[] { + return Object.entries(table.schema) + .filter(entry => entry[1].type === FieldType.LINK) + .map(entry => ({ + name: entry[0], + definition: entry[1] as RelationshipFieldMetadata, + })) +} + +export function getTableIDList( + tables: Table[] +): { name: string; id: string }[] { + return tables + .filter(table => table.originalName) + .map(table => ({ id: table._id!, name: table.originalName! })) +} + +export function updateFilterKeys( + filters: SearchFilters, + updates: { original: string; updated: string }[] +): SearchFilters { + // sort the updates by length first - this is necessary to avoid replacing sub-strings + updates = updates.sort((a, b) => b.original.length - a.original.length) + const makeFilterKeyRegex = (str: string) => + new RegExp(`^${str}.|:${str}.`, "g") + for (let filter of Object.values(filters)) { + if (typeof filter !== "object") { + continue + } + for (let [key, keyFilter] of Object.entries(filter)) { + if (keyFilter === "") { + delete filter[key] + } + const possibleKey = updates.find(({ original }) => + key.match(makeFilterKeyRegex(original)) + ) + if (possibleKey && possibleKey.original !== possibleKey.updated) { + // only replace the first, not replaceAll + filter[key.replace(possibleKey.original, possibleKey.updated)] = + filter[key] + delete filter[key] + } + } + } + return filters +} diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index bb1d62affc..174ecc0e38 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -1,4 +1,5 @@ import { + Datasource, DocumentType, FieldType, Operation, @@ -12,7 +13,6 @@ import { SortType, SqlClient, Table, - Datasource, } from "@budibase/types" import { buildInternalRelationships, @@ -30,6 +30,11 @@ import AliasTables from "../sqlAlias" import { outputProcessing } from "../../../../utilities/rowProcessor" import pick from "lodash/pick" import { processRowCountResponse } from "../utils" +import { + updateFilterKeys, + getRelationshipColumns, + getTableIDList, +} from "./filters" const builder = new sql.Sql(SqlClient.SQL_LITE) @@ -60,34 +65,31 @@ function buildInternalFieldList( return fieldList } -function tableNameInFieldRegex(tableName: string) { - return new RegExp(`^${tableName}.|:${tableName}.`, "g") -} - -function cleanupFilters(filters: SearchFilters, tables: Table[]) { - for (let filter of Object.values(filters)) { - if (typeof filter !== "object") { - continue - } - for (let [key, keyFilter] of Object.entries(filter)) { - if (keyFilter === "") { - delete filter[key] - } - - // relationship, switch to table ID - const tableRelated = tables.find( - table => - table.originalName && - key.match(tableNameInFieldRegex(table.originalName)) +function cleanupFilters( + filters: SearchFilters, + table: Table, + allTables: Table[] +) { + // get a list of all relationship columns in the table for updating + const relationshipColumns = getRelationshipColumns(table) + // get table names to ID map for relationships + const tableNameToID = getTableIDList(allTables) + // all should be applied at once + filters = updateFilterKeys( + filters, + relationshipColumns + .map(({ name, definition }) => ({ + original: name, + updated: definition.tableId, + })) + .concat( + tableNameToID.map(({ name, id }) => ({ + original: name, + updated: id, + })) ) - if (tableRelated && tableRelated.originalName) { - // only replace the first, not replaceAll - filter[key.replace(tableRelated.originalName, tableRelated._id!)] = - filter[key] - delete filter[key] - } - } - } + ) + return filters } @@ -176,7 +178,7 @@ export async function search( operation: Operation.READ, }, filters: { - ...cleanupFilters(query, allTables), + ...cleanupFilters(query, table, allTables), documentType: DocumentType.ROW, }, table, From 337584f5b2d4f9d53c6c8ee2c22177ce54cb3c99 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 21 Jun 2024 17:51:02 +0100 Subject: [PATCH 20/27] Updating the regex to correctly find within the filter keys. --- packages/server/src/sdk/app/rows/search/filters.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/filters.ts b/packages/server/src/sdk/app/rows/search/filters.ts index 3f8facc78e..32b8526697 100644 --- a/packages/server/src/sdk/app/rows/search/filters.ts +++ b/packages/server/src/sdk/app/rows/search/filters.ts @@ -29,10 +29,8 @@ export function updateFilterKeys( filters: SearchFilters, updates: { original: string; updated: string }[] ): SearchFilters { - // sort the updates by length first - this is necessary to avoid replacing sub-strings - updates = updates.sort((a, b) => b.original.length - a.original.length) const makeFilterKeyRegex = (str: string) => - new RegExp(`^${str}.|:${str}.`, "g") + new RegExp(`^${str}\\.|:${str}\\.`) for (let filter of Object.values(filters)) { if (typeof filter !== "object") { continue From fcf67f729743db26647a15bb7900c101afaf2f3c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 21 Jun 2024 19:29:30 +0100 Subject: [PATCH 21/27] Fixing an issue raised by Poirazis around empty relationships coming back as related to themselves. --- .../server/src/api/controllers/row/utils/basic.ts | 2 +- packages/server/src/api/routes/tests/search.spec.ts | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index afb98d0255..bca2494ac3 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -99,7 +99,7 @@ export function basicProcessing({ row, tableName: table._id!, fieldName: internalColumn, - isLinked: false, + isLinked, }) } } diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index c92738479f..b9c25b98aa 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1950,10 +1950,7 @@ describe.each([ }) }) - // This will never work for Lucene. !isLucene && - // It also can't work for in-memory searching because the related table name - // isn't available. !isInMemory && describe("relations", () => { let productCategoryTable: Table, productCatRows: Row[] @@ -1996,6 +1993,10 @@ describe.each([ name: "bar", productCat: [productCatRows[1]._id], }), + config.api.row.save(table._id!, { + name: "baz", + productCat: [], + }), ]) }) @@ -2014,6 +2015,12 @@ describe.each([ { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, ]) }) + + it("shouldn't return any relationship for last row", async () => { + await expectQuery({ + equal: { ["name"]: "baz" }, + }).toContainExactly([{ name: "baz", productCat: undefined }]) + }) }) // lucene can't count the total rows From 05ea231d203323b769474c6d3617ef2fef39e7eb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 24 Jun 2024 11:53:02 +0100 Subject: [PATCH 22/27] Adding back missing comments. --- packages/server/src/api/routes/tests/search.spec.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index b9c25b98aa..8ff35e7232 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1950,8 +1950,10 @@ describe.each([ }) }) + // This will never work for Lucene. !isLucene && - !isInMemory && + // It also can't work for in-memory searching because the related table name + // isn't available. describe("relations", () => { let productCategoryTable: Table, productCatRows: Row[] From 965725d022b360fd8b5e1b80d6b269514ec7be23 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 24 Jun 2024 12:43:26 +0100 Subject: [PATCH 23/27] First PR comments. --- .../api/controllers/row/ExternalRequest.ts | 25 ++++++------------- .../src/api/routes/tests/search.spec.ts | 1 + .../server/src/sdk/app/rows/search/filters.ts | 24 ++++++++++++------ 3 files changed, 24 insertions(+), 26 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 3209416544..75d3da6b03 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -176,24 +176,13 @@ export class ExternalRequest { const relationshipColumns = sdk.rows.filters.getRelationshipColumns(table) filters = sdk.rows.filters.updateFilterKeys( filters, - relationshipColumns - .map(({ name, definition }) => { - const { tableName } = breakExternalTableId(definition.tableId) - return { - original: name, - updated: tableName!, - } - }) - // don't update table names - include this for context incase a column would be replaced - .concat( - tables.map(table => { - const tableName = table.originalName || table.name - return { - original: tableName, - updated: tableName, - } - }) - ) + relationshipColumns.map(({ name, definition }) => { + const { tableName } = breakExternalTableId(definition.tableId) + return { + original: name, + updated: tableName!, + } + }) ) const primary = table.primary // if passed in array need to copy for shifting etc diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 8ff35e7232..23c2ca819c 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1954,6 +1954,7 @@ describe.each([ !isLucene && // It also can't work for in-memory searching because the related table name // isn't available. + !isInMemory && describe("relations", () => { let productCategoryTable: Table, productCatRows: Row[] diff --git a/packages/server/src/sdk/app/rows/search/filters.ts b/packages/server/src/sdk/app/rows/search/filters.ts index 32b8526697..ccce0ab86a 100644 --- a/packages/server/src/sdk/app/rows/search/filters.ts +++ b/packages/server/src/sdk/app/rows/search/filters.ts @@ -4,24 +4,32 @@ import { SearchFilters, Table, } from "@budibase/types" +import { isPlainObject } from "lodash" export function getRelationshipColumns(table: Table): { name: string definition: RelationshipFieldMetadata }[] { - return Object.entries(table.schema) - .filter(entry => entry[1].type === FieldType.LINK) - .map(entry => ({ - name: entry[0], - definition: entry[1] as RelationshipFieldMetadata, - })) + // performing this with a for loop rather than an array filter improves + // type guarding, as no casts are required + const linkEntries: [string, RelationshipFieldMetadata][] = [] + for (let entry of Object.entries(table.schema)) { + if (entry[1].type === FieldType.LINK) { + const linkColumn: RelationshipFieldMetadata = entry[1] + linkEntries.push([entry[0], linkColumn]) + } + } + return linkEntries.map(entry => ({ + name: entry[0], + definition: entry[1], + })) } export function getTableIDList( tables: Table[] ): { name: string; id: string }[] { return tables - .filter(table => table.originalName) + .filter(table => table.originalName && table._id) .map(table => ({ id: table._id!, name: table.originalName! })) } @@ -32,7 +40,7 @@ export function updateFilterKeys( const makeFilterKeyRegex = (str: string) => new RegExp(`^${str}\\.|:${str}\\.`) for (let filter of Object.values(filters)) { - if (typeof filter !== "object") { + if (!isPlainObject(filter)) { continue } for (let [key, keyFilter] of Object.entries(filter)) { From 1402716f5cd9c37a5dc84830000822c083be4e4a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 24 Jun 2024 13:10:30 +0100 Subject: [PATCH 24/27] Some type updates. --- packages/backend-core/src/sql/sqlTable.ts | 6 ++-- packages/backend-core/src/sql/utils.ts | 8 ++--- .../api/controllers/row/ExternalRequest.ts | 31 ++++++++----------- .../src/api/controllers/row/external.ts | 7 ++--- .../src/api/controllers/row/utils/sqlUtils.ts | 26 +++++++++------- .../src/api/controllers/table/external.ts | 4 +-- .../src/sdk/app/rows/search/external.ts | 10 +++--- packages/server/src/sdk/app/tables/getters.ts | 6 ++-- packages/server/src/sdk/app/views/external.ts | 24 +++++++------- 9 files changed, 60 insertions(+), 62 deletions(-) diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts index 09f9908baa..bdc8a3dd69 100644 --- a/packages/backend-core/src/sql/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -109,8 +109,10 @@ function generateSchema( const { tableName } = breakExternalTableId(column.tableId) // @ts-ignore const relatedTable = tables[tableName] - if (!relatedTable) { - throw new Error("Referenced table doesn't exist") + if (!relatedTable || !relatedTable.primary) { + throw new Error( + "Referenced table doesn't exist or has no primary keys" + ) } const relatedPrimary = relatedTable.primary[0] const externalType = relatedTable.schema[relatedPrimary].externalType diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts index 2d9b289417..45ab510948 100644 --- a/packages/backend-core/src/sql/utils.ts +++ b/packages/backend-core/src/sql/utils.ts @@ -55,10 +55,7 @@ export function buildExternalTableId(datasourceId: string, tableName: string) { return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}` } -export function breakExternalTableId(tableId: string | undefined) { - if (!tableId) { - return {} - } +export function breakExternalTableId(tableId: string) { const parts = tableId.split(DOUBLE_SEPARATOR) let datasourceId = parts.shift() // if they need joined @@ -67,6 +64,9 @@ export function breakExternalTableId(tableId: string | undefined) { if (tableName.includes(ENCODED_SPACE)) { tableName = decodeURIComponent(tableName) } + if (!datasourceId || !tableName) { + throw new Error("Unable to get datasource/table name from table ID") + } return { datasourceId, tableName } } diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 75d3da6b03..7cbe023fd2 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -126,8 +126,8 @@ function getEndpoint(tableId: string | undefined, operation: string) { } const { datasourceId, tableName } = breakExternalTableId(tableId) return { - datasourceId: datasourceId!, - entityId: tableName!, + datasourceId: datasourceId, + entityId: tableName, operation: operation as Operation, } } @@ -180,7 +180,7 @@ export class ExternalRequest { const { tableName } = breakExternalTableId(definition.tableId) return { original: name, - updated: tableName!, + updated: tableName, } }) ) @@ -267,12 +267,10 @@ export class ExternalRequest { getTable(tableId: string | undefined): Table | undefined { if (!tableId) { - throw "Table ID is unknown, cannot find table" + throw new Error("Table ID is unknown, cannot find table") } const { tableName } = breakExternalTableId(tableId) - if (tableName) { - return this.tables[tableName] - } + return this.tables[tableName] } // seeds the object with table and datasource information @@ -323,9 +321,7 @@ export class ExternalRequest { if (field.type === FieldType.NUMBER && !isNaN(parseFloat(row[key]))) { newRow[key] = parseFloat(row[key]) } else if (field.type === FieldType.LINK) { - const { tableName: linkTableName } = breakExternalTableId( - field?.tableId - ) + const { tableName: linkTableName } = breakExternalTableId(field.tableId) // table has to exist for many to many if (!linkTableName || !this.tables[linkTableName]) { continue @@ -406,9 +402,6 @@ export class ExternalRequest { [key: string]: { rows: Row[]; isMany: boolean; tableId: string } } = {} const { tableName } = breakExternalTableId(tableId) - if (!tableName) { - return related - } const table = this.tables[tableName] // @ts-ignore const primaryKey = table.primary[0] @@ -602,17 +595,19 @@ export class ExternalRequest { async run(config: RunConfig): Promise> { const { operation, tableId } = this - let { datasourceId, tableName } = breakExternalTableId(tableId) - if (!tableName) { - throw "Unable to run without a table name" + if (!tableId) { + throw new Error("Unable to run without a table ID") } + let { datasourceId, tableName } = breakExternalTableId(tableId) if (!this.datasource) { - await this.retrieveMetadata(datasourceId!) + await this.retrieveMetadata(datasourceId) } const table = this.tables[tableName] let isSql = isSQL(this.datasource!) if (!table) { - throw `Unable to process query, table "${tableName}" not defined.` + throw new Error( + `Unable to process query, table "${tableName}" not defined.` + ) } // look for specific components of config which may not be considered acceptable let { id, row, filters, sort, paginate, rows } = cleanupConfig( diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 5b12b5c207..126b11d0c1 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -136,10 +136,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { const id = ctx.params.rowId const tableId = utils.getTableId(ctx) const { datasourceId, tableName } = breakExternalTableId(tableId) - const datasource: Datasource = await sdk.datasources.get(datasourceId!) - if (!tableName) { - ctx.throw(400, "Unable to find table.") - } + const datasource: Datasource = await sdk.datasources.get(datasourceId) if (!datasource || !datasource.entities) { ctx.throw(400, "Datasource has not been configured for plus API.") } @@ -163,7 +160,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { } const links = row[fieldName] const linkedTableId = field.tableId - const linkedTableName = breakExternalTableId(linkedTableId).tableName! + const linkedTableName = breakExternalTableId(linkedTableId).tableName const linkedTable = tables[linkedTableName] // don't support composite keys right now const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0]) diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index 6f7bdc7335..767916616c 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -2,6 +2,8 @@ import { DatasourcePlusQueryResponse, DSPlusOperation, FieldType, + isManyToOne, + isOneToMany, ManyToManyRelationshipFieldMetadata, RelationshipFieldMetadata, RelationshipsJson, @@ -93,12 +95,12 @@ export function buildExternalRelationships( ): RelationshipsJson[] { const relationships = [] for (let [fieldName, field] of Object.entries(table.schema)) { - if (field.type !== FieldType.LINK) { + if (field.type !== FieldType.LINK || !field.tableId) { continue } const { tableName: linkTableName } = breakExternalTableId(field.tableId) // no table to link to, this is not a valid relationships - if (!linkTableName || !tables[linkTableName]) { + if (!tables[linkTableName]) { continue } const linkTable = tables[linkTableName] @@ -110,7 +112,7 @@ export function buildExternalRelationships( // need to specify where to put this back into column: fieldName, } - if (isManyToMany(field)) { + if (isManyToMany(field) && field.through) { const { tableName: throughTableName } = breakExternalTableId( field.through ) @@ -120,7 +122,7 @@ export function buildExternalRelationships( definition.to = field.throughFrom || linkTable.primary[0] definition.fromPrimary = table.primary[0] definition.toPrimary = linkTable.primary[0] - } else { + } else if (isManyToOne(field) || isOneToMany(field)) { // if no foreign key specified then use the name of the field in other table definition.from = field.foreignKey || table.primary[0] definition.to = field.fieldName @@ -180,16 +182,18 @@ export function buildSqlFieldList( } let fields = extractRealFields(table) for (let field of Object.values(table.schema)) { - if (field.type !== FieldType.LINK || !opts?.relationships) { + if ( + field.type !== FieldType.LINK || + !opts?.relationships || + !field.tableId + ) { continue } const { tableName: linkTableName } = breakExternalTableId(field.tableId) - if (linkTableName) { - const linkTable = tables[linkTableName] - if (linkTable) { - const linkedFields = extractRealFields(linkTable, fields) - fields = fields.concat(linkedFields) - } + const linkTable = tables[linkTableName] + if (linkTable) { + const linkedFields = extractRealFields(linkTable, fields) + fields = fields.concat(linkedFields) } } return fields diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index f1b186c233..fabc4d195a 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -18,8 +18,8 @@ import { builderSocket } from "../../../websockets" import { inputProcessing } from "../../../utilities/rowProcessor" function getDatasourceId(table: Table) { - if (!table) { - throw "No table supplied" + if (!table || !table._id) { + throw new Error("No table/table ID supplied") } if (table.sourceId) { return table.sourceId diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 9fc3487f62..93c46d8cc3 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -145,6 +145,10 @@ export async function exportRows( delimiter, customHeaders, } = options + + if (!tableId) { + throw new HTTPError("No table ID for search provided.", 400) + } const { datasourceId, tableName } = breakExternalTableId(tableId) let requestQuery: SearchFilters = {} @@ -167,7 +171,7 @@ export async function exportRows( requestQuery = query || {} } - const datasource = await sdk.datasources.get(datasourceId!) + const datasource = await sdk.datasources.get(datasourceId) const table = await sdk.tables.getTable(tableId) if (!datasource || !datasource.entities) { throw new HTTPError("Datasource has not been configured for plus API.", 400) @@ -180,10 +184,6 @@ export async function exportRows( let rows: Row[] = [] let headers - if (!tableName) { - throw new HTTPError("Could not find table name.", 400) - } - // Filter data to only specified columns if required if (columns && columns.length) { for (let i = 0; i < result.rows.length; i++) { diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 355493579d..738e57eff8 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -90,10 +90,10 @@ export async function getExternalTable( export async function getTable(tableId: string): Promise { const db = context.getAppDB() let output: Table - if (isExternalTableID(tableId)) { + if (tableId && isExternalTableID(tableId)) { let { datasourceId, tableName } = breakExternalTableId(tableId) - const datasource = await datasources.get(datasourceId!) - const table = await getExternalTable(datasourceId!, tableName!) + const datasource = await datasources.get(datasourceId) + const table = await getExternalTable(datasourceId, tableName) output = { ...table, sql: isSQL(datasource) } } else { output = await db.get
(tableId) diff --git a/packages/server/src/sdk/app/views/external.ts b/packages/server/src/sdk/app/views/external.ts index 0f96bcc061..2b3e271597 100644 --- a/packages/server/src/sdk/app/views/external.ts +++ b/packages/server/src/sdk/app/views/external.ts @@ -10,9 +10,9 @@ export async function get(viewId: string): Promise { const { tableId } = utils.extractViewInfoFromID(viewId) const { datasourceId, tableName } = breakExternalTableId(tableId) - const ds = await sdk.datasources.get(datasourceId!) + const ds = await sdk.datasources.get(datasourceId) - const table = ds.entities![tableName!] + const table = ds.entities![tableName] const views = Object.values(table.views!).filter(isV2) const found = views.find(v => v.id === viewId) if (!found) { @@ -25,9 +25,9 @@ export async function getEnriched(viewId: string): Promise { const { tableId } = utils.extractViewInfoFromID(viewId) const { datasourceId, tableName } = breakExternalTableId(tableId) - const ds = await sdk.datasources.get(datasourceId!) + const ds = await sdk.datasources.get(datasourceId) - const table = ds.entities![tableName!] + const table = ds.entities![tableName] const views = Object.values(table.views!).filter(isV2) const found = views.find(v => v.id === viewId) if (!found) { @@ -49,9 +49,9 @@ export async function create( const db = context.getAppDB() const { datasourceId, tableName } = breakExternalTableId(tableId) - const ds = await sdk.datasources.get(datasourceId!) - ds.entities![tableName!].views ??= {} - ds.entities![tableName!].views![view.name] = view + const ds = await sdk.datasources.get(datasourceId) + ds.entities![tableName].views ??= {} + ds.entities![tableName].views![view.name] = view await db.put(ds) return view } @@ -60,9 +60,9 @@ export async function update(tableId: string, view: ViewV2): Promise { const db = context.getAppDB() const { datasourceId, tableName } = breakExternalTableId(tableId) - const ds = await sdk.datasources.get(datasourceId!) - ds.entities![tableName!].views ??= {} - const views = ds.entities![tableName!].views! + const ds = await sdk.datasources.get(datasourceId) + ds.entities![tableName].views ??= {} + const views = ds.entities![tableName].views! const existingView = Object.values(views).find( v => isV2(v) && v.id === view.id @@ -87,9 +87,9 @@ export async function remove(viewId: string): Promise { } const { datasourceId, tableName } = breakExternalTableId(view.tableId) - const ds = await sdk.datasources.get(datasourceId!) + const ds = await sdk.datasources.get(datasourceId) - delete ds.entities![tableName!].views![view?.name] + delete ds.entities![tableName].views![view?.name] await db.put(ds) return view } From 75cee3c4fda301ef9fe6a1b3b5953270c34b48e8 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 24 Jun 2024 13:28:13 +0100 Subject: [PATCH 25/27] Quick type improvement. --- .../src/api/controllers/row/ExternalRequest.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 7cbe023fd2..1ce8d29e5f 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -170,8 +170,7 @@ export class ExternalRequest { id: string | undefined | string[], filters: SearchFilters, table: Table - ) { - const tables = this.tableList + ): SearchFilters { // replace any relationship columns initially, table names and relationship column names are acceptable const relationshipColumns = sdk.rows.filters.getRelationshipColumns(table) filters = sdk.rows.filters.updateFilterKeys( @@ -214,7 +213,7 @@ export class ExternalRequest { if (!Array.isArray(idCopy)) { idCopy = breakRowIdField(idCopy) } - const equal: any = {} + const equal: SearchFilters["equal"] = {} if (primary && idCopy) { for (let field of primary) { // work through the ID and get the parts @@ -274,7 +273,9 @@ export class ExternalRequest { } // seeds the object with table and datasource information - async retrieveMetadata(datasourceId: string) { + async retrieveMetadata( + datasourceId: string + ): Promise<{ tables: Record; datasource: Datasource }> { if (!this.datasource) { this.datasource = await sdk.datasources.get(datasourceId) if (!this.datasource || !this.datasource.entities) { @@ -283,6 +284,7 @@ export class ExternalRequest { this.tables = this.datasource.entities this.tableList = Object.values(this.tables) } + return { tables: this.tables, datasource: this.datasource } } async getRow(table: Table, rowId: string): Promise { @@ -599,11 +601,13 @@ export class ExternalRequest { throw new Error("Unable to run without a table ID") } let { datasourceId, tableName } = breakExternalTableId(tableId) - if (!this.datasource) { - await this.retrieveMetadata(datasourceId) + let datasource = this.datasource + if (!datasource) { + const { datasource: ds } = await this.retrieveMetadata(datasourceId) + datasource = ds } const table = this.tables[tableName] - let isSql = isSQL(this.datasource!) + let isSql = isSQL(datasource) if (!table) { throw new Error( `Unable to process query, table "${tableName}" not defined.` From b597bd3dbec1e8a5f2ba35da348fd7eb21609dae Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 24 Jun 2024 13:30:18 +0100 Subject: [PATCH 26/27] Fixing an issue detected by tests. --- packages/server/src/api/controllers/table/external.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index fabc4d195a..6ca8cdd82c 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -18,12 +18,15 @@ import { builderSocket } from "../../../websockets" import { inputProcessing } from "../../../utilities/rowProcessor" function getDatasourceId(table: Table) { - if (!table || !table._id) { - throw new Error("No table/table ID supplied") + if (!table) { + throw new Error("No table supplied") } if (table.sourceId) { return table.sourceId } + if (!table._id) { + throw new Error("No table ID supplied") + } return breakExternalTableId(table._id).datasourceId } From aefe46b253e02a2a7b456c38088e7a634a06ee93 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 24 Jun 2024 14:31:27 +0100 Subject: [PATCH 27/27] Adds _id and _rev back to internal datasource filter options (#13977) * Adds _id and _rev back to internal datasource filter options * add bb default datasource const into shared-core * re-export var from shared-core --- packages/backend-core/src/constants/db.ts | 2 +- .../src/components/FilterBuilder.svelte | 21 ++++++++++++++++--- packages/frontend-core/src/constants.js | 6 +++++- packages/shared-core/src/constants/index.ts | 2 ++ 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index 2fd713119b..3085b91ef1 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -72,4 +72,4 @@ export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs" export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory" export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses" export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee" -export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default" +export { DEFAULT_BB_DATASOURCE_ID } from "@budibase/shared-core" diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte index 0d254186f2..6d1e1fa502 100644 --- a/packages/frontend-core/src/components/FilterBuilder.svelte +++ b/packages/frontend-core/src/components/FilterBuilder.svelte @@ -18,7 +18,7 @@ import FilterUsers from "./FilterUsers.svelte" import { getFields } from "../utils/searchFields" - const { OperatorOptions } = Constants + const { OperatorOptions, DEFAULT_BB_DATASOURCE_ID } = Constants export let schemaFields export let filters = [] @@ -28,6 +28,23 @@ export let allowBindings = false export let filtersLabel = "Filters" + $: { + if ( + tables.find( + table => + table._id === datasource.tableId && + table.sourceId === DEFAULT_BB_DATASOURCE_ID + ) && + !schemaFields.some(field => field.name === "_id") + ) { + schemaFields = [ + ...schemaFields, + { name: "_id", type: "string" }, + { name: "_rev", type: "string" }, + ] + } + } + $: matchAny = filters?.find(filter => filter.operator === "allOr") != null $: onEmptyFilter = filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all" @@ -35,7 +52,6 @@ $: fieldFilters = filters.filter( filter => filter.operator !== "allOr" && !filter.onEmptyFilter ) - const behaviourOptions = [ { value: "and", label: "Match all filters" }, { value: "or", label: "Match any filter" }, @@ -44,7 +60,6 @@ { value: "all", label: "Return all table rows" }, { value: "none", label: "Return no rows" }, ] - const context = getContext("context") $: fieldOptions = getFields(tables, schemaFields || [], { diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js index 0d6261f5f8..e5869a3b98 100644 --- a/packages/frontend-core/src/constants.js +++ b/packages/frontend-core/src/constants.js @@ -1,7 +1,11 @@ /** * Operator options for lucene queries */ -export { OperatorOptions, SqlNumberTypeRangeMap } from "@budibase/shared-core" +export { + OperatorOptions, + SqlNumberTypeRangeMap, + DEFAULT_BB_DATASOURCE_ID, +} from "@budibase/shared-core" export { Feature as Features } from "@budibase/types" import { BpmCorrelationKey } from "@budibase/shared-core" import { FieldType, BBReferenceFieldSubType } from "@budibase/types" diff --git a/packages/shared-core/src/constants/index.ts b/packages/shared-core/src/constants/index.ts index 084bc8fe9e..c9d1a8fc8f 100644 --- a/packages/shared-core/src/constants/index.ts +++ b/packages/shared-core/src/constants/index.ts @@ -180,3 +180,5 @@ export enum BpmStatusValue { VERIFYING_EMAIL = "verifying_email", COMPLETED = "completed", } + +export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"