From 4f05f33b08f681a8a74f9292c6a69cb9f858ceae Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 22 Jul 2024 15:17:34 +0100 Subject: [PATCH 1/3] Fixing the issue, making sure if error occurs to re-evaluate. --- packages/server/src/sdk/app/rows/search/sqs.ts | 7 +++++-- packages/server/src/sdk/app/tables/internal/sqs.ts | 9 +++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index dada90b1be..44fd718871 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -49,6 +49,7 @@ import { dataFilters } from "@budibase/shared-core" const builder = new sql.Sql(SqlClient.SQL_LITE) const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`) const MISSING_TABLE_REGX = new RegExp(`no such table: .+`) +const DUPLICATE_COLUMN_REGEX = new RegExp(`duplicate column name: .+`) function buildInternalFieldList( table: Table, @@ -237,9 +238,11 @@ function resyncDefinitionsRequired(status: number, message: string) { // pre data_ prefix on column names, need to resync return ( // there are tables missing - try a resync - (status === 400 && message.match(MISSING_TABLE_REGX)) || + (status === 400 && message?.match(MISSING_TABLE_REGX)) || // there are columns missing - try a resync - (status === 400 && message.match(MISSING_COLUMN_REGEX)) || + (status === 400 && message?.match(MISSING_COLUMN_REGEX)) || + // duplicate column name in definitions - need to re-run definition sync + (status === 400 && message?.match(DUPLICATE_COLUMN_REGEX)) || // no design document found, needs a full sync (status === 404 && message?.includes(SQLITE_DESIGN_DOC_ID)) ) diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts index 9db10f2b41..3c14e2fc67 100644 --- a/packages/server/src/sdk/app/tables/internal/sqs.ts +++ b/packages/server/src/sdk/app/tables/internal/sqs.ts @@ -94,6 +94,9 @@ export function mapToUserColumn(key: string) { function mapTable(table: Table): SQLiteTables { const tables: SQLiteTables = {} const fields: Record = {} + // a list to make sure no duplicates - the fields are mapped by SQS with case sensitivity + // but need to make sure there are no duplicate columns + const usedColumns: string[] = [] for (let [key, column] of Object.entries(table.schema)) { // relationships should be handled differently if (column.type === FieldType.LINK) { @@ -106,6 +109,12 @@ function mapTable(table: Table): SQLiteTables { if (!FieldTypeMap[column.type]) { throw new Error(`Unable to map type "${column.type}" to SQLite type`) } + const lcKey = key.toLowerCase() + // ignore duplicates + if (usedColumns.includes(lcKey)) { + continue + } + usedColumns.push(lcKey) fields[mapToUserColumn(key)] = { field: key, type: FieldTypeMap[column.type], From 3247e13a32c4ca3b4d09928630074231a4f35211 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 22 Jul 2024 15:17:42 +0100 Subject: [PATCH 2/3] Test case. --- .../src/api/routes/tests/search.spec.ts | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 68f61ba28d..ee804c9914 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -6,9 +6,11 @@ import { } from "../../../integrations/tests/utils" import { db as dbCore, + context, MAX_VALID_DATE, MIN_VALID_DATE, utils, + SQLITE_DESIGN_DOC_ID, } from "@budibase/backend-core" import * as setup from "./utilities" @@ -2524,4 +2526,38 @@ describe.each([ }).toContainExactly([{ [" name"]: "foo" }]) }) }) + + isSqs && + describe("duplicate columns", () => { + beforeAll(async () => { + table = await createTable({ + name: { + name: "name", + type: FieldType.STRING, + }, + }) + await context.doInAppContext(config.appId!, async () => { + const db = context.getAppDB() + const tableDoc = await db.get(table._id!) + tableDoc.schema.Name = { + name: "Name", + type: FieldType.STRING, + } + try { + // remove the SQLite definitions so that they can be rebuilt as part of the search + const sqliteDoc = await db.get(SQLITE_DESIGN_DOC_ID) + await db.remove(sqliteDoc) + } catch (err) { + // no-op + } + }) + await createRows([{ name: "foo", Name: "bar" }]) + }) + + it("should handle invalid duplicate column names", async () => { + await expectSearch({ + query: {}, + }).toContainExactly([{ name: "foo" }]) + }) + }) }) From 3ed1c43915015d96e044d17f9b1471b1feaf972f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 22 Jul 2024 16:47:37 +0100 Subject: [PATCH 3/3] PR comments. --- packages/server/src/api/routes/tests/search.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index ee804c9914..e774158c23 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2536,7 +2536,7 @@ describe.each([ type: FieldType.STRING, }, }) - await context.doInAppContext(config.appId!, async () => { + await context.doInAppContext(config.getAppId(), async () => { const db = context.getAppDB() const tableDoc = await db.get
(table._id!) tableDoc.schema.Name = {