Merge pull request #14212 from Budibase/fix/dupe-columns-sqs

Fix for duplicate columns in SQS

packages/server/src/api/routes/tests/search.spec.ts

@@ -6,9 +6,11 @@ import {
 } from "../../../integrations/tests/utils"
 import {
   db as dbCore,
+  context,
   MAX_VALID_DATE,
   MIN_VALID_DATE,
   utils,
+  SQLITE_DESIGN_DOC_ID,
 } from "@budibase/backend-core"
 
 import * as setup from "./utilities"
@@ -2524,4 +2526,38 @@ describe.each([
         }).toContainExactly([{ [" name"]: "foo" }])
       })
     })
+
+  isSqs &&
+    describe("duplicate columns", () => {
+      beforeAll(async () => {
+        table = await createTable({
+          name: {
+            name: "name",
+            type: FieldType.STRING,
+          },
+        })
+        await context.doInAppContext(config.getAppId(), async () => {
+          const db = context.getAppDB()
+          const tableDoc = await db.get<Table>(table._id!)
+          tableDoc.schema.Name = {
+            name: "Name",
+            type: FieldType.STRING,
+          }
+          try {
+            // remove the SQLite definitions so that they can be rebuilt as part of the search
+            const sqliteDoc = await db.get(SQLITE_DESIGN_DOC_ID)
+            await db.remove(sqliteDoc)
+          } catch (err) {
+            // no-op
+          }
+        })
+        await createRows([{ name: "foo", Name: "bar" }])
+      })
+
+      it("should handle invalid duplicate column names", async () => {
+        await expectSearch({
+          query: {},
+        }).toContainExactly([{ name: "foo" }])
+      })
+    })
 })

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -49,6 +49,7 @@ import { dataFilters } from "@budibase/shared-core"
 const builder = new sql.Sql(SqlClient.SQL_LITE)
 const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`)
 const MISSING_TABLE_REGX = new RegExp(`no such table: .+`)
+const DUPLICATE_COLUMN_REGEX = new RegExp(`duplicate column name: .+`)
 
 function buildInternalFieldList(
   table: Table,
@@ -237,9 +238,11 @@ function resyncDefinitionsRequired(status: number, message: string) {
   // pre data_ prefix on column names, need to resync
   return (
     // there are tables missing - try a resync
-    (status === 400 && message.match(MISSING_TABLE_REGX)) ||
+    (status === 400 && message?.match(MISSING_TABLE_REGX)) ||
     // there are columns missing - try a resync
-    (status === 400 && message.match(MISSING_COLUMN_REGEX)) ||
+    (status === 400 && message?.match(MISSING_COLUMN_REGEX)) ||
+    // duplicate column name in definitions - need to re-run definition sync
+    (status === 400 && message?.match(DUPLICATE_COLUMN_REGEX)) ||
     // no design document found, needs a full sync
     (status === 404 && message?.includes(SQLITE_DESIGN_DOC_ID))
   )

packages/server/src/sdk/app/tables/internal/sqs.ts

@@ -94,6 +94,9 @@ export function mapToUserColumn(key: string) {
 function mapTable(table: Table): SQLiteTables {
   const tables: SQLiteTables = {}
   const fields: Record<string, { field: string; type: SQLiteType }> = {}
+  // a list to make sure no duplicates - the fields are mapped by SQS with case sensitivity
+  // but need to make sure there are no duplicate columns
+  const usedColumns: string[] = []
   for (let [key, column] of Object.entries(table.schema)) {
     // relationships should be handled differently
     if (column.type === FieldType.LINK) {
@@ -106,6 +109,12 @@ function mapTable(table: Table): SQLiteTables {
     if (!FieldTypeMap[column.type]) {
       throw new Error(`Unable to map type "${column.type}" to SQLite type`)
     }
+    const lcKey = key.toLowerCase()
+    // ignore duplicates
+    if (usedColumns.includes(lcKey)) {
+      continue
+    }
+    usedColumns.push(lcKey)
     fields[mapToUserColumn(key)] = {
       field: key,
       type: FieldTypeMap[column.type],