Merge pull request #14094 from Budibase/support-non-ascii-in-sqs

Support non-ascii column in SQS.

packages/server/src/api/routes/tests/search.spec.ts

@@ -2166,4 +2166,47 @@ describe.each([
       })
     }
   )
+
+  describe.each([
+    "名前", // Japanese for "name"
+    "Benutzer-ID", // German for "user ID", includes a hyphen
+    "numéro", // French for "number", includes an accent
+    "år", // Swedish for "year", includes a ring above
+    "naïve", // English word borrowed from French, includes an umlaut
+    "الاسم", // Arabic for "name"
+    "оплата", // Russian for "payment"
+    "पता", // Hindi for "address"
+    "用戶名", // Chinese for "username"
+    "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla
+    "preço", // Portuguese for "price", includes a cedilla
+    "사용자명", // Korean for "username"
+    "usuario_ñoño", // Spanish, uses an underscore and includes "ñ"
+    "файл", // Bulgarian for "file"
+    "δεδομένα", // Greek for "data"
+    "geändert_am", // German for "modified on", includes an umlaut
+    "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore
+    "São_Paulo", // Portuguese, includes an underscore and a tilde
+    "età", // Italian for "age", includes an accent
+    "ชื่อผู้ใช้", // Thai for "username"
+  ])("non-ascii column name: %s", name => {
+    beforeAll(async () => {
+      table = await createTable({
+        [name]: {
+          name,
+          type: FieldType.STRING,
+        },
+      })
+      await createRows([{ [name]: "a" }, { [name]: "b" }])
+    })
+
+    it("should be able to query a column with non-ascii characters", async () => {
+      await expectSearch({
+        query: {
+          equal: {
+            [`1:${name}`]: "a",
+          },
+        },
+      }).toContainExactly([{ [name]: "a" }])
+    })
+  })
 })

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -18,7 +18,11 @@ import {
   buildInternalRelationships,
   sqlOutputProcessing,
 } from "../../../../api/controllers/row/utils"
-import { mapToUserColumn, USER_COLUMN_PREFIX } from "../../tables/internal/sqs"
+import {
+  decodeNonAscii,
+  mapToUserColumn,
+  USER_COLUMN_PREFIX,
+} from "../../tables/internal/sqs"
 import sdk from "../../../index"
 import {
   context,
@@ -150,7 +154,8 @@ function reverseUserColumnMapping(rows: Row[]) {
       if (index !== -1) {
         // cut out the prefix
         const newKey = key.slice(0, index) + key.slice(index + prefixLength)
-        finalRow[newKey] = row[key]
+        const decoded = decodeNonAscii(newKey)
+        finalRow[decoded] = row[key]
       } else {
         finalRow[key] = row[key]
       }

packages/server/src/sdk/app/tables/internal/sqs.ts

@@ -64,10 +64,29 @@ function buildRelationshipDefinitions(
 
 export const USER_COLUMN_PREFIX = "data_"
 
+// SQS does not support non-ASCII characters in column names, so we need to
+// replace them with unicode escape sequences.
+function encodeNonAscii(str: string): string {
+  return str
+    .split("")
+    .map(char => {
+      return char.charCodeAt(0) > 127
+        ? "\\u" + char.charCodeAt(0).toString(16).padStart(4, "0")
+        : char
+    })
+    .join("")
+}
+
+export function decodeNonAscii(str: string): string {
+  return str.replace(/\\u([0-9a-fA-F]{4})/g, (match, p1) =>
+    String.fromCharCode(parseInt(p1, 16))
+  )
+}
+
 // utility function to denote that columns in SQLite are mapped to avoid overlap issues
 // the overlaps can occur due to case insensitivity and some of the columns which Budibase requires
 export function mapToUserColumn(key: string) {
-  return `${USER_COLUMN_PREFIX}${key}`
+  return `${USER_COLUMN_PREFIX}${encodeNonAscii(key)}`
 }
 
 // this can generate relationship tables as part of the mapping