diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5cd28f4506..0d797d646a 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2166,4 +2166,47 @@ describe.each([ }) } ) + + describe.only.each([ + "名前", // Japanese for "name" + "Benutzer-ID", // German for "user ID", includes a hyphen + "numéro", // French for "number", includes an accent + "år", // Swedish for "year", includes a ring above + "naïve", // English word borrowed from French, includes an umlaut + "الاسم", // Arabic for "name" + "оплата", // Russian for "payment" + "पता", // Hindi for "address" + "用戶名", // Chinese for "username" + "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla + "preço", // Portuguese for "price", includes a cedilla + "사용자명", // Korean for "username" + "usuario_ñoño", // Spanish, uses an underscore and includes "ñ" + "файл", // Bulgarian for "file" + "δεδομένα", // Greek for "data" + "geändert_am", // German for "modified on", includes an umlaut + "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore + "São_Paulo", // Portuguese, includes an underscore and a tilde + "età", // Italian for "age", includes an accent + "ชื่อผู้ใช้", // Thai for "username" + ])("non-ascii column name: %s", name => { + beforeAll(async () => { + table = await createTable({ + [name]: { + name, + type: FieldType.STRING, + }, + }) + await createRows([{ [name]: "a" }, { [name]: "b" }]) + }) + + it("should be able to query a column with non-ascii characters", async () => { + await expectSearch({ + query: { + equal: { + [`1:${name}`]: "a", + }, + }, + }).toContainExactly([{ [name]: "a" }]) + }) + }) }) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index e3aedf9de8..2c9ee1356c 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -18,7 +18,11 @@ import { buildInternalRelationships, sqlOutputProcessing, } from "../../../../api/controllers/row/utils" -import { mapToUserColumn, USER_COLUMN_PREFIX } from "../../tables/internal/sqs" +import { + decodeNonAscii, + mapToUserColumn, + USER_COLUMN_PREFIX, +} from "../../tables/internal/sqs" import sdk from "../../../index" import { context, @@ -150,7 +154,8 @@ function reverseUserColumnMapping(rows: Row[]) { if (index !== -1) { // cut out the prefix const newKey = key.slice(0, index) + key.slice(index + prefixLength) - finalRow[newKey] = row[key] + const decoded = decodeNonAscii(newKey) + finalRow[decoded] = row[key] } else { finalRow[key] = row[key] } diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts index f892a9c6c8..9e831f4af7 100644 --- a/packages/server/src/sdk/app/tables/internal/sqs.ts +++ b/packages/server/src/sdk/app/tables/internal/sqs.ts @@ -64,10 +64,29 @@ function buildRelationshipDefinitions( export const USER_COLUMN_PREFIX = "data_" +// SQS does not support non-ASCII characters in column names, so we need to +// replace them with unicode escape sequences. +function encodeNonAscii(str: string): string { + return str + .split("") + .map(char => { + return char.charCodeAt(0) > 127 + ? "\\u" + char.charCodeAt(0).toString(16).padStart(4, "0") + : char + }) + .join("") +} + +export function decodeNonAscii(str: string): string { + return str.replace(/\\u([0-9a-fA-F]{4})/g, (match, p1) => + String.fromCharCode(parseInt(p1, 16)) + ) +} + // utility function to denote that columns in SQLite are mapped to avoid overlap issues // the overlaps can occur due to case insensitivity and some of the columns which Budibase requires export function mapToUserColumn(key: string) { - return `${USER_COLUMN_PREFIX}${key}` + return `${USER_COLUMN_PREFIX}${encodeNonAscii(key)}` } // this can generate relationship tables as part of the mapping