diff --git a/lerna.json b/lerna.json index d991a1e813..c3419a3f87 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.29.12", + "version": "2.29.13", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/account-portal b/packages/account-portal index ff16525b73..b03e584e46 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit ff16525b73c5751d344f5c161a682609c0a993f2 +Subproject commit b03e584e465f620b49a1b688ff4afc973e6c0758 diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 685f2988ad..e06d51f918 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -205,6 +205,23 @@ const environment = { OPENAI_API_KEY: process.env.OPENAI_API_KEY, } +type EnvironmentKey = keyof typeof environment +export const SECRETS: EnvironmentKey[] = [ + "API_ENCRYPTION_KEY", + "BB_ADMIN_USER_PASSWORD", + "COUCH_DB_PASSWORD", + "COUCH_DB_SQL_URL", + "COUCH_DB_URL", + "GOOGLE_CLIENT_SECRET", + "INTERNAL_API_KEY_FALLBACK", + "INTERNAL_API_KEY", + "JWT_SECRET", + "MINIO_ACCESS_KEY", + "MINIO_SECRET_KEY", + "OPENAI_API_KEY", + "REDIS_PASSWORD", +] + // clean up any environment variable edge cases for (let [key, value] of Object.entries(environment)) { // handle the edge case of "0" to disable an environment variable diff --git a/packages/backend-core/src/middleware/errorHandling.ts b/packages/backend-core/src/middleware/errorHandling.ts index 08f9f3214d..6ceda9cd3a 100644 --- a/packages/backend-core/src/middleware/errorHandling.ts +++ b/packages/backend-core/src/middleware/errorHandling.ts @@ -1,6 +1,7 @@ import { APIError } from "@budibase/types" import * as errors from "../errors" import environment from "../environment" +import { stringContainsSecret } from "../security/secrets" export async function errorHandling(ctx: any, next: any) { try { @@ -17,11 +18,19 @@ export async function errorHandling(ctx: any, next: any) { let error: APIError = { message: err.message, - status: status, + status, validationErrors: err.validation, error: errors.getPublicError(err), } + if (stringContainsSecret(JSON.stringify(error))) { + error = { + message: "Unexpected error", + status, + error: "Unexpected error", + } + } + if (environment.isTest() && ctx.headers["x-budibase-include-stacktrace"]) { // @ts-ignore error.stack = err.stack diff --git a/packages/backend-core/src/security/secrets.ts b/packages/backend-core/src/security/secrets.ts new file mode 100644 index 0000000000..65bc33a1dc --- /dev/null +++ b/packages/backend-core/src/security/secrets.ts @@ -0,0 +1,20 @@ +import environment, { SECRETS } from "../environment" + +export function stringContainsSecret(str: string) { + if (str.includes("-----BEGIN PRIVATE KEY-----")) { + return true + } + + for (const key of SECRETS) { + const value = environment[key] + if (typeof value !== "string" || value === "") { + continue + } + + if (str.includes(value)) { + return true + } + } + + return false +} diff --git a/packages/backend-core/src/security/tests/secrets.spec.ts b/packages/backend-core/src/security/tests/secrets.spec.ts new file mode 100644 index 0000000000..19bf174973 --- /dev/null +++ b/packages/backend-core/src/security/tests/secrets.spec.ts @@ -0,0 +1,35 @@ +import { randomUUID } from "crypto" +import environment, { SECRETS } from "../../environment" +import { stringContainsSecret } from "../secrets" + +describe("secrets", () => { + describe("stringContainsSecret", () => { + it.each(SECRETS)("detects that a string contains a secret in: %s", key => { + const needle = randomUUID() + const haystack = `this is a secret: ${needle}` + const old = environment[key] + environment._set(key, needle) + + try { + expect(stringContainsSecret(haystack)).toBe(true) + } finally { + environment._set(key, old) + } + }) + + it.each(SECRETS)( + "detects that a string does not contain a secret in: %s", + key => { + const needle = randomUUID() + const haystack = `this does not contain a secret` + const old = environment[key] + environment._set(key, needle) + try { + expect(stringContainsSecret(haystack)).toBe(false) + } finally { + environment._set(key, old) + } + } + ) + }) +}) diff --git a/packages/builder/src/pages/builder/portal/users/users/_components/EmailTableRenderer.svelte b/packages/builder/src/pages/builder/portal/users/users/_components/EmailTableRenderer.svelte index 99ba5abc2f..e68fa46071 100644 --- a/packages/builder/src/pages/builder/portal/users/users/_components/EmailTableRenderer.svelte +++ b/packages/builder/src/pages/builder/portal/users/users/_components/EmailTableRenderer.svelte @@ -5,7 +5,17 @@ export let row -{value} + + {value} + {#if row.scimInfo?.isSync} {/if} + + diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5cd28f4506..145db9b4a3 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2166,4 +2166,47 @@ describe.each([ }) } ) + + describe.each([ + "名前", // Japanese for "name" + "Benutzer-ID", // German for "user ID", includes a hyphen + "numéro", // French for "number", includes an accent + "år", // Swedish for "year", includes a ring above + "naïve", // English word borrowed from French, includes an umlaut + "الاسم", // Arabic for "name" + "оплата", // Russian for "payment" + "पता", // Hindi for "address" + "用戶名", // Chinese for "username" + "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla + "preço", // Portuguese for "price", includes a cedilla + "사용자명", // Korean for "username" + "usuario_ñoño", // Spanish, uses an underscore and includes "ñ" + "файл", // Bulgarian for "file" + "δεδομένα", // Greek for "data" + "geändert_am", // German for "modified on", includes an umlaut + "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore + "São_Paulo", // Portuguese, includes an underscore and a tilde + "età", // Italian for "age", includes an accent + "ชื่อผู้ใช้", // Thai for "username" + ])("non-ascii column name: %s", name => { + beforeAll(async () => { + table = await createTable({ + [name]: { + name, + type: FieldType.STRING, + }, + }) + await createRows([{ [name]: "a" }, { [name]: "b" }]) + }) + + it("should be able to query a column with non-ascii characters", async () => { + await expectSearch({ + query: { + equal: { + [`1:${name}`]: "a", + }, + }, + }).toContainExactly([{ [name]: "a" }]) + }) + }) }) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index e3aedf9de8..2c9ee1356c 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -18,7 +18,11 @@ import { buildInternalRelationships, sqlOutputProcessing, } from "../../../../api/controllers/row/utils" -import { mapToUserColumn, USER_COLUMN_PREFIX } from "../../tables/internal/sqs" +import { + decodeNonAscii, + mapToUserColumn, + USER_COLUMN_PREFIX, +} from "../../tables/internal/sqs" import sdk from "../../../index" import { context, @@ -150,7 +154,8 @@ function reverseUserColumnMapping(rows: Row[]) { if (index !== -1) { // cut out the prefix const newKey = key.slice(0, index) + key.slice(index + prefixLength) - finalRow[newKey] = row[key] + const decoded = decodeNonAscii(newKey) + finalRow[decoded] = row[key] } else { finalRow[key] = row[key] } diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts index f892a9c6c8..9e831f4af7 100644 --- a/packages/server/src/sdk/app/tables/internal/sqs.ts +++ b/packages/server/src/sdk/app/tables/internal/sqs.ts @@ -64,10 +64,29 @@ function buildRelationshipDefinitions( export const USER_COLUMN_PREFIX = "data_" +// SQS does not support non-ASCII characters in column names, so we need to +// replace them with unicode escape sequences. +function encodeNonAscii(str: string): string { + return str + .split("") + .map(char => { + return char.charCodeAt(0) > 127 + ? "\\u" + char.charCodeAt(0).toString(16).padStart(4, "0") + : char + }) + .join("") +} + +export function decodeNonAscii(str: string): string { + return str.replace(/\\u([0-9a-fA-F]{4})/g, (match, p1) => + String.fromCharCode(parseInt(p1, 16)) + ) +} + // utility function to denote that columns in SQLite are mapped to avoid overlap issues // the overlaps can occur due to case insensitivity and some of the columns which Budibase requires export function mapToUserColumn(key: string) { - return `${USER_COLUMN_PREFIX}${key}` + return `${USER_COLUMN_PREFIX}${encodeNonAscii(key)}` } // this can generate relationship tables as part of the mapping diff --git a/packages/string-templates/scripts/gen-collection-info.ts b/packages/string-templates/scripts/gen-collection-info.ts index ae2a726661..d176665a5b 100644 --- a/packages/string-templates/scripts/gen-collection-info.ts +++ b/packages/string-templates/scripts/gen-collection-info.ts @@ -1,16 +1,23 @@ -const HELPER_LIBRARY = "@budibase/handlebars-helpers" -const helpers = require(HELPER_LIBRARY) -const { HelperFunctionBuiltin } = require("../src/helpers/constants") -const fs = require("fs") +import { HelperFunctionBuiltin } from "../src/helpers/constants" +import { readFileSync, writeFileSync } from "fs" +import { marked } from "marked" +import { join, dirname } from "path" + +const helpers = require("@budibase/handlebars-helpers") const doctrine = require("doctrine") -const marked = require("marked") + +type HelperInfo = { + acceptsInline?: boolean + acceptsBlock?: boolean + example?: string + description: string + tags?: any[] +} /** * full list of supported helpers can be found here: * https://github.com/budibase/handlebars-helpers */ -const { join } = require("path") -const path = require("path") const COLLECTIONS = [ "math", @@ -23,7 +30,7 @@ const COLLECTIONS = [ "uuid", ] const FILENAME = join(__dirname, "..", "src", "manifest.json") -const outputJSON = {} +const outputJSON: any = {} const ADDED_HELPERS = { date: { date: { @@ -43,7 +50,7 @@ const ADDED_HELPERS = { }, } -function fixSpecialCases(name, obj) { +function fixSpecialCases(name: string, obj: any) { const args = obj.args if (name === "ifNth") { args[0] = "a" @@ -61,7 +68,7 @@ function fixSpecialCases(name, obj) { return obj } -function lookForward(lines, funcLines, idx) { +function lookForward(lines: string[], funcLines: string[], idx: number) { const funcLen = funcLines.length for (let i = idx, j = 0; i < idx + funcLen; ++i, j++) { if (!lines[i].includes(funcLines[j])) { @@ -71,7 +78,7 @@ function lookForward(lines, funcLines, idx) { return true } -function getCommentInfo(file, func) { +function getCommentInfo(file: string, func: string): HelperInfo { const lines = file.split("\n") const funcLines = func.split("\n") let comment = null @@ -98,7 +105,13 @@ function getCommentInfo(file, func) { if (comment == null) { return { description: "" } } - const docs = doctrine.parse(comment, { unwrap: true }) + const docs: { + acceptsInline?: boolean + acceptsBlock?: boolean + example: string + description: string + tags: any[] + } = doctrine.parse(comment, { unwrap: true }) // some hacky fixes docs.description = docs.description.replace(/\n/g, " ") docs.description = docs.description.replace(/[ ]{2,}/g, " ") @@ -120,7 +133,7 @@ function getCommentInfo(file, func) { return docs } -const excludeFunctions = { string: ["raw"] } +const excludeFunctions: Record = { string: ["raw"] } /** * This script is very specific to purpose, parsing the handlebars-helpers files to attempt to get information about them. @@ -128,11 +141,13 @@ const excludeFunctions = { string: ["raw"] } function run() { const foundNames: string[] = [] for (let collection of COLLECTIONS) { - const collectionFile = fs.readFileSync( - `${path.dirname(require.resolve(HELPER_LIBRARY))}/lib/${collection}.js`, + const collectionFile = readFileSync( + `${dirname( + require.resolve("@budibase/handlebars-helpers") + )}/lib/${collection}.js`, "utf8" ) - const collectionInfo = {} + const collectionInfo: any = {} // collect information about helper let hbsHelperInfo = helpers[collection]() for (let entry of Object.entries(hbsHelperInfo)) { @@ -181,7 +196,7 @@ function run() { helper.description = marked.parse(helper.description) } } - fs.writeFileSync(FILENAME, JSON.stringify(outputJSON, null, 2)) + writeFileSync(FILENAME, JSON.stringify(outputJSON, null, 2)) } run()