From 253110ac6f696fcd03ccd725db928b73bbdbd986 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 3 Jul 2024 15:25:36 +0100 Subject: [PATCH 01/12] Detect secrets in error messages. --- packages/backend-core/src/environment.ts | 15 ++++++++ .../src/middleware/errorHandling.ts | 11 +++++- packages/backend-core/src/security/secrets.ts | 20 +++++++++++ .../src/security/tests/secrets.spec.ts | 35 +++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 packages/backend-core/src/security/secrets.ts create mode 100644 packages/backend-core/src/security/tests/secrets.spec.ts diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 685f2988ad..0296e1fd48 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -205,6 +205,21 @@ const environment = { OPENAI_API_KEY: process.env.OPENAI_API_KEY, } +type EnvironmentKey = keyof typeof environment +export const SECRETS: EnvironmentKey[] = [ + "API_ENCRYPTION_KEY", + "COUCH_DB_PASSWORD", + "COUCH_DB_SQL_URL", + "COUCH_DB_URL", + "GOOGLE_CLIENT_SECRET", + "INTERNAL_API_KEY_FALLBACK", + "INTERNAL_API_KEY", + "JWT_SECRET", + "MINIO_ACCESS_KEY", + "MINIO_SECRET_KEY", + "REDIS_PASSWORD", +] + // clean up any environment variable edge cases for (let [key, value] of Object.entries(environment)) { // handle the edge case of "0" to disable an environment variable diff --git a/packages/backend-core/src/middleware/errorHandling.ts b/packages/backend-core/src/middleware/errorHandling.ts index 08f9f3214d..3f6d069337 100644 --- a/packages/backend-core/src/middleware/errorHandling.ts +++ b/packages/backend-core/src/middleware/errorHandling.ts @@ -1,6 +1,7 @@ import { APIError } from "@budibase/types" import * as errors from "../errors" import environment from "../environment" +import { stringContainsSecret } from "../security/secrets" export async function errorHandling(ctx: any, next: any) { try { @@ -17,7 +18,7 @@ export async function errorHandling(ctx: any, next: any) { let error: APIError = { message: err.message, - status: status, + status, validationErrors: err.validation, error: errors.getPublicError(err), } @@ -27,6 +28,14 @@ export async function errorHandling(ctx: any, next: any) { error.stack = err.stack } + if (stringContainsSecret(JSON.stringify(error))) { + error = { + message: "Unexpected error", + status, + error: "Unexpected error", + } + } + ctx.body = error } } diff --git a/packages/backend-core/src/security/secrets.ts b/packages/backend-core/src/security/secrets.ts new file mode 100644 index 0000000000..24fb556bdb --- /dev/null +++ b/packages/backend-core/src/security/secrets.ts @@ -0,0 +1,20 @@ +import environment, { SECRETS } from "../environment" + +export function stringContainsSecret(str: string) { + if (str.includes("-----BEGIN PRIVATE KEY-----")) { + return true + } + + for (const key of SECRETS) { + const value = environment[key] + if (typeof value !== "string") { + continue + } + + if (str.includes(value)) { + return true + } + } + + return false +} diff --git a/packages/backend-core/src/security/tests/secrets.spec.ts b/packages/backend-core/src/security/tests/secrets.spec.ts new file mode 100644 index 0000000000..19bf174973 --- /dev/null +++ b/packages/backend-core/src/security/tests/secrets.spec.ts @@ -0,0 +1,35 @@ +import { randomUUID } from "crypto" +import environment, { SECRETS } from "../../environment" +import { stringContainsSecret } from "../secrets" + +describe("secrets", () => { + describe("stringContainsSecret", () => { + it.each(SECRETS)("detects that a string contains a secret in: %s", key => { + const needle = randomUUID() + const haystack = `this is a secret: ${needle}` + const old = environment[key] + environment._set(key, needle) + + try { + expect(stringContainsSecret(haystack)).toBe(true) + } finally { + environment._set(key, old) + } + }) + + it.each(SECRETS)( + "detects that a string does not contain a secret in: %s", + key => { + const needle = randomUUID() + const haystack = `this does not contain a secret` + const old = environment[key] + environment._set(key, needle) + try { + expect(stringContainsSecret(haystack)).toBe(false) + } finally { + environment._set(key, old) + } + } + ) + }) +}) From d9b94c1dcfc8a8b64b85fece38b7761ffa91a4f9 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 3 Jul 2024 15:35:46 +0100 Subject: [PATCH 02/12] Don't detect empty strings. --- packages/backend-core/src/security/secrets.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/src/security/secrets.ts b/packages/backend-core/src/security/secrets.ts index 24fb556bdb..65bc33a1dc 100644 --- a/packages/backend-core/src/security/secrets.ts +++ b/packages/backend-core/src/security/secrets.ts @@ -7,7 +7,7 @@ export function stringContainsSecret(str: string) { for (const key of SECRETS) { const value = environment[key] - if (typeof value !== "string") { + if (typeof value !== "string" || value === "") { continue } From bab3c077274d714ce59e23de0faf89ef2be9419f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 3 Jul 2024 16:33:32 +0100 Subject: [PATCH 03/12] Add a couple more secrets. --- packages/backend-core/src/environment.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 0296e1fd48..e06d51f918 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -208,6 +208,7 @@ const environment = { type EnvironmentKey = keyof typeof environment export const SECRETS: EnvironmentKey[] = [ "API_ENCRYPTION_KEY", + "BB_ADMIN_USER_PASSWORD", "COUCH_DB_PASSWORD", "COUCH_DB_SQL_URL", "COUCH_DB_URL", @@ -217,6 +218,7 @@ export const SECRETS: EnvironmentKey[] = [ "JWT_SECRET", "MINIO_ACCESS_KEY", "MINIO_SECRET_KEY", + "OPENAI_API_KEY", "REDIS_PASSWORD", ] From 16e293a9ff5919d7b077ef4f726d8bd0259b8f57 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 4 Jul 2024 09:55:36 +0100 Subject: [PATCH 04/12] Fix tests. --- packages/backend-core/src/middleware/errorHandling.ts | 10 +++++----- packages/backend-core/src/security/secrets.ts | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/middleware/errorHandling.ts b/packages/backend-core/src/middleware/errorHandling.ts index 3f6d069337..6ceda9cd3a 100644 --- a/packages/backend-core/src/middleware/errorHandling.ts +++ b/packages/backend-core/src/middleware/errorHandling.ts @@ -23,11 +23,6 @@ export async function errorHandling(ctx: any, next: any) { error: errors.getPublicError(err), } - if (environment.isTest() && ctx.headers["x-budibase-include-stacktrace"]) { - // @ts-ignore - error.stack = err.stack - } - if (stringContainsSecret(JSON.stringify(error))) { error = { message: "Unexpected error", @@ -36,6 +31,11 @@ export async function errorHandling(ctx: any, next: any) { } } + if (environment.isTest() && ctx.headers["x-budibase-include-stacktrace"]) { + // @ts-ignore + error.stack = err.stack + } + ctx.body = error } } diff --git a/packages/backend-core/src/security/secrets.ts b/packages/backend-core/src/security/secrets.ts index 65bc33a1dc..ab7ea36728 100644 --- a/packages/backend-core/src/security/secrets.ts +++ b/packages/backend-core/src/security/secrets.ts @@ -12,6 +12,7 @@ export function stringContainsSecret(str: string) { } if (str.includes(value)) { + throw new Error(`String contains secret: ${key}=${value}`) return true } } From 257ee8fb70247838f09fa6d44ce9e554595bfae4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 4 Jul 2024 10:46:09 +0100 Subject: [PATCH 05/12] Fix tests actually. --- packages/backend-core/src/security/secrets.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/backend-core/src/security/secrets.ts b/packages/backend-core/src/security/secrets.ts index ab7ea36728..65bc33a1dc 100644 --- a/packages/backend-core/src/security/secrets.ts +++ b/packages/backend-core/src/security/secrets.ts @@ -12,7 +12,6 @@ export function stringContainsSecret(str: string) { } if (str.includes(value)) { - throw new Error(`String contains secret: ${key}=${value}`) return true } } From b318850c7ebd46e3a8a022d13475faa010940198 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 4 Jul 2024 11:37:18 +0100 Subject: [PATCH 06/12] Support non-ascii column in SQS. --- .../src/api/routes/tests/search.spec.ts | 43 +++++++++++++++++++ .../server/src/sdk/app/rows/search/sqs.ts | 9 +++- .../server/src/sdk/app/tables/internal/sqs.ts | 21 ++++++++- 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5cd28f4506..0d797d646a 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2166,4 +2166,47 @@ describe.each([ }) } ) + + describe.only.each([ + "名前", // Japanese for "name" + "Benutzer-ID", // German for "user ID", includes a hyphen + "numéro", // French for "number", includes an accent + "år", // Swedish for "year", includes a ring above + "naïve", // English word borrowed from French, includes an umlaut + "الاسم", // Arabic for "name" + "оплата", // Russian for "payment" + "पता", // Hindi for "address" + "用戶名", // Chinese for "username" + "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla + "preço", // Portuguese for "price", includes a cedilla + "사용자명", // Korean for "username" + "usuario_ñoño", // Spanish, uses an underscore and includes "ñ" + "файл", // Bulgarian for "file" + "δεδομένα", // Greek for "data" + "geändert_am", // German for "modified on", includes an umlaut + "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore + "São_Paulo", // Portuguese, includes an underscore and a tilde + "età", // Italian for "age", includes an accent + "ชื่อผู้ใช้", // Thai for "username" + ])("non-ascii column name: %s", name => { + beforeAll(async () => { + table = await createTable({ + [name]: { + name, + type: FieldType.STRING, + }, + }) + await createRows([{ [name]: "a" }, { [name]: "b" }]) + }) + + it("should be able to query a column with non-ascii characters", async () => { + await expectSearch({ + query: { + equal: { + [`1:${name}`]: "a", + }, + }, + }).toContainExactly([{ [name]: "a" }]) + }) + }) }) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index e3aedf9de8..2c9ee1356c 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -18,7 +18,11 @@ import { buildInternalRelationships, sqlOutputProcessing, } from "../../../../api/controllers/row/utils" -import { mapToUserColumn, USER_COLUMN_PREFIX } from "../../tables/internal/sqs" +import { + decodeNonAscii, + mapToUserColumn, + USER_COLUMN_PREFIX, +} from "../../tables/internal/sqs" import sdk from "../../../index" import { context, @@ -150,7 +154,8 @@ function reverseUserColumnMapping(rows: Row[]) { if (index !== -1) { // cut out the prefix const newKey = key.slice(0, index) + key.slice(index + prefixLength) - finalRow[newKey] = row[key] + const decoded = decodeNonAscii(newKey) + finalRow[decoded] = row[key] } else { finalRow[key] = row[key] } diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts index f892a9c6c8..9e831f4af7 100644 --- a/packages/server/src/sdk/app/tables/internal/sqs.ts +++ b/packages/server/src/sdk/app/tables/internal/sqs.ts @@ -64,10 +64,29 @@ function buildRelationshipDefinitions( export const USER_COLUMN_PREFIX = "data_" +// SQS does not support non-ASCII characters in column names, so we need to +// replace them with unicode escape sequences. +function encodeNonAscii(str: string): string { + return str + .split("") + .map(char => { + return char.charCodeAt(0) > 127 + ? "\\u" + char.charCodeAt(0).toString(16).padStart(4, "0") + : char + }) + .join("") +} + +export function decodeNonAscii(str: string): string { + return str.replace(/\\u([0-9a-fA-F]{4})/g, (match, p1) => + String.fromCharCode(parseInt(p1, 16)) + ) +} + // utility function to denote that columns in SQLite are mapped to avoid overlap issues // the overlaps can occur due to case insensitivity and some of the columns which Budibase requires export function mapToUserColumn(key: string) { - return `${USER_COLUMN_PREFIX}${key}` + return `${USER_COLUMN_PREFIX}${encodeNonAscii(key)}` } // this can generate relationship tables as part of the mapping From ce406e0c5d9a70bee700d3084aeca9007bf32454 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 4 Jul 2024 11:43:13 +0100 Subject: [PATCH 07/12] Unfocus test, you numpty. --- packages/server/src/api/routes/tests/search.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 0d797d646a..145db9b4a3 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2167,7 +2167,7 @@ describe.each([ } ) - describe.only.each([ + describe.each([ "名前", // Japanese for "name" "Benutzer-ID", // German for "user ID", includes a hyphen "numéro", // French for "number", includes an accent From 788cce2b9a67c565486cd5db893602cc6385121b Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 4 Jul 2024 11:08:18 +0000 Subject: [PATCH 08/12] Bump version to 2.29.13 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index d991a1e813..c3419a3f87 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.29.12", + "version": "2.29.13", "npmClient": "yarn", "packages": [ "packages/*", From 6d412cdf32b0629ac1c7f1fd526781267d8f9f6a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 4 Jul 2024 13:54:08 +0200 Subject: [PATCH 09/12] Ellipsis on long emails --- .../users/_components/EmailTableRenderer.svelte | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/portal/users/users/_components/EmailTableRenderer.svelte b/packages/builder/src/pages/builder/portal/users/users/_components/EmailTableRenderer.svelte index 99ba5abc2f..e2d1671b60 100644 --- a/packages/builder/src/pages/builder/portal/users/users/_components/EmailTableRenderer.svelte +++ b/packages/builder/src/pages/builder/portal/users/users/_components/EmailTableRenderer.svelte @@ -5,7 +5,17 @@ export let row -{value} + {#if row.scimInfo?.isSync} {/if} + + From 8595a3ab40762d78627f0ca5c7d6aeb9ec7ccdbd Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 4 Jul 2024 13:55:30 +0100 Subject: [PATCH 10/12] Update account-portal submodule to latest master. --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index ff16525b73..b03e584e46 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit ff16525b73c5751d344f5c161a682609c0a993f2 +Subproject commit b03e584e465f620b49a1b688ff4afc973e6c0758 From 332bf0911675316607605a09448f9bb445e88c41 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 4 Jul 2024 14:50:59 +0100 Subject: [PATCH 11/12] Fixing up the typing in the gen collection script so that it is valid TS again and will work. --- .../scripts/gen-collection-info.ts | 49 ++++++++++++------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/packages/string-templates/scripts/gen-collection-info.ts b/packages/string-templates/scripts/gen-collection-info.ts index ae2a726661..d176665a5b 100644 --- a/packages/string-templates/scripts/gen-collection-info.ts +++ b/packages/string-templates/scripts/gen-collection-info.ts @@ -1,16 +1,23 @@ -const HELPER_LIBRARY = "@budibase/handlebars-helpers" -const helpers = require(HELPER_LIBRARY) -const { HelperFunctionBuiltin } = require("../src/helpers/constants") -const fs = require("fs") +import { HelperFunctionBuiltin } from "../src/helpers/constants" +import { readFileSync, writeFileSync } from "fs" +import { marked } from "marked" +import { join, dirname } from "path" + +const helpers = require("@budibase/handlebars-helpers") const doctrine = require("doctrine") -const marked = require("marked") + +type HelperInfo = { + acceptsInline?: boolean + acceptsBlock?: boolean + example?: string + description: string + tags?: any[] +} /** * full list of supported helpers can be found here: * https://github.com/budibase/handlebars-helpers */ -const { join } = require("path") -const path = require("path") const COLLECTIONS = [ "math", @@ -23,7 +30,7 @@ const COLLECTIONS = [ "uuid", ] const FILENAME = join(__dirname, "..", "src", "manifest.json") -const outputJSON = {} +const outputJSON: any = {} const ADDED_HELPERS = { date: { date: { @@ -43,7 +50,7 @@ const ADDED_HELPERS = { }, } -function fixSpecialCases(name, obj) { +function fixSpecialCases(name: string, obj: any) { const args = obj.args if (name === "ifNth") { args[0] = "a" @@ -61,7 +68,7 @@ function fixSpecialCases(name, obj) { return obj } -function lookForward(lines, funcLines, idx) { +function lookForward(lines: string[], funcLines: string[], idx: number) { const funcLen = funcLines.length for (let i = idx, j = 0; i < idx + funcLen; ++i, j++) { if (!lines[i].includes(funcLines[j])) { @@ -71,7 +78,7 @@ function lookForward(lines, funcLines, idx) { return true } -function getCommentInfo(file, func) { +function getCommentInfo(file: string, func: string): HelperInfo { const lines = file.split("\n") const funcLines = func.split("\n") let comment = null @@ -98,7 +105,13 @@ function getCommentInfo(file, func) { if (comment == null) { return { description: "" } } - const docs = doctrine.parse(comment, { unwrap: true }) + const docs: { + acceptsInline?: boolean + acceptsBlock?: boolean + example: string + description: string + tags: any[] + } = doctrine.parse(comment, { unwrap: true }) // some hacky fixes docs.description = docs.description.replace(/\n/g, " ") docs.description = docs.description.replace(/[ ]{2,}/g, " ") @@ -120,7 +133,7 @@ function getCommentInfo(file, func) { return docs } -const excludeFunctions = { string: ["raw"] } +const excludeFunctions: Record = { string: ["raw"] } /** * This script is very specific to purpose, parsing the handlebars-helpers files to attempt to get information about them. @@ -128,11 +141,13 @@ const excludeFunctions = { string: ["raw"] } function run() { const foundNames: string[] = [] for (let collection of COLLECTIONS) { - const collectionFile = fs.readFileSync( - `${path.dirname(require.resolve(HELPER_LIBRARY))}/lib/${collection}.js`, + const collectionFile = readFileSync( + `${dirname( + require.resolve("@budibase/handlebars-helpers") + )}/lib/${collection}.js`, "utf8" ) - const collectionInfo = {} + const collectionInfo: any = {} // collect information about helper let hbsHelperInfo = helpers[collection]() for (let entry of Object.entries(hbsHelperInfo)) { @@ -181,7 +196,7 @@ function run() { helper.description = marked.parse(helper.description) } } - fs.writeFileSync(FILENAME, JSON.stringify(outputJSON, null, 2)) + writeFileSync(FILENAME, JSON.stringify(outputJSON, null, 2)) } run() From 4fae45e018b3d792fbe3a69ccedbcd44b233fac3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 4 Jul 2024 16:49:35 +0200 Subject: [PATCH 12/12] Add tooltip --- .../portal/users/users/_components/EmailTableRenderer.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/portal/users/users/_components/EmailTableRenderer.svelte b/packages/builder/src/pages/builder/portal/users/users/_components/EmailTableRenderer.svelte index e2d1671b60..e68fa46071 100644 --- a/packages/builder/src/pages/builder/portal/users/users/_components/EmailTableRenderer.svelte +++ b/packages/builder/src/pages/builder/portal/users/users/_components/EmailTableRenderer.svelte @@ -5,7 +5,7 @@ export let row -