Merge branch 'master' of github.com:Budibase/budibase into dependabot/npm_and_yarn/browserify-sign-4.2.3

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.29.12",
+  "version": "2.29.13",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/account-portal

@@ -1 +1 @@
-Subproject commit ff16525b73c5751d344f5c161a682609c0a993f2
+Subproject commit b03e584e465f620b49a1b688ff4afc973e6c0758

packages/backend-core/src/environment.ts

@@ -205,6 +205,23 @@ const environment = {
   OPENAI_API_KEY: process.env.OPENAI_API_KEY,
 }
 
+type EnvironmentKey = keyof typeof environment
+export const SECRETS: EnvironmentKey[] = [
+  "API_ENCRYPTION_KEY",
+  "BB_ADMIN_USER_PASSWORD",
+  "COUCH_DB_PASSWORD",
+  "COUCH_DB_SQL_URL",
+  "COUCH_DB_URL",
+  "GOOGLE_CLIENT_SECRET",
+  "INTERNAL_API_KEY_FALLBACK",
+  "INTERNAL_API_KEY",
+  "JWT_SECRET",
+  "MINIO_ACCESS_KEY",
+  "MINIO_SECRET_KEY",
+  "OPENAI_API_KEY",
+  "REDIS_PASSWORD",
+]
+
 // clean up any environment variable edge cases
 for (let [key, value] of Object.entries(environment)) {
   // handle the edge case of "0" to disable an environment variable

packages/backend-core/src/middleware/errorHandling.ts

@@ -1,6 +1,7 @@
 import { APIError } from "@budibase/types"
 import * as errors from "../errors"
 import environment from "../environment"
+import { stringContainsSecret } from "../security/secrets"
 
 export async function errorHandling(ctx: any, next: any) {
   try {
@@ -17,11 +18,19 @@ export async function errorHandling(ctx: any, next: any) {
 
     let error: APIError = {
       message: err.message,
-      status: status,
+      status,
       validationErrors: err.validation,
       error: errors.getPublicError(err),
     }
 
+    if (stringContainsSecret(JSON.stringify(error))) {
+      error = {
+        message: "Unexpected error",
+        status,
+        error: "Unexpected error",
+      }
+    }
+
     if (environment.isTest() && ctx.headers["x-budibase-include-stacktrace"]) {
       // @ts-ignore
       error.stack = err.stack

packages/backend-core/src/security/secrets.ts

@@ -0,0 +1,20 @@
+import environment, { SECRETS } from "../environment"
+
+export function stringContainsSecret(str: string) {
+  if (str.includes("-----BEGIN PRIVATE KEY-----")) {
+    return true
+  }
+
+  for (const key of SECRETS) {
+    const value = environment[key]
+    if (typeof value !== "string" || value === "") {
+      continue
+    }
+
+    if (str.includes(value)) {
+      return true
+    }
+  }
+
+  return false
+}

packages/backend-core/src/security/tests/secrets.spec.ts

@@ -0,0 +1,35 @@
+import { randomUUID } from "crypto"
+import environment, { SECRETS } from "../../environment"
+import { stringContainsSecret } from "../secrets"
+
+describe("secrets", () => {
+  describe("stringContainsSecret", () => {
+    it.each(SECRETS)("detects that a string contains a secret in: %s", key => {
+      const needle = randomUUID()
+      const haystack = `this is a secret: ${needle}`
+      const old = environment[key]
+      environment._set(key, needle)
+
+      try {
+        expect(stringContainsSecret(haystack)).toBe(true)
+      } finally {
+        environment._set(key, old)
+      }
+    })
+
+    it.each(SECRETS)(
+      "detects that a string does not contain a secret in: %s",
+      key => {
+        const needle = randomUUID()
+        const haystack = `this does not contain a secret`
+        const old = environment[key]
+        environment._set(key, needle)
+        try {
+          expect(stringContainsSecret(haystack)).toBe(false)
+        } finally {
+          environment._set(key, old)
+        }
+      }
+    )
+  })
+})

packages/builder/src/pages/builder/portal/users/users/_components/EmailTableRenderer.svelte

@@ -5,7 +5,17 @@
   export let row
 </script>
 
+<span title={value} class="email">
   {value}
+</span>
 {#if row.scimInfo?.isSync}
   <ActiveDirectoryInfo iconSize="XS" />
 {/if}
+
+<style>
+  .email {
+    text-overflow: ellipsis;
+    overflow: hidden;
+    white-space: nowrap;
+  }
+</style>

packages/server/src/api/routes/tests/search.spec.ts

@@ -2166,4 +2166,47 @@ describe.each([
       })
     }
   )
+
+  describe.each([
+    "名前", // Japanese for "name"
+    "Benutzer-ID", // German for "user ID", includes a hyphen
+    "numéro", // French for "number", includes an accent
+    "år", // Swedish for "year", includes a ring above
+    "naïve", // English word borrowed from French, includes an umlaut
+    "الاسم", // Arabic for "name"
+    "оплата", // Russian for "payment"
+    "पता", // Hindi for "address"
+    "用戶名", // Chinese for "username"
+    "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla
+    "preço", // Portuguese for "price", includes a cedilla
+    "사용자명", // Korean for "username"
+    "usuario_ñoño", // Spanish, uses an underscore and includes "ñ"
+    "файл", // Bulgarian for "file"
+    "δεδομένα", // Greek for "data"
+    "geändert_am", // German for "modified on", includes an umlaut
+    "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore
+    "São_Paulo", // Portuguese, includes an underscore and a tilde
+    "età", // Italian for "age", includes an accent
+    "ชื่อผู้ใช้", // Thai for "username"
+  ])("non-ascii column name: %s", name => {
+    beforeAll(async () => {
+      table = await createTable({
+        [name]: {
+          name,
+          type: FieldType.STRING,
+        },
+      })
+      await createRows([{ [name]: "a" }, { [name]: "b" }])
+    })
+
+    it("should be able to query a column with non-ascii characters", async () => {
+      await expectSearch({
+        query: {
+          equal: {
+            [`1:${name}`]: "a",
+          },
+        },
+      }).toContainExactly([{ [name]: "a" }])
+    })
+  })
 })

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -18,7 +18,11 @@ import {
   buildInternalRelationships,
   sqlOutputProcessing,
 } from "../../../../api/controllers/row/utils"
-import { mapToUserColumn, USER_COLUMN_PREFIX } from "../../tables/internal/sqs"
+import {
+  decodeNonAscii,
+  mapToUserColumn,
+  USER_COLUMN_PREFIX,
+} from "../../tables/internal/sqs"
 import sdk from "../../../index"
 import {
   context,
@@ -150,7 +154,8 @@ function reverseUserColumnMapping(rows: Row[]) {
       if (index !== -1) {
         // cut out the prefix
         const newKey = key.slice(0, index) + key.slice(index + prefixLength)
-        finalRow[newKey] = row[key]
+        const decoded = decodeNonAscii(newKey)
+        finalRow[decoded] = row[key]
       } else {
         finalRow[key] = row[key]
       }

packages/server/src/sdk/app/tables/internal/sqs.ts

@@ -64,10 +64,29 @@ function buildRelationshipDefinitions(
 
 export const USER_COLUMN_PREFIX = "data_"
 
+// SQS does not support non-ASCII characters in column names, so we need to
+// replace them with unicode escape sequences.
+function encodeNonAscii(str: string): string {
+  return str
+    .split("")
+    .map(char => {
+      return char.charCodeAt(0) > 127
+        ? "\\u" + char.charCodeAt(0).toString(16).padStart(4, "0")
+        : char
+    })
+    .join("")
+}
+
+export function decodeNonAscii(str: string): string {
+  return str.replace(/\\u([0-9a-fA-F]{4})/g, (match, p1) =>
+    String.fromCharCode(parseInt(p1, 16))
+  )
+}
+
 // utility function to denote that columns in SQLite are mapped to avoid overlap issues
 // the overlaps can occur due to case insensitivity and some of the columns which Budibase requires
 export function mapToUserColumn(key: string) {
-  return `${USER_COLUMN_PREFIX}${key}`
+  return `${USER_COLUMN_PREFIX}${encodeNonAscii(key)}`
 }
 
 // this can generate relationship tables as part of the mapping

packages/string-templates/scripts/gen-collection-info.ts

@@ -1,16 +1,23 @@
-const HELPER_LIBRARY = "@budibase/handlebars-helpers"
+import { HelperFunctionBuiltin } from "../src/helpers/constants"
-const helpers = require(HELPER_LIBRARY)
+import { readFileSync, writeFileSync } from "fs"
-const { HelperFunctionBuiltin } = require("../src/helpers/constants")
+import { marked } from "marked"
-const fs = require("fs")
+import { join, dirname } from "path"
+
+const helpers = require("@budibase/handlebars-helpers")
 const doctrine = require("doctrine")
-const marked = require("marked")
+
+type HelperInfo = {
+  acceptsInline?: boolean
+  acceptsBlock?: boolean
+  example?: string
+  description: string
+  tags?: any[]
+}
 
 /**
  * full list of supported helpers can be found here:
  * https://github.com/budibase/handlebars-helpers
  */
-const { join } = require("path")
-const path = require("path")
 
 const COLLECTIONS = [
   "math",
@@ -23,7 +30,7 @@ const COLLECTIONS = [
   "uuid",
 ]
 const FILENAME = join(__dirname, "..", "src", "manifest.json")
-const outputJSON = {}
+const outputJSON: any = {}
 const ADDED_HELPERS = {
   date: {
     date: {
@@ -43,7 +50,7 @@ const ADDED_HELPERS = {
   },
 }
 
-function fixSpecialCases(name, obj) {
+function fixSpecialCases(name: string, obj: any) {
   const args = obj.args
   if (name === "ifNth") {
     args[0] = "a"
@@ -61,7 +68,7 @@ function fixSpecialCases(name, obj) {
   return obj
 }
 
-function lookForward(lines, funcLines, idx) {
+function lookForward(lines: string[], funcLines: string[], idx: number) {
   const funcLen = funcLines.length
   for (let i = idx, j = 0; i < idx + funcLen; ++i, j++) {
     if (!lines[i].includes(funcLines[j])) {
@@ -71,7 +78,7 @@ function lookForward(lines, funcLines, idx) {
   return true
 }
 
-function getCommentInfo(file, func) {
+function getCommentInfo(file: string, func: string): HelperInfo {
   const lines = file.split("\n")
   const funcLines = func.split("\n")
   let comment = null
@@ -98,7 +105,13 @@ function getCommentInfo(file, func) {
   if (comment == null) {
     return { description: "" }
   }
-  const docs = doctrine.parse(comment, { unwrap: true })
+  const docs: {
+    acceptsInline?: boolean
+    acceptsBlock?: boolean
+    example: string
+    description: string
+    tags: any[]
+  } = doctrine.parse(comment, { unwrap: true })
   // some hacky fixes
   docs.description = docs.description.replace(/\n/g, " ")
   docs.description = docs.description.replace(/[ ]{2,}/g, " ")
@@ -120,7 +133,7 @@ function getCommentInfo(file, func) {
   return docs
 }
 
-const excludeFunctions = { string: ["raw"] }
+const excludeFunctions: Record<string, string[]> = { string: ["raw"] }
 
 /**
  * This script is very specific to purpose, parsing the handlebars-helpers files to attempt to get information about them.
@@ -128,11 +141,13 @@ const excludeFunctions = { string: ["raw"] }
 function run() {
   const foundNames: string[] = []
   for (let collection of COLLECTIONS) {
-    const collectionFile = fs.readFileSync(
+    const collectionFile = readFileSync(
-      `${path.dirname(require.resolve(HELPER_LIBRARY))}/lib/${collection}.js`,
+      `${dirname(
+        require.resolve("@budibase/handlebars-helpers")
+      )}/lib/${collection}.js`,
       "utf8"
     )
-    const collectionInfo = {}
+    const collectionInfo: any = {}
     // collect information about helper
     let hbsHelperInfo = helpers[collection]()
     for (let entry of Object.entries(hbsHelperInfo)) {
@@ -181,7 +196,7 @@ function run() {
       helper.description = marked.parse(helper.description)
     }
   }
-  fs.writeFileSync(FILENAME, JSON.stringify(outputJSON, null, 2))
+  writeFileSync(FILENAME, JSON.stringify(outputJSON, null, 2))
 }
 
 run()