Merge remote-tracking branch 'origin/master' into feature/create-automations-in-data-section

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.29.11",
+  "version": "2.29.13",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/package.json

@@ -22,10 +22,9 @@
   },
   "dependencies": {
     "@budibase/nano": "10.1.5",
-    "@budibase/pouchdb-replication-stream": "1.2.10",
+    "@budibase/pouchdb-replication-stream": "1.2.11",
     "@budibase/shared-core": "0.0.0",
     "@budibase/types": "0.0.0",
-    "@govtechsg/passport-openidconnect": "^1.0.2",
     "aws-cloudfront-sign": "3.0.2",
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.1.0",

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -80,6 +80,11 @@ export function DatabaseWithConnection(
   connection: string,
   opts?: DatabaseOpts
 ) {
+  if (!dbName || !connection) {
+    throw new Error(
+      "Unable to create database without database name or connection"
+    )
+  }
   const db = new DatabaseImpl(dbName, opts, connection)
   return new DDInstrumentedDatabase(db)
 }

packages/builder/package.json

@@ -74,7 +74,7 @@
     "lodash": "4.17.21",
     "posthog-js": "^1.118.0",
     "remixicon": "2.5.0",
-    "sanitize-html": "^2.7.0",
+    "sanitize-html": "^2.13.0",
     "shortid": "2.2.15",
     "svelte-dnd-action": "^0.9.8",
     "svelte-loading-spinners": "^0.1.1",

packages/builder/src/pages/builder/portal/users/users/_components/EmailTableRenderer.svelte

@@ -5,7 +5,17 @@
   export let row
 </script>
 
-{value}
+<span class="email">
+  {value}
+</span>
 {#if row.scimInfo?.isSync}
   <ActiveDirectoryInfo iconSize="XS" />
 {/if}
+
+<style>
+  .email {
+    text-overflow: ellipsis;
+    overflow: hidden;
+    white-space: nowrap;
+  }
+</style>

packages/builder/vite.config.js

@@ -82,7 +82,7 @@ export default defineConfig(({ mode }) => {
       ...(isProduction ? [] : devOnlyPlugins),
     ],
     optimizeDeps: {
-      exclude: ["@roxi/routify"],
+      exclude: ["@roxi/routify", "fsevents"],
     },
     resolve: {
       dedupe: ["@roxi/routify"],

packages/cli/package.json

@@ -30,7 +30,7 @@
     "node-fetch": "2.6.7",
     "posthog-node": "1.3.0",
     "pouchdb": "7.3.0",
-    "pouchdb-replication-stream": "1.2.9",
+    "@budibase/pouchdb-replication-stream": "1.2.11",
     "randomstring": "1.1.5",
     "tar": "6.2.1",
     "yaml": "^2.1.1"

packages/cli/src/core/db.ts

@@ -17,7 +17,7 @@ export function getPouch(url?: string) {
       prefix: url,
     }
   }
-  const replicationStream = require("pouchdb-replication-stream")
+  const replicationStream = require("@budibase/pouchdb-replication-stream")
   PouchDB.plugin(replicationStream.plugin)
   // @ts-ignore
   PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)

packages/client/package.json

@@ -30,7 +30,7 @@
     "downloadjs": "1.4.7",
     "html5-qrcode": "^2.2.1",
     "leaflet": "^1.7.1",
-    "sanitize-html": "^2.7.0",
+    "sanitize-html": "^2.13.0",
     "screenfull": "^6.0.1",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^4.0.1",

packages/frontend-core/package.json

@@ -12,6 +12,6 @@
     "dayjs": "^1.10.8",
     "lodash": "4.17.21",
     "shortid": "2.2.15",
-    "socket.io-client": "^4.6.1"
+    "socket.io-client": "^4.7.5"
   }
 }

packages/pro

@@ -1 +1 @@
-Subproject commit dbb78c8737c291871500bc655e30f331f6ffccbf
+Subproject commit 11379517b76264a7f938c2d520bd259f586edada

packages/sdk/package.json

@@ -14,7 +14,6 @@
     "@rollup/plugin-commonjs": "^25.0.7",
     "@rollup/plugin-node-resolve": "^15.2.3",
     "rollup": "^4.9.6",
-    "rollup-plugin-terser": "^7.0.2",
     "rollup-plugin-polyfill-node": "^0.13.0"
   }
 }

packages/server/package.json

@@ -58,7 +58,7 @@
     "@bull-board/api": "5.10.2",
     "@bull-board/koa": "5.10.2",
     "@elastic/elasticsearch": "7.10.0",
-    "@google-cloud/firestore": "6.8.0",
+    "@google-cloud/firestore": "7.8.0",
     "@koa/router": "8.0.8",
     "@socket.io/redis-adapter": "^8.2.1",
     "@types/xml2js": "^0.4.14",
@@ -81,7 +81,7 @@
     "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.2",
     "ioredis": "5.3.2",
     "isolated-vm": "^4.7.2",
-    "jimp": "0.22.10",
+    "jimp": "0.22.12",
     "joi": "17.6.0",
     "js-yaml": "4.1.0",
     "jsonschema": "1.4.0",
@@ -109,7 +109,7 @@
     "serialize-error": "^7.0.1",
     "server-destroy": "1.0.1",
     "snowflake-promise": "^4.5.0",
-    "socket.io": "4.6.2",
+    "socket.io": "4.7.5",
     "tar": "6.2.1",
     "to-json-schema": "0.2.5",
     "uuid": "^8.3.2",

packages/server/src/api/routes/tests/search.spec.ts

@@ -2166,4 +2166,47 @@ describe.each([
       })
     }
   )
+
+  describe.each([
+    "名前", // Japanese for "name"
+    "Benutzer-ID", // German for "user ID", includes a hyphen
+    "numéro", // French for "number", includes an accent
+    "år", // Swedish for "year", includes a ring above
+    "naïve", // English word borrowed from French, includes an umlaut
+    "الاسم", // Arabic for "name"
+    "оплата", // Russian for "payment"
+    "पता", // Hindi for "address"
+    "用戶名", // Chinese for "username"
+    "çalışma_zamanı", // Turkish for "runtime", includes an underscore and a cedilla
+    "preço", // Portuguese for "price", includes a cedilla
+    "사용자명", // Korean for "username"
+    "usuario_ñoño", // Spanish, uses an underscore and includes "ñ"
+    "файл", // Bulgarian for "file"
+    "δεδομένα", // Greek for "data"
+    "geändert_am", // German for "modified on", includes an umlaut
+    "ব্যবহারকারীর_নাম", // Bengali for "user name", includes an underscore
+    "São_Paulo", // Portuguese, includes an underscore and a tilde
+    "età", // Italian for "age", includes an accent
+    "ชื่อผู้ใช้", // Thai for "username"
+  ])("non-ascii column name: %s", name => {
+    beforeAll(async () => {
+      table = await createTable({
+        [name]: {
+          name,
+          type: FieldType.STRING,
+        },
+      })
+      await createRows([{ [name]: "a" }, { [name]: "b" }])
+    })
+
+    it("should be able to query a column with non-ascii characters", async () => {
+      await expectSearch({
+        query: {
+          equal: {
+            [`1:${name}`]: "a",
+          },
+        },
+      }).toContainExactly([{ [name]: "a" }])
+    })
+  })
 })

packages/server/src/integrations/couchdb.ts

@@ -71,6 +71,9 @@ class CouchDBIntegration implements IntegrationBase {
   private readonly client: Database
 
   constructor(config: CouchDBConfig) {
+    if (!config.url || !config.database) {
+      throw new Error("Unable to connect without URL or database")
+    }
     this.client = dbCore.DatabaseWithConnection(config.database, config.url)
   }
 
@@ -79,45 +82,30 @@ class CouchDBIntegration implements IntegrationBase {
       connected: false,
     }
     try {
-      const result = await this.query("exists", "validation error", {})
+      response.connected = await this.client.exists()
-      response.connected = result === true
     } catch (e: any) {
       response.error = e.message as string
     }
     return response
   }
 
-  async query(
-    command: string,
-    errorMsg: string,
-    query: { json?: object; id?: string }
-  ) {
-    try {
-      return await (this.client as any)[command](query.id || query.json)
-    } catch (err) {
-      console.error(errorMsg, err)
-      throw err
-    }
-  }
-
   private parse(query: { json: string | object }) {
     return typeof query.json === "string" ? JSON.parse(query.json) : query.json
   }
 
   async create(query: { json: string | object }) {
     const parsed = this.parse(query)
-    return this.query("post", "Error writing to couchDB", { json: parsed })
+    return await this.client.put(parsed)
   }
 
   async read(query: { json: string | object }) {
     const parsed = this.parse(query)
-    const result = await this.query("allDocs", "Error querying couchDB", {
+    const params = {
-      json: {
+      include_docs: true,
-        include_docs: true,
+      ...parsed,
-        ...parsed,
+    }
-      },
+    const result = await this.client.allDocs(params)
-    })
+    return result.rows.map(row => row.doc)
-    return result.rows.map((row: { doc: object }) => row.doc)
   }
 
   async update(query: { json: string | object }) {
@@ -126,22 +114,15 @@ class CouchDBIntegration implements IntegrationBase {
       const oldDoc = await this.get({ id: parsed._id })
       parsed._rev = oldDoc._rev
     }
-    return this.query("put", "Error updating couchDB document", {
+    return await this.client.put(parsed)
-      json: parsed,
-    })
   }
 
   async get(query: { id: string }) {
-    return this.query("get", "Error retrieving couchDB document by ID", {
+    return await this.client.get(query.id)
-      id: query.id,
-    })
   }
 
   async delete(query: { id: string }) {
-    const doc = await this.query("get", "Cannot find doc to be deleted", query)
+    return await this.client.remove(query.id)
-    return this.query("remove", "Error deleting couchDB document", {
-      json: doc,
-    })
   }
 }
 

packages/server/src/integrations/tests/couchdb.spec.ts

@@ -6,7 +6,6 @@ jest.mock("@budibase/backend-core", () => {
       ...core.db,
       DatabaseWithConnection: function () {
         return {
-          post: jest.fn(),
           allDocs: jest.fn().mockReturnValue({ rows: [] }),
           put: jest.fn(),
           get: jest.fn().mockReturnValue({ _rev: "a" }),
@@ -43,7 +42,7 @@ describe("CouchDB Integration", () => {
     await config.integration.create({
       json: JSON.stringify(doc),
     })
-    expect(config.integration.client.post).toHaveBeenCalledWith(doc)
+    expect(config.integration.client.put).toHaveBeenCalledWith(doc)
   })
 
   it("calls the read method with the correct params", async () => {
@@ -80,7 +79,6 @@ describe("CouchDB Integration", () => {
   it("calls the delete method with the correct params", async () => {
     const id = "1234"
     await config.integration.delete({ id })
-    expect(config.integration.client.get).toHaveBeenCalledWith(id)
+    expect(config.integration.client.remove).toHaveBeenCalledWith(id)
-    expect(config.integration.client.remove).toHaveBeenCalled()
   })
 })

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -18,7 +18,11 @@ import {
   buildInternalRelationships,
   sqlOutputProcessing,
 } from "../../../../api/controllers/row/utils"
-import { mapToUserColumn, USER_COLUMN_PREFIX } from "../../tables/internal/sqs"
+import {
+  decodeNonAscii,
+  mapToUserColumn,
+  USER_COLUMN_PREFIX,
+} from "../../tables/internal/sqs"
 import sdk from "../../../index"
 import {
   context,
@@ -150,7 +154,8 @@ function reverseUserColumnMapping(rows: Row[]) {
       if (index !== -1) {
         // cut out the prefix
         const newKey = key.slice(0, index) + key.slice(index + prefixLength)
-        finalRow[newKey] = row[key]
+        const decoded = decodeNonAscii(newKey)
+        finalRow[decoded] = row[key]
       } else {
         finalRow[key] = row[key]
       }

packages/server/src/sdk/app/tables/internal/sqs.ts

@@ -64,10 +64,29 @@ function buildRelationshipDefinitions(
 
 export const USER_COLUMN_PREFIX = "data_"
 
+// SQS does not support non-ASCII characters in column names, so we need to
+// replace them with unicode escape sequences.
+function encodeNonAscii(str: string): string {
+  return str
+    .split("")
+    .map(char => {
+      return char.charCodeAt(0) > 127
+        ? "\\u" + char.charCodeAt(0).toString(16).padStart(4, "0")
+        : char
+    })
+    .join("")
+}
+
+export function decodeNonAscii(str: string): string {
+  return str.replace(/\\u([0-9a-fA-F]{4})/g, (match, p1) =>
+    String.fromCharCode(parseInt(p1, 16))
+  )
+}
+
 // utility function to denote that columns in SQLite are mapped to avoid overlap issues
 // the overlaps can occur due to case insensitivity and some of the columns which Budibase requires
 export function mapToUserColumn(key: string) {
-  return `${USER_COLUMN_PREFIX}${key}`
+  return `${USER_COLUMN_PREFIX}${encodeNonAscii(key)}`
 }
 
 // this can generate relationship tables as part of the mapping

packages/string-templates/package.json

@@ -34,6 +34,7 @@
   "devDependencies": {
     "@rollup/plugin-commonjs": "^17.1.0",
     "@rollup/plugin-inject": "^5.0.5",
+    "@rollup/plugin-json": "^4.1.0",
     "@rollup/plugin-typescript": "8.3.0",
     "doctrine": "^3.0.0",
     "jest": "29.7.0",

packages/types/package.json

@@ -17,7 +17,6 @@
   "devDependencies": {
     "@budibase/nano": "10.1.5",
     "@types/koa": "2.13.4",
-    "@types/pouchdb": "6.4.0",
     "@types/redlock": "4.0.7",
     "rimraf": "3.0.2",
     "typescript": "5.5.2"

packages/worker/package.json

@@ -42,7 +42,7 @@
     "@budibase/string-templates": "0.0.0",
     "@budibase/types": "0.0.0",
     "@koa/router": "8.0.8",
-    "@techpass/passport-openidconnect": "0.3.2",
+    "@techpass/passport-openidconnect": "0.3.3",
     "@types/global-agent": "2.1.1",
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.1.0",
@@ -69,8 +69,6 @@
     "pouchdb": "7.3.0",
     "pouchdb-all-dbs": "1.1.1",
     "server-destroy": "1.0.1",
-    "undici": "^6.0.1",
-    "undici-types": "^6.0.1",
     "knex": "2.4.2"
   },
   "devDependencies": {