Merge pull request #15460 from Budibase/BUDI-9020/fix-isolatedvm-query-issues

Fix isolatedvm query issues

.prettierignore

@@ -9,4 +9,5 @@ packages/backend-core/coverage
 packages/builder/.routify
 packages/sdk/sdk
 packages/pro/coverage
-**/*.ivm.bundle.js
+**/*.ivm.bundle.js
+!**/bson-polyfills.ivm.bundle.js

packages/server/src/api/routes/tests/queries/mongodb.spec.ts

@@ -634,6 +634,130 @@ if (descriptions.length) {
             }
           })
         })
+
+        it("should be able to select a ObjectId in a transformer", async () => {
+          const query = await createQuery({
+            fields: {
+              json: {},
+              extra: {
+                actionType: "find",
+              },
+            },
+            transformer: "return data.map(x => ({ id: x._id }))",
+          })
+
+          const result = await config.api.query.execute(query._id!)
+
+          expect(result.data).toEqual([
+            { id: expectValidId },
+            { id: expectValidId },
+            { id: expectValidId },
+            { id: expectValidId },
+            { id: expectValidId },
+          ])
+        })
+
+        it("can handle all bson field types with transformers", async () => {
+          collection = generator.guid()
+          await withCollection(async collection => {
+            await collection.insertOne({
+              _id: new BSON.ObjectId("65b0123456789abcdef01234"),
+              stringField: "This is a string",
+              numberField: 42,
+              doubleField: new BSON.Double(42.42),
+              integerField: new BSON.Int32(123),
+              longField: new BSON.Long("9223372036854775807"),
+              booleanField: true,
+              nullField: null,
+              arrayField: [1, 2, 3, "four", { nested: true }],
+              objectField: {
+                nestedString: "nested",
+                nestedNumber: 99,
+              },
+              dateField: new Date(Date.UTC(2025, 0, 30, 12, 30, 20)),
+              timestampField: new BSON.Timestamp({ t: 1706616000, i: 1 }),
+              binaryField: new BSON.Binary(
+                new TextEncoder().encode("bufferValue")
+              ),
+              objectIdField: new BSON.ObjectId("65b0123456789abcdef01235"),
+              regexField: new BSON.BSONRegExp("^Hello.*", "i"),
+              minKeyField: new BSON.MinKey(),
+              maxKeyField: new BSON.MaxKey(),
+              decimalField: new BSON.Decimal128("12345.6789"),
+              codeField: new BSON.Code(
+                "function() { return 'Hello, World!'; }"
+              ),
+              codeWithScopeField: new BSON.Code(
+                "function(x) { return x * 2; }",
+                { x: 10 }
+              ),
+            })
+          })
+
+          const query = await createQuery({
+            fields: {
+              json: {},
+              extra: {
+                actionType: "find",
+                collection,
+              },
+            },
+            transformer: `return data.map(x => ({ 
+                  ...x,
+                  binaryField: x.binaryField?.toString('utf8'),
+                  decimalField: x.decimalField.toString(),
+                  longField: x.longField.toString(),
+                  regexField: x.regexField.toString(),
+                  // TODO: currenlty not supported, it looks like there is bug in the library. Getting: Timestamp constructed from { t, i } must provide t as a number
+                  timestampField: null
+              }))`,
+          })
+
+          const result = await config.api.query.execute(query._id!)
+
+          expect(result.data).toEqual([
+            {
+              _id: "65b0123456789abcdef01234",
+              arrayField: [
+                1,
+                2,
+                3,
+                "four",
+                {
+                  nested: true,
+                },
+              ],
+              binaryField: "bufferValue",
+              booleanField: true,
+              codeField: {
+                code: "function() { return 'Hello, World!'; }",
+              },
+              codeWithScopeField: {
+                code: "function(x) { return x * 2; }",
+                scope: {
+                  x: 10,
+                },
+              },
+              dateField: "2025-01-30T12:30:20.000Z",
+              decimalField: "12345.6789",
+              doubleField: 42.42,
+              integerField: 123,
+              longField: "9223372036854775807",
+              maxKeyField: {},
+              minKeyField: {},
+              nullField: null,
+              numberField: 42,
+              objectField: {
+                nestedNumber: 99,
+                nestedString: "nested",
+              },
+              objectIdField: "65b0123456789abcdef01235",
+              regexField: "/^Hello.*/i",
+              stringField: "This is a string",
+              timestampField: null,
+            },
+          ])
+        })
       })
 
       it("should throw an error if the incorrect actionType is specified", async () => {

packages/server/src/jsRunner/bundles/bson-polyfills.ivm.bundle.js

@@ -0,0 +1,122 @@
+if (typeof btoa !== "function") {
+  var chars = {
+    ascii: function () {
+      return "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
+    },
+    indices: function () {
+      if (!this.cache) {
+        this.cache = {}
+        var ascii = chars.ascii()
+
+        for (var c = 0; c < ascii.length; c++) {
+          var chr = ascii[c]
+          this.cache[chr] = c
+        }
+      }
+      return this.cache
+    },
+  }
+
+  function atob(b64) {
+    var indices = chars.indices(),
+      pos = b64.indexOf("="),
+      padded = pos > -1,
+      len = padded ? pos : b64.length,
+      i = -1,
+      data = ""
+
+    while (i < len) {
+      var code =
+        (indices[b64[++i]] << 18) |
+        (indices[b64[++i]] << 12) |
+        (indices[b64[++i]] << 6) |
+        indices[b64[++i]]
+      if (code !== 0) {
+        data += String.fromCharCode(
+          (code >>> 16) & 255,
+          (code >>> 8) & 255,
+          code & 255
+        )
+      }
+    }
+
+    if (padded) {
+      data = data.slice(0, pos - b64.length)
+    }
+
+    return data
+  }
+
+  function btoa(data) {
+    var ascii = chars.ascii(),
+      len = data.length - 1,
+      i = -1,
+      b64 = ""
+
+    while (i < len) {
+      var code =
+        (data.charCodeAt(++i) << 16) |
+        (data.charCodeAt(++i) << 8) |
+        data.charCodeAt(++i)
+      b64 +=
+        ascii[(code >>> 18) & 63] +
+        ascii[(code >>> 12) & 63] +
+        ascii[(code >>> 6) & 63] +
+        ascii[code & 63]
+    }
+
+    var pads = data.length % 3
+    if (pads > 0) {
+      b64 = b64.slice(0, pads - 3)
+
+      while (b64.length % 4 !== 0) {
+        b64 += "="
+      }
+    }
+
+    return b64
+  }
+}
+
+if (typeof TextDecoder === "undefined") {
+  globalThis.TextDecoder = class {
+    constructor(encoding = "utf8") {
+      if (encoding !== "utf8") {
+        throw new Error(
+          `Only UTF-8 is supported in this polyfill. Recieved: ${encoding}`
+        )
+      }
+    }
+    decode(buffer) {
+      return String.fromCharCode(...buffer)
+    }
+  }
+}
+
+if (typeof TextEncoder === "undefined") {
+  globalThis.TextEncoder = class {
+    encode(str) {
+      const utf8 = []
+      for (const i = 0; i < str.length; i++) {
+        const codePoint = str.charCodeAt(i)
+
+        if (codePoint < 0x80) {
+          utf8.push(codePoint)
+        } else if (codePoint < 0x800) {
+          utf8.push(0xc0 | (codePoint >> 6))
+          utf8.push(0x80 | (codePoint & 0x3f))
+        } else if (codePoint < 0x10000) {
+          utf8.push(0xe0 | (codePoint >> 12))
+          utf8.push(0x80 | ((codePoint >> 6) & 0x3f))
+          utf8.push(0x80 | (codePoint & 0x3f))
+        } else {
+          utf8.push(0xf0 | (codePoint >> 18))
+          utf8.push(0x80 | ((codePoint >> 12) & 0x3f))
+          utf8.push(0x80 | ((codePoint >> 6) & 0x3f))
+          utf8.push(0x80 | (codePoint & 0x3f))
+        }
+      }
+      return new Uint8Array(utf8)
+    }
+  }
+}

packages/server/src/jsRunner/bundles/index.ts

@@ -5,6 +5,7 @@ export const enum BundleType {
   BSON = "bson",
   SNIPPETS = "snippets",
   BUFFER = "buffer",
+  BSON_POLYFILLS = "bson_polyfills",
 }
 
 const bundleSourceFile: Record<BundleType, string> = {
@@ -12,6 +13,7 @@ const bundleSourceFile: Record<BundleType, string> = {
   [BundleType.BSON]: "./bson.ivm.bundle.js",
   [BundleType.SNIPPETS]: "./snippets.ivm.bundle.js",
   [BundleType.BUFFER]: "./buffer.ivm.bundle.js",
+  [BundleType.BSON_POLYFILLS]: "./bson-polyfills.ivm.bundle.js",
 }
 const bundleSourceCode: Partial<Record<BundleType, string>> = {}
 

packages/server/src/jsRunner/bundles/snippets.ts

@@ -1,6 +1,5 @@
 // @ts-ignore
-// eslint-disable-next-line local-rules/no-budibase-imports
-import { iifeWrapper } from "@budibase/string-templates/iife"
+import { iifeWrapper } from "@budibase/string-templates"
 
 export default new Proxy(
   {},

packages/server/src/jsRunner/vm/isolated-vm.ts

@@ -161,42 +161,10 @@ export class IsolatedVM implements VM {
 
     const bsonSource = loadBundle(BundleType.BSON)
 
-    this.addToContext({
-      textDecoderCb: new ivm.Callback(
-        (args: {
-          constructorArgs: any
-          functionArgs: Parameters<InstanceType<typeof TextDecoder>["decode"]>
-        }) => {
-          const result = new TextDecoder(...args.constructorArgs).decode(
-            ...args.functionArgs
-          )
-          return result
-        }
-      ),
-    })
-
-    // "Polyfilling" text decoder. `bson.deserialize` requires decoding. We are creating a bridge function so we don't need to inject the full library
-    const textDecoderPolyfill = class TextDecoderMock {
-      constructorArgs
-
-      constructor(...constructorArgs: any) {
-        this.constructorArgs = constructorArgs
-      }
-
-      decode(...input: any) {
-        // @ts-expect-error - this is going to run in the isolate, where this function will be available
-        // eslint-disable-next-line no-undef
-        return textDecoderCb({
-          constructorArgs: this.constructorArgs,
-          functionArgs: input,
-        })
-      }
-    }
-      .toString()
-      .replace(/TextDecoderMock/, "TextDecoder")
+    const bsonPolyfills = loadBundle(BundleType.BSON_POLYFILLS)
 
     const script = this.isolate.compileScriptSync(
-      `${textDecoderPolyfill};${bsonSource}`
+      `${bsonPolyfills};${bsonSource}`
     )
     script.runSync(this.vm, { timeout: this.invocationTimeout, release: false })
     new Promise(() => {

packages/string-templates/package.json

@@ -11,8 +11,7 @@
       "require": "./dist/bundle.cjs",
       "import": "./dist/bundle.mjs"
     },
-    "./package.json": "./package.json",
-    "./iife": "./dist/iife.mjs"
+    "./package.json": "./package.json"
   },
   "scripts": {
     "build": "tsc --emitDeclarationOnly && rollup -c",