diff --git a/.prettierignore b/.prettierignore index b1ee287391..b0f9f8cdbf 100644 --- a/.prettierignore +++ b/.prettierignore @@ -9,4 +9,5 @@ packages/backend-core/coverage packages/builder/.routify packages/sdk/sdk packages/pro/coverage -**/*.ivm.bundle.js \ No newline at end of file +**/*.ivm.bundle.js +!**/bson-polyfills.ivm.bundle.js \ No newline at end of file diff --git a/packages/server/src/api/routes/tests/queries/mongodb.spec.ts b/packages/server/src/api/routes/tests/queries/mongodb.spec.ts index a37957fe7e..37af4e74e1 100644 --- a/packages/server/src/api/routes/tests/queries/mongodb.spec.ts +++ b/packages/server/src/api/routes/tests/queries/mongodb.spec.ts @@ -634,6 +634,130 @@ if (descriptions.length) { } }) }) + + it("should be able to select a ObjectId in a transformer", async () => { + const query = await createQuery({ + fields: { + json: {}, + extra: { + actionType: "find", + }, + }, + transformer: "return data.map(x => ({ id: x._id }))", + }) + + const result = await config.api.query.execute(query._id!) + + expect(result.data).toEqual([ + { id: expectValidId }, + { id: expectValidId }, + { id: expectValidId }, + { id: expectValidId }, + { id: expectValidId }, + ]) + }) + + it("can handle all bson field types with transformers", async () => { + collection = generator.guid() + await withCollection(async collection => { + await collection.insertOne({ + _id: new BSON.ObjectId("65b0123456789abcdef01234"), + stringField: "This is a string", + numberField: 42, + doubleField: new BSON.Double(42.42), + integerField: new BSON.Int32(123), + longField: new BSON.Long("9223372036854775807"), + booleanField: true, + nullField: null, + arrayField: [1, 2, 3, "four", { nested: true }], + objectField: { + nestedString: "nested", + nestedNumber: 99, + }, + dateField: new Date(Date.UTC(2025, 0, 30, 12, 30, 20)), + timestampField: new BSON.Timestamp({ t: 1706616000, i: 1 }), + binaryField: new BSON.Binary( + new TextEncoder().encode("bufferValue") + ), + objectIdField: new BSON.ObjectId("65b0123456789abcdef01235"), + regexField: new BSON.BSONRegExp("^Hello.*", "i"), + minKeyField: new BSON.MinKey(), + maxKeyField: new BSON.MaxKey(), + decimalField: new BSON.Decimal128("12345.6789"), + codeField: new BSON.Code( + "function() { return 'Hello, World!'; }" + ), + codeWithScopeField: new BSON.Code( + "function(x) { return x * 2; }", + { x: 10 } + ), + }) + }) + + const query = await createQuery({ + fields: { + json: {}, + extra: { + actionType: "find", + collection, + }, + }, + transformer: `return data.map(x => ({ + ...x, + binaryField: x.binaryField?.toString('utf8'), + decimalField: x.decimalField.toString(), + longField: x.longField.toString(), + regexField: x.regexField.toString(), + // TODO: currenlty not supported, it looks like there is bug in the library. Getting: Timestamp constructed from { t, i } must provide t as a number + timestampField: null + }))`, + }) + + const result = await config.api.query.execute(query._id!) + + expect(result.data).toEqual([ + { + _id: "65b0123456789abcdef01234", + arrayField: [ + 1, + 2, + 3, + "four", + { + nested: true, + }, + ], + binaryField: "bufferValue", + booleanField: true, + codeField: { + code: "function() { return 'Hello, World!'; }", + }, + codeWithScopeField: { + code: "function(x) { return x * 2; }", + scope: { + x: 10, + }, + }, + dateField: "2025-01-30T12:30:20.000Z", + decimalField: "12345.6789", + doubleField: 42.42, + integerField: 123, + longField: "9223372036854775807", + maxKeyField: {}, + minKeyField: {}, + nullField: null, + numberField: 42, + objectField: { + nestedNumber: 99, + nestedString: "nested", + }, + objectIdField: "65b0123456789abcdef01235", + regexField: "/^Hello.*/i", + stringField: "This is a string", + timestampField: null, + }, + ]) + }) }) it("should throw an error if the incorrect actionType is specified", async () => { diff --git a/packages/server/src/jsRunner/bundles/bson-polyfills.ivm.bundle.js b/packages/server/src/jsRunner/bundles/bson-polyfills.ivm.bundle.js new file mode 100644 index 0000000000..94d08b84ed --- /dev/null +++ b/packages/server/src/jsRunner/bundles/bson-polyfills.ivm.bundle.js @@ -0,0 +1,122 @@ +if (typeof btoa !== "function") { + var chars = { + ascii: function () { + return "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" + }, + indices: function () { + if (!this.cache) { + this.cache = {} + var ascii = chars.ascii() + + for (var c = 0; c < ascii.length; c++) { + var chr = ascii[c] + this.cache[chr] = c + } + } + return this.cache + }, + } + + function atob(b64) { + var indices = chars.indices(), + pos = b64.indexOf("="), + padded = pos > -1, + len = padded ? pos : b64.length, + i = -1, + data = "" + + while (i < len) { + var code = + (indices[b64[++i]] << 18) | + (indices[b64[++i]] << 12) | + (indices[b64[++i]] << 6) | + indices[b64[++i]] + if (code !== 0) { + data += String.fromCharCode( + (code >>> 16) & 255, + (code >>> 8) & 255, + code & 255 + ) + } + } + + if (padded) { + data = data.slice(0, pos - b64.length) + } + + return data + } + + function btoa(data) { + var ascii = chars.ascii(), + len = data.length - 1, + i = -1, + b64 = "" + + while (i < len) { + var code = + (data.charCodeAt(++i) << 16) | + (data.charCodeAt(++i) << 8) | + data.charCodeAt(++i) + b64 += + ascii[(code >>> 18) & 63] + + ascii[(code >>> 12) & 63] + + ascii[(code >>> 6) & 63] + + ascii[code & 63] + } + + var pads = data.length % 3 + if (pads > 0) { + b64 = b64.slice(0, pads - 3) + + while (b64.length % 4 !== 0) { + b64 += "=" + } + } + + return b64 + } +} + +if (typeof TextDecoder === "undefined") { + globalThis.TextDecoder = class { + constructor(encoding = "utf8") { + if (encoding !== "utf8") { + throw new Error( + `Only UTF-8 is supported in this polyfill. Recieved: ${encoding}` + ) + } + } + decode(buffer) { + return String.fromCharCode(...buffer) + } + } +} + +if (typeof TextEncoder === "undefined") { + globalThis.TextEncoder = class { + encode(str) { + const utf8 = [] + for (const i = 0; i < str.length; i++) { + const codePoint = str.charCodeAt(i) + + if (codePoint < 0x80) { + utf8.push(codePoint) + } else if (codePoint < 0x800) { + utf8.push(0xc0 | (codePoint >> 6)) + utf8.push(0x80 | (codePoint & 0x3f)) + } else if (codePoint < 0x10000) { + utf8.push(0xe0 | (codePoint >> 12)) + utf8.push(0x80 | ((codePoint >> 6) & 0x3f)) + utf8.push(0x80 | (codePoint & 0x3f)) + } else { + utf8.push(0xf0 | (codePoint >> 18)) + utf8.push(0x80 | ((codePoint >> 12) & 0x3f)) + utf8.push(0x80 | ((codePoint >> 6) & 0x3f)) + utf8.push(0x80 | (codePoint & 0x3f)) + } + } + return new Uint8Array(utf8) + } + } +} diff --git a/packages/server/src/jsRunner/bundles/index.ts b/packages/server/src/jsRunner/bundles/index.ts index b62adac1cc..3a00ee96cc 100644 --- a/packages/server/src/jsRunner/bundles/index.ts +++ b/packages/server/src/jsRunner/bundles/index.ts @@ -5,6 +5,7 @@ export const enum BundleType { BSON = "bson", SNIPPETS = "snippets", BUFFER = "buffer", + BSON_POLYFILLS = "bson_polyfills", } const bundleSourceFile: Record = { @@ -12,6 +13,7 @@ const bundleSourceFile: Record = { [BundleType.BSON]: "./bson.ivm.bundle.js", [BundleType.SNIPPETS]: "./snippets.ivm.bundle.js", [BundleType.BUFFER]: "./buffer.ivm.bundle.js", + [BundleType.BSON_POLYFILLS]: "./bson-polyfills.ivm.bundle.js", } const bundleSourceCode: Partial> = {} diff --git a/packages/server/src/jsRunner/bundles/snippets.ts b/packages/server/src/jsRunner/bundles/snippets.ts index 8244b2eef8..343cccd36f 100644 --- a/packages/server/src/jsRunner/bundles/snippets.ts +++ b/packages/server/src/jsRunner/bundles/snippets.ts @@ -1,6 +1,5 @@ // @ts-ignore -// eslint-disable-next-line local-rules/no-budibase-imports -import { iifeWrapper } from "@budibase/string-templates/iife" +import { iifeWrapper } from "@budibase/string-templates" export default new Proxy( {}, diff --git a/packages/server/src/jsRunner/vm/isolated-vm.ts b/packages/server/src/jsRunner/vm/isolated-vm.ts index 3863be742d..37ee048dc2 100644 --- a/packages/server/src/jsRunner/vm/isolated-vm.ts +++ b/packages/server/src/jsRunner/vm/isolated-vm.ts @@ -161,42 +161,10 @@ export class IsolatedVM implements VM { const bsonSource = loadBundle(BundleType.BSON) - this.addToContext({ - textDecoderCb: new ivm.Callback( - (args: { - constructorArgs: any - functionArgs: Parameters["decode"]> - }) => { - const result = new TextDecoder(...args.constructorArgs).decode( - ...args.functionArgs - ) - return result - } - ), - }) - - // "Polyfilling" text decoder. `bson.deserialize` requires decoding. We are creating a bridge function so we don't need to inject the full library - const textDecoderPolyfill = class TextDecoderMock { - constructorArgs - - constructor(...constructorArgs: any) { - this.constructorArgs = constructorArgs - } - - decode(...input: any) { - // @ts-expect-error - this is going to run in the isolate, where this function will be available - // eslint-disable-next-line no-undef - return textDecoderCb({ - constructorArgs: this.constructorArgs, - functionArgs: input, - }) - } - } - .toString() - .replace(/TextDecoderMock/, "TextDecoder") + const bsonPolyfills = loadBundle(BundleType.BSON_POLYFILLS) const script = this.isolate.compileScriptSync( - `${textDecoderPolyfill};${bsonSource}` + `${bsonPolyfills};${bsonSource}` ) script.runSync(this.vm, { timeout: this.invocationTimeout, release: false }) new Promise(() => { diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index b1b4b9ef55..acd0ea4fa8 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -11,8 +11,7 @@ "require": "./dist/bundle.cjs", "import": "./dist/bundle.mjs" }, - "./package.json": "./package.json", - "./iife": "./dist/iife.mjs" + "./package.json": "./package.json" }, "scripts": { "build": "tsc --emitDeclarationOnly && rollup -c",