From 51b19633f4daf3742ad4c76e43162dd2f7bbdfd5 Mon Sep 17 00:00:00 2001 From: Dean Date: Wed, 7 Feb 2024 10:03:37 +0000 Subject: [PATCH 01/15] Ensure insert config items are defaulted to avoid destructring issue --- .../builder/src/components/common/CodeEditor/index.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/common/CodeEditor/index.js b/packages/builder/src/components/common/CodeEditor/index.js index 38d377b47a..0d71a475f0 100644 --- a/packages/builder/src/components/common/CodeEditor/index.js +++ b/packages/builder/src/components/common/CodeEditor/index.js @@ -286,7 +286,13 @@ export const hbInsert = (value, from, to, text) => { return parsedInsert } -export function jsInsert(value, from, to, text, { helper, disableWrapping }) { +export function jsInsert( + value, + from, + to, + text, + { helper, disableWrapping } = {} +) { let parsedInsert = "" const left = from ? value.substring(0, from) : "" From 7dc2c3551f5eb6246a61abe9257f562b4588ba8b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 7 Feb 2024 17:03:44 +0000 Subject: [PATCH 02/15] Updating the listObjects functionality to correctly handle truncated responses, when not all objects can be returned at once we need to loop, but we weren't correctly picking up the token that should be passed. --- .../backend-core/src/objectStore/objectStore.ts | 3 ++- packages/backend-core/src/queue/queue.ts | 13 ++++++++++--- packages/pro | 2 +- yarn.lock | 5 +++++ 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 3a3b9cdaab..8d18fb97fd 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -255,7 +255,8 @@ export async function listAllObjects(bucketName: string, path: string) { objects = objects.concat(response.Contents) } isTruncated = !!response.IsTruncated - } while (isTruncated) + token = response.NextContinuationToken + } while (isTruncated && token) return objects } diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index b95dace5b2..0bcb25a35f 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -2,7 +2,7 @@ import env from "../environment" import { getRedisOptions } from "../redis/utils" import { JobQueue } from "./constants" import InMemoryQueue from "./inMemoryQueue" -import BullQueue, { QueueOptions } from "bull" +import BullQueue, { QueueOptions, JobOptions } from "bull" import { addListeners, StalledFn } from "./listeners" import { Duration } from "../utils" import * as timers from "../timers" @@ -24,17 +24,24 @@ async function cleanup() { export function createQueue( jobQueue: JobQueue, - opts: { removeStalledCb?: StalledFn } = {} + opts: { + removeStalledCb?: StalledFn + maxStalledCount?: number + jobOptions?: JobOptions + } = {} ): BullQueue.Queue { const redisOpts = getRedisOptions() const queueConfig: QueueOptions = { redis: redisOpts, settings: { - maxStalledCount: 0, + maxStalledCount: opts.maxStalledCount ? opts.maxStalledCount : 0, lockDuration: QUEUE_LOCK_MS, lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS, }, } + if (opts.jobOptions) { + queueConfig.defaultJobOptions = opts.jobOptions + } let queue: any if (!env.isTest()) { queue = new BullQueue(jobQueue, queueConfig) diff --git a/packages/pro b/packages/pro index aaf7101cd1..93cd6cc2bd 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit aaf7101cd1493215155cc8f83124c70d53eb1be4 +Subproject commit 93cd6cc2bd16afff16a9d7cf7c89d614b1c0b5ae diff --git a/yarn.lock b/yarn.lock index 2c12097fb5..fb231e2824 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15867,6 +15867,11 @@ nodemailer@6.7.2: resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.7.2.tgz#44b2ad5f7ed71b7067f7a21c4fedabaec62b85e0" integrity sha512-Dz7zVwlef4k5R71fdmxwR8Q39fiboGbu3xgswkzGwczUfjp873rVxt1O46+Fh0j1ORnAC6L9+heI8uUpO6DT7Q== +nodemailer@6.9.9: + version "6.9.9" + resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.9.9.tgz#4549bfbf710cc6addec5064dd0f19874d24248d9" + integrity sha512-dexTll8zqQoVJEZPwQAKzxxtFn0qTnjdQTchoU6Re9BUUGBJiOy3YMn/0ShTW6J5M0dfQ1NeDeRTTl4oIWgQMA== + nodemon@2.0.15: version "2.0.15" resolved "https://registry.yarnpkg.com/nodemon/-/nodemon-2.0.15.tgz#504516ce3b43d9dc9a955ccd9ec57550a31a8d4e" From 943ad85d5b20db1631056ae469336509c3b7d1f6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 7 Feb 2024 17:29:35 +0000 Subject: [PATCH 03/15] Update pro. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 93cd6cc2bd..9dc28c2efd 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 93cd6cc2bd16afff16a9d7cf7c89d614b1c0b5ae +Subproject commit 9dc28c2efd2c8e6b044731f3f5dc857e44836e3b From 7f13457ae0f420919ac9ac99b9162746e969538f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 7 Feb 2024 17:55:03 +0000 Subject: [PATCH 04/15] Updating pro. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 9dc28c2efd..aaf7101cd1 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 9dc28c2efd2c8e6b044731f3f5dc857e44836e3b +Subproject commit aaf7101cd1493215155cc8f83124c70d53eb1be4 From 2616dbf46afacf6c0b9b3770af8a8e787639c07c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 7 Feb 2024 18:24:56 +0000 Subject: [PATCH 05/15] Updating pro sub-reference after merging recent pro PR for updating queue params. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index aaf7101cd1..9b14e5d518 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit aaf7101cd1493215155cc8f83124c70d53eb1be4 +Subproject commit 9b14e5d5182bf5e5ee98f717997e7352e5904799 From 218ba1d2831f041f6c958f2d7e37969e135c6bd6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 7 Feb 2024 18:07:05 +0100 Subject: [PATCH 06/15] VM types --- packages/backend-core/src/context/types.ts | 3 ++- packages/types/src/sdk/index.ts | 1 + packages/types/src/sdk/vm.ts | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 packages/types/src/sdk/vm.ts diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts index cc052ca505..75f5234651 100644 --- a/packages/backend-core/src/context/types.ts +++ b/packages/backend-core/src/context/types.ts @@ -1,4 +1,4 @@ -import { IdentityContext } from "@budibase/types" +import { IdentityContext, VM } from "@budibase/types" import { Isolate, Context, Module } from "isolated-vm" // keep this out of Budibase types, don't want to expose context info @@ -15,4 +15,5 @@ export type ContextMap = { jsContext: Context helpersModule: Module } + vm?: VM } diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts index 0eab2ba556..36faaae9c3 100644 --- a/packages/types/src/sdk/index.ts +++ b/packages/types/src/sdk/index.ts @@ -20,3 +20,4 @@ export * from "./cli" export * from "./websocket" export * from "./permissions" export * from "./row" +export * from "./vm" diff --git a/packages/types/src/sdk/vm.ts b/packages/types/src/sdk/vm.ts new file mode 100644 index 0000000000..0c85cac925 --- /dev/null +++ b/packages/types/src/sdk/vm.ts @@ -0,0 +1,4 @@ +export interface VM { + cpuTime: bigint + execute(code: string): string +} From c5abb4f8462812b086d680fbaf9bbf4a09ca3c05 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 7 Feb 2024 18:07:23 +0100 Subject: [PATCH 07/15] Create wrapper --- packages/server/src/jsRunner/vm/index.ts | 154 +++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 packages/server/src/jsRunner/vm/index.ts diff --git a/packages/server/src/jsRunner/vm/index.ts b/packages/server/src/jsRunner/vm/index.ts new file mode 100644 index 0000000000..0f9b11e531 --- /dev/null +++ b/packages/server/src/jsRunner/vm/index.ts @@ -0,0 +1,154 @@ +import ivm from "isolated-vm" + +import url from "url" +import crypto from "crypto" +import querystring from "querystring" + +import { BundleType, loadBundle } from "../bundles" +import { VM } from "@budibase/types" + +export class IsolatedVM implements VM { + #isolate: ivm.Isolate + #vm: ivm.Context + #jail: ivm.Reference + #timeout: number + + #modules: Record< + string, + { + headCode: string + module: ivm.Module + } + > = {} + + readonly #resultKey = "results" + + constructor({ + memoryLimit, + timeout, + }: { + memoryLimit: number + timeout: number + }) { + this.#isolate = new ivm.Isolate({ memoryLimit }) + this.#vm = this.#isolate.createContextSync() + this.#jail = this.#vm.global + this.#jail.setSync("global", this.#jail.derefInto()) + + this.#addToContext({ + [this.#resultKey]: { out: "" }, + }) + + this.#timeout = timeout + } + + get cpuTime() { + return this.#isolate.cpuTime + } + + withHelpers() { + const injectedRequire = ` + const require = function(val){ + switch (val) { + case "url": + return { + resolve: (...params) => urlResolveCb(...params), + parse: (...params) => urlParseCb(...params), + } + case "querystring": + return { + escape: (...params) => querystringEscapeCb(...params), + } + } + };` + + const helpersSource = loadBundle(BundleType.HELPERS) + const helpersModule = this.#isolate.compileModuleSync( + `${injectedRequire};${helpersSource}` + ) + + this.#addToContext({ + urlResolveCb: new ivm.Callback( + (...params: Parameters) => url.resolve(...params) + ), + urlParseCb: new ivm.Callback((...params: Parameters) => + url.parse(...params) + ), + querystringEscapeCb: new ivm.Callback( + (...params: Parameters) => + querystring.escape(...params) + ), + helpersStripProtocol: new ivm.Callback((str: string) => { + var parsed = url.parse(str) as any + parsed.protocol = "" + return parsed.format() + }), + }) + + const cryptoModule = this.#isolate.compileModuleSync( + `export default { randomUUID: cryptoRandomUUIDCb }` + ) + cryptoModule.instantiateSync(this.#vm, specifier => { + throw new Error(`No imports allowed. Required: ${specifier}`) + }) + + this.#addToContext({ + cryptoRandomUUIDCb: new ivm.Callback( + (...params: Parameters) => { + return crypto.randomUUID(...params) + } + ), + }) + + helpersModule.instantiateSync(this.#vm, specifier => { + if (specifier === "crypto") { + return cryptoModule + } + throw new Error(`No imports allowed. Required: ${specifier}`) + }) + + this.#modules["compiled_module"] = { + headCode: 'import helpers from "compiled_module"', + module: helpersModule, + } + return this + } + + execute(code: string): string { + code = [ + ...Object.values(this.#modules).map(m => m.headCode), + `results.out=${code};`, + ].join(";") + + const script = this.#isolate.compileModuleSync(code) + + script.instantiateSync(this.#vm, specifier => { + if (specifier === "compiled_module") { + return this.#modules[specifier].module + } + + throw new Error(`"${specifier}" import not allowed`) + }) + + script.evaluateSync({ timeout: this.#timeout }) + + const result = this.#getResult() + return result + } + + #addToContext(context: Record) { + for (let key in context) { + this.#jail.setSync( + key, + new ivm.ExternalCopy(context[key]).copyInto({ release: true }) + ) + } + } + + #getResult() { + const ref = this.#vm.global.getSync(this.#resultKey, { reference: true }) + const result = ref.copySync() + ref.release() + return result.out + } +} From 3b8b60aa037be0af9d3ca670ae0fb5a8b631f03d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 7 Feb 2024 18:08:15 +0100 Subject: [PATCH 08/15] Use wrapper --- packages/backend-core/src/context/types.ts | 6 - packages/server/src/jsRunner/index.ts | 133 ++------------------- 2 files changed, 11 insertions(+), 128 deletions(-) diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts index 75f5234651..0f4c2106d0 100644 --- a/packages/backend-core/src/context/types.ts +++ b/packages/backend-core/src/context/types.ts @@ -1,5 +1,4 @@ import { IdentityContext, VM } from "@budibase/types" -import { Isolate, Context, Module } from "isolated-vm" // keep this out of Budibase types, don't want to expose context info export type ContextMap = { @@ -10,10 +9,5 @@ export type ContextMap = { isScim?: boolean automationId?: string isMigrating?: boolean - isolateRefs?: { - jsIsolate: Isolate - jsContext: Context - helpersModule: Module - } vm?: VM } diff --git a/packages/server/src/jsRunner/index.ts b/packages/server/src/jsRunner/index.ts index 9c54779567..140881aa21 100644 --- a/packages/server/src/jsRunner/index.ts +++ b/packages/server/src/jsRunner/index.ts @@ -1,12 +1,9 @@ -import ivm from "isolated-vm" import env from "../environment" import { setJSRunner, JsErrorTimeout } from "@budibase/string-templates" -import { context } from "@budibase/backend-core" import tracer from "dd-trace" -import url from "url" -import crypto from "crypto" -import querystring from "querystring" -import { BundleType, loadBundle } from "./bundles" + +import { IsolatedVM } from "./vm" +import { context } from "@budibase/backend-core" class ExecutionTimeoutError extends Error { constructor(message: string) { @@ -16,109 +13,24 @@ class ExecutionTimeoutError extends Error { } export function init() { - const helpersSource = loadBundle(BundleType.HELPERS) setJSRunner((js: string, ctx: Record) => { return tracer.trace("runJS", {}, span => { try { const bbCtx = context.getCurrentContext()! - const isolateRefs = bbCtx.isolateRefs - if (!isolateRefs) { - const jsIsolate = new ivm.Isolate({ + let { vm } = bbCtx + if (!vm) { + vm = new IsolatedVM({ memoryLimit: env.JS_RUNNER_MEMORY_LIMIT, - }) - const jsContext = jsIsolate.createContextSync() + timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS, + }).withHelpers() - const injectedRequire = ` - const require = function(val){ - switch (val) { - case "url": - return { - resolve: (...params) => urlResolveCb(...params), - parse: (...params) => urlParseCb(...params), - } - case "querystring": - return { - escape: (...params) => querystringEscapeCb(...params), - } - } - };` - - const global = jsContext.global - global.setSync( - "urlResolveCb", - new ivm.Callback((...params: Parameters) => - url.resolve(...params) - ) - ) - - global.setSync( - "urlParseCb", - new ivm.Callback((...params: Parameters) => - url.parse(...params) - ) - ) - - global.setSync( - "querystringEscapeCb", - new ivm.Callback( - (...params: Parameters) => - querystring.escape(...params) - ) - ) - - global.setSync( - "helpersStripProtocol", - new ivm.Callback((str: string) => { - var parsed = url.parse(str) as any - parsed.protocol = "" - return parsed.format() - }) - ) - - const helpersModule = jsIsolate.compileModuleSync( - `${injectedRequire};${helpersSource}` - ) - - const cryptoModule = jsIsolate.compileModuleSync( - `export default { randomUUID: cryptoRandomUUIDCb }` - ) - cryptoModule.instantiateSync(jsContext, specifier => { - throw new Error(`No imports allowed. Required: ${specifier}`) - }) - - global.setSync( - "cryptoRandomUUIDCb", - new ivm.Callback( - (...params: Parameters) => { - return crypto.randomUUID(...params) - } - ) - ) - - helpersModule.instantiateSync(jsContext, specifier => { - if (specifier === "crypto") { - return cryptoModule - } - throw new Error(`No imports allowed. Required: ${specifier}`) - }) - - for (const [key, value] of Object.entries(ctx)) { - if (key === "helpers") { - // Can't copy the native helpers into the isolate. We just ignore them as they are handled properly from the helpersSource - continue - } - global.setSync(key, value) - } - - bbCtx.isolateRefs = { jsContext, jsIsolate, helpersModule } + bbCtx.vm = vm } - let { jsIsolate, jsContext, helpersModule } = bbCtx.isolateRefs! - const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS if (perRequestLimit) { - const cpuMs = Number(jsIsolate.cpuTime) / 1e6 + const cpuMs = Number(vm.cpuTime) / 1e6 if (cpuMs > perRequestLimit) { throw new ExecutionTimeoutError( `CPU time limit exceeded (${cpuMs}ms > ${perRequestLimit}ms)` @@ -126,30 +38,7 @@ export function init() { } } - const script = jsIsolate.compileModuleSync( - `import helpers from "compiled_module";const result=${js};cb(result)`, - {} - ) - - script.instantiateSync(jsContext, specifier => { - if (specifier === "compiled_module") { - return helpersModule - } - - throw new Error(`"${specifier}" import not allowed`) - }) - - let result - jsContext.global.setSync( - "cb", - new ivm.Callback((value: any) => { - result = value - }) - ) - - script.evaluateSync({ - timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS, - }) + const result = vm.execute(js) return result } catch (error: any) { From 0d0171fa086e3a5ce875db0ea18e99c9785ff78b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 7 Feb 2024 18:12:14 +0100 Subject: [PATCH 09/15] Move cpulimits responsability --- packages/server/src/jsRunner/index.ts | 17 ---------------- packages/server/src/jsRunner/vm/index.ts | 26 ++++++++++++++++++++---- packages/types/src/sdk/vm.ts | 1 - 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/packages/server/src/jsRunner/index.ts b/packages/server/src/jsRunner/index.ts index 140881aa21..aab5a64098 100644 --- a/packages/server/src/jsRunner/index.ts +++ b/packages/server/src/jsRunner/index.ts @@ -5,13 +5,6 @@ import tracer from "dd-trace" import { IsolatedVM } from "./vm" import { context } from "@budibase/backend-core" -class ExecutionTimeoutError extends Error { - constructor(message: string) { - super(message) - this.name = "ExecutionTimeoutError" - } -} - export function init() { setJSRunner((js: string, ctx: Record) => { return tracer.trace("runJS", {}, span => { @@ -28,16 +21,6 @@ export function init() { bbCtx.vm = vm } - const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS - if (perRequestLimit) { - const cpuMs = Number(vm.cpuTime) / 1e6 - if (cpuMs > perRequestLimit) { - throw new ExecutionTimeoutError( - `CPU time limit exceeded (${cpuMs}ms > ${perRequestLimit}ms)` - ) - } - } - const result = vm.execute(js) return result diff --git a/packages/server/src/jsRunner/vm/index.ts b/packages/server/src/jsRunner/vm/index.ts index 0f9b11e531..150456ad03 100644 --- a/packages/server/src/jsRunner/vm/index.ts +++ b/packages/server/src/jsRunner/vm/index.ts @@ -7,11 +7,19 @@ import querystring from "querystring" import { BundleType, loadBundle } from "../bundles" import { VM } from "@budibase/types" +class ExecutionTimeoutError extends Error { + constructor(message: string) { + super(message) + this.name = "ExecutionTimeoutError" + } +} + export class IsolatedVM implements VM { #isolate: ivm.Isolate #vm: ivm.Context #jail: ivm.Reference #timeout: number + #perRequestLimit?: number #modules: Record< string, @@ -26,9 +34,11 @@ export class IsolatedVM implements VM { constructor({ memoryLimit, timeout, + perRequestLimit, }: { memoryLimit: number timeout: number + perRequestLimit?: number }) { this.#isolate = new ivm.Isolate({ memoryLimit }) this.#vm = this.#isolate.createContextSync() @@ -40,10 +50,7 @@ export class IsolatedVM implements VM { }) this.#timeout = timeout - } - - get cpuTime() { - return this.#isolate.cpuTime + this.#perRequestLimit = perRequestLimit } withHelpers() { @@ -115,6 +122,17 @@ export class IsolatedVM implements VM { } execute(code: string): string { + const perRequestLimit = this.#perRequestLimit + + if (perRequestLimit) { + const cpuMs = Number(this.#isolate.cpuTime) / 1e6 + if (cpuMs > perRequestLimit) { + throw new ExecutionTimeoutError( + `CPU time limit exceeded (${cpuMs}ms > ${perRequestLimit}ms)` + ) + } + } + code = [ ...Object.values(this.#modules).map(m => m.headCode), `results.out=${code};`, diff --git a/packages/types/src/sdk/vm.ts b/packages/types/src/sdk/vm.ts index 0c85cac925..3abec4d39d 100644 --- a/packages/types/src/sdk/vm.ts +++ b/packages/types/src/sdk/vm.ts @@ -1,4 +1,3 @@ export interface VM { - cpuTime: bigint execute(code: string): string } From c44119b3f9a5a7442cf4819e2f55dd9c341748d2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 8 Feb 2024 00:34:23 +0100 Subject: [PATCH 10/15] Callbacks --- packages/server/src/jsRunner/vm/index.ts | 88 +++++++++++++----------- 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/packages/server/src/jsRunner/vm/index.ts b/packages/server/src/jsRunner/vm/index.ts index 150456ad03..6723673b07 100644 --- a/packages/server/src/jsRunner/vm/index.ts +++ b/packages/server/src/jsRunner/vm/index.ts @@ -54,37 +54,23 @@ export class IsolatedVM implements VM { } withHelpers() { - const injectedRequire = ` - const require = function(val){ - switch (val) { - case "url": - return { - resolve: (...params) => urlResolveCb(...params), - parse: (...params) => urlParseCb(...params), - } - case "querystring": - return { - escape: (...params) => querystringEscapeCb(...params), - } - } - };` + const urlModule = this.#registerCallbacks({ + resolve: url.resolve, + parse: url.parse, + }) - const helpersSource = loadBundle(BundleType.HELPERS) - const helpersModule = this.#isolate.compileModuleSync( - `${injectedRequire};${helpersSource}` - ) + const querystringModule = this.#registerCallbacks({ + escape: querystring.escape, + }) + + const injectedRequire = `const require=function req(val) { + switch (val) { + case "url": return ${urlModule}; + case "querystring": return ${querystringModule}; + } + }` this.#addToContext({ - urlResolveCb: new ivm.Callback( - (...params: Parameters) => url.resolve(...params) - ), - urlParseCb: new ivm.Callback((...params: Parameters) => - url.parse(...params) - ), - querystringEscapeCb: new ivm.Callback( - (...params: Parameters) => - querystring.escape(...params) - ), helpersStripProtocol: new ivm.Callback((str: string) => { var parsed = url.parse(str) as any parsed.protocol = "" @@ -92,24 +78,24 @@ export class IsolatedVM implements VM { }), }) - const cryptoModule = this.#isolate.compileModuleSync( - `export default { randomUUID: cryptoRandomUUIDCb }` + const helpersSource = loadBundle(BundleType.HELPERS) + const helpersModule = this.#isolate.compileModuleSync( + `${injectedRequire};${helpersSource}` ) - cryptoModule.instantiateSync(this.#vm, specifier => { - throw new Error(`No imports allowed. Required: ${specifier}`) - }) - this.#addToContext({ - cryptoRandomUUIDCb: new ivm.Callback( - (...params: Parameters) => { - return crypto.randomUUID(...params) - } - ), + const cryptoModule = this.#registerCallbacks({ + randomUUID: crypto.randomUUID, }) helpersModule.instantiateSync(this.#vm, specifier => { if (specifier === "crypto") { - return cryptoModule + const module = this.#isolate.compileModuleSync( + `export default ${cryptoModule}` + ) + module.instantiateSync(this.#vm, specifier => { + throw new Error(`No imports allowed. Required: ${specifier}`) + }) + return module } throw new Error(`No imports allowed. Required: ${specifier}`) }) @@ -154,6 +140,28 @@ export class IsolatedVM implements VM { return result } + #registerCallbacks(functions: Record) { + const libId = crypto.randomUUID().replace(/-/g, "") + + const x: Record = {} + for (const [funcName, func] of Object.entries(functions)) { + const key = `f${libId}${funcName}cb` + x[funcName] = key + + this.#addToContext({ + [key]: new ivm.Callback((...params: any[]) => (func as any)(...params)), + }) + } + + const mod = + `{` + + Object.entries(x) + .map(([key, func]) => `${key}: ${func}`) + .join() + + "}" + return mod + } + #addToContext(context: Record) { for (let key in context) { this.#jail.setSync( From 7693a1fc6945944ef1185ce50da1511095318850 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 8 Feb 2024 00:43:13 +0100 Subject: [PATCH 11/15] Fix imports --- packages/server/src/jsRunner/vm/index.ts | 46 ++++++++++++------------ 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/packages/server/src/jsRunner/vm/index.ts b/packages/server/src/jsRunner/vm/index.ts index 6723673b07..c05d9c03f2 100644 --- a/packages/server/src/jsRunner/vm/index.ts +++ b/packages/server/src/jsRunner/vm/index.ts @@ -21,13 +21,11 @@ export class IsolatedVM implements VM { #timeout: number #perRequestLimit?: number - #modules: Record< - string, - { - headCode: string - module: ivm.Module - } - > = {} + #modules: { + import: string + moduleKey: string + module: ivm.Module + }[] = [] readonly #resultKey = "results" @@ -63,13 +61,6 @@ export class IsolatedVM implements VM { escape: querystring.escape, }) - const injectedRequire = `const require=function req(val) { - switch (val) { - case "url": return ${urlModule}; - case "querystring": return ${querystringModule}; - } - }` - this.#addToContext({ helpersStripProtocol: new ivm.Callback((str: string) => { var parsed = url.parse(str) as any @@ -78,17 +69,22 @@ export class IsolatedVM implements VM { }), }) + const injectedRequire = `const require=function req(val) { + switch (val) { + case "url": return ${urlModule}; + case "querystring": return ${querystringModule}; + } + }` const helpersSource = loadBundle(BundleType.HELPERS) const helpersModule = this.#isolate.compileModuleSync( `${injectedRequire};${helpersSource}` ) - const cryptoModule = this.#registerCallbacks({ - randomUUID: crypto.randomUUID, - }) - helpersModule.instantiateSync(this.#vm, specifier => { if (specifier === "crypto") { + const cryptoModule = this.#registerCallbacks({ + randomUUID: crypto.randomUUID, + }) const module = this.#isolate.compileModuleSync( `export default ${cryptoModule}` ) @@ -100,10 +96,11 @@ export class IsolatedVM implements VM { throw new Error(`No imports allowed. Required: ${specifier}`) }) - this.#modules["compiled_module"] = { - headCode: 'import helpers from "compiled_module"', + this.#modules.push({ + import: "helpers", + moduleKey: `i${crypto.randomUUID().replace(/-/g, "")}`, module: helpersModule, - } + }) return this } @@ -120,15 +117,16 @@ export class IsolatedVM implements VM { } code = [ - ...Object.values(this.#modules).map(m => m.headCode), + ...this.#modules.map(m => `import ${m.import} from "${m.moduleKey}"`), `results.out=${code};`, ].join(";") const script = this.#isolate.compileModuleSync(code) script.instantiateSync(this.#vm, specifier => { - if (specifier === "compiled_module") { - return this.#modules[specifier].module + const module = this.#modules.find(m => m.moduleKey === specifier) + if (module) { + return module.module } throw new Error(`"${specifier}" import not allowed`) From e4285e30f1cbb00dab89debfc3ee4f455b4db815 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 8 Feb 2024 00:46:50 +0100 Subject: [PATCH 12/15] Use wrapper for queries --- packages/server/src/utilities/scriptRunner.ts | 61 +++---------------- 1 file changed, 10 insertions(+), 51 deletions(-) diff --git a/packages/server/src/utilities/scriptRunner.ts b/packages/server/src/utilities/scriptRunner.ts index b6e597cc55..c89d266b7e 100644 --- a/packages/server/src/utilities/scriptRunner.ts +++ b/packages/server/src/utilities/scriptRunner.ts @@ -1,63 +1,22 @@ -import ivm from "isolated-vm" +import env from "../environment" +import { IsolatedVM } from "../jsRunner/vm" const JS_TIMEOUT_MS = 1000 - class ScriptRunner { - vm: IsolatedVM + #code constructor(script: string, context: any) { - const code = `let fn = () => {\n${script}\n}; results.out = fn();` - this.vm = new IsolatedVM({ memoryLimit: 8 }) - this.vm.context = { - ...context, - results: { out: "" }, - } - this.vm.code = code + this.#code = `let fn = () => {\n${script}\n}; results.out = fn();` } execute() { - this.vm.runScript() - const results = this.vm.getValue("results") - return results.out - } -} - -class IsolatedVM { - isolate: ivm.Isolate - vm: ivm.Context - #jail: ivm.Reference - script: any - - constructor({ memoryLimit }: { memoryLimit: number }) { - this.isolate = new ivm.Isolate({ memoryLimit }) - this.vm = this.isolate.createContextSync() - this.#jail = this.vm.global - this.#jail.setSync("global", this.#jail.derefInto()) - } - - getValue(key: string) { - const ref = this.vm.global.getSync(key, { reference: true }) - const result = ref.copySync() - ref.release() + const vm = new IsolatedVM({ + memoryLimit: env.JS_RUNNER_MEMORY_LIMIT, + timeout: JS_TIMEOUT_MS, + }).withHelpers() + const result = vm.execute(this.#code) return result } - - set context(context: Record) { - for (let key in context) { - this.#jail.setSync(key, this.copyRefToVm(context[key])) - } - } - - set code(code: string) { - this.script = this.isolate.compileScriptSync(code) - } - - runScript() { - this.script.runSync(this.vm, { timeout: JS_TIMEOUT_MS }) - } - - copyRefToVm(value: Object): ivm.Copy { - return new ivm.ExternalCopy(value).copyInto({ release: true }) - } } + export default ScriptRunner From 008b39abf44ce218f897a258ee5e9fc97ac758ba Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 8 Feb 2024 00:54:03 +0100 Subject: [PATCH 13/15] Use wrapper for scripts --- packages/server/src/jsRunner/vm/index.ts | 5 +++++ packages/server/src/utilities/scriptRunner.ts | 13 +++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/server/src/jsRunner/vm/index.ts b/packages/server/src/jsRunner/vm/index.ts index c05d9c03f2..3cdf05b873 100644 --- a/packages/server/src/jsRunner/vm/index.ts +++ b/packages/server/src/jsRunner/vm/index.ts @@ -104,6 +104,11 @@ export class IsolatedVM implements VM { return this } + withContext(context: Record) { + this.#addToContext(context) + return this + } + execute(code: string): string { const perRequestLimit = this.#perRequestLimit diff --git a/packages/server/src/utilities/scriptRunner.ts b/packages/server/src/utilities/scriptRunner.ts index c89d266b7e..597c4269ac 100644 --- a/packages/server/src/utilities/scriptRunner.ts +++ b/packages/server/src/utilities/scriptRunner.ts @@ -4,17 +4,18 @@ import { IsolatedVM } from "../jsRunner/vm" const JS_TIMEOUT_MS = 1000 class ScriptRunner { #code + #vm constructor(script: string, context: any) { - this.#code = `let fn = () => {\n${script}\n}; results.out = fn();` + this.#code = `(() => {${script}})();` + this.#vm = new IsolatedVM({ + memoryLimit: env.JS_RUNNER_MEMORY_LIMIT, + timeout: JS_TIMEOUT_MS, + }).withContext(context) } execute() { - const vm = new IsolatedVM({ - memoryLimit: env.JS_RUNNER_MEMORY_LIMIT, - timeout: JS_TIMEOUT_MS, - }).withHelpers() - const result = vm.execute(this.#code) + const result = this.#vm.execute(this.#code) return result } } From 9d335b7fb1c692275494d8836d8839f5dbaec18d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 8 Feb 2024 00:56:45 +0100 Subject: [PATCH 14/15] Fix perRequestLimit --- packages/server/src/jsRunner/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/jsRunner/index.ts b/packages/server/src/jsRunner/index.ts index aab5a64098..20a48916c8 100644 --- a/packages/server/src/jsRunner/index.ts +++ b/packages/server/src/jsRunner/index.ts @@ -16,6 +16,7 @@ export function init() { vm = new IsolatedVM({ memoryLimit: env.JS_RUNNER_MEMORY_LIMIT, timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS, + perRequestLimit: env.JS_PER_REQUEST_TIME_LIMIT_MS, }).withHelpers() bbCtx.vm = vm From 72c122105f6d0308cd1bfad9491ca850504442d3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 8 Feb 2024 10:51:42 +0100 Subject: [PATCH 15/15] Clean code --- packages/server/src/jsRunner/vm/index.ts | 50 ++++++++++++++++-------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/packages/server/src/jsRunner/vm/index.ts b/packages/server/src/jsRunner/vm/index.ts index 3cdf05b873..d0d5793dee 100644 --- a/packages/server/src/jsRunner/vm/index.ts +++ b/packages/server/src/jsRunner/vm/index.ts @@ -14,6 +14,35 @@ class ExecutionTimeoutError extends Error { } } +class ModuleHandler { + #modules: { + import: string + moduleKey: string + module: ivm.Module + }[] = [] + + #generateRandomKey = () => `i${crypto.randomUUID().replace(/-/g, "")}` + + registerModule(module: ivm.Module, imports: string) { + this.#modules.push({ + moduleKey: this.#generateRandomKey(), + import: imports, + module: module, + }) + } + + generateImports() { + return this.#modules + .map(m => `import ${m.import} from "${m.moduleKey}"`) + .join(";") + } + + getModule(key: string) { + const module = this.#modules.find(m => m.moduleKey === key) + return module?.module + } +} + export class IsolatedVM implements VM { #isolate: ivm.Isolate #vm: ivm.Context @@ -21,11 +50,7 @@ export class IsolatedVM implements VM { #timeout: number #perRequestLimit?: number - #modules: { - import: string - moduleKey: string - module: ivm.Module - }[] = [] + #moduleHandler = new ModuleHandler() readonly #resultKey = "results" @@ -96,11 +121,7 @@ export class IsolatedVM implements VM { throw new Error(`No imports allowed. Required: ${specifier}`) }) - this.#modules.push({ - import: "helpers", - moduleKey: `i${crypto.randomUUID().replace(/-/g, "")}`, - module: helpersModule, - }) + this.#moduleHandler.registerModule(helpersModule, "helpers") return this } @@ -121,17 +142,14 @@ export class IsolatedVM implements VM { } } - code = [ - ...this.#modules.map(m => `import ${m.import} from "${m.moduleKey}"`), - `results.out=${code};`, - ].join(";") + code = `${this.#moduleHandler.generateImports()};results.out=${code};` const script = this.#isolate.compileModuleSync(code) script.instantiateSync(this.#vm, specifier => { - const module = this.#modules.find(m => m.moduleKey === specifier) + const module = this.#moduleHandler.getModule(specifier) if (module) { - return module.module + return module } throw new Error(`"${specifier}" import not allowed`)