diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts index cc052ca505..0f4c2106d0 100644 --- a/packages/backend-core/src/context/types.ts +++ b/packages/backend-core/src/context/types.ts @@ -1,5 +1,4 @@ -import { IdentityContext } from "@budibase/types" -import { Isolate, Context, Module } from "isolated-vm" +import { IdentityContext, VM } from "@budibase/types" // keep this out of Budibase types, don't want to expose context info export type ContextMap = { @@ -10,9 +9,5 @@ export type ContextMap = { isScim?: boolean automationId?: string isMigrating?: boolean - isolateRefs?: { - jsIsolate: Isolate - jsContext: Context - helpersModule: Module - } + vm?: VM } diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 3a3b9cdaab..8d18fb97fd 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -255,7 +255,8 @@ export async function listAllObjects(bucketName: string, path: string) { objects = objects.concat(response.Contents) } isTruncated = !!response.IsTruncated - } while (isTruncated) + token = response.NextContinuationToken + } while (isTruncated && token) return objects } diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index b95dace5b2..0bcb25a35f 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -2,7 +2,7 @@ import env from "../environment" import { getRedisOptions } from "../redis/utils" import { JobQueue } from "./constants" import InMemoryQueue from "./inMemoryQueue" -import BullQueue, { QueueOptions } from "bull" +import BullQueue, { QueueOptions, JobOptions } from "bull" import { addListeners, StalledFn } from "./listeners" import { Duration } from "../utils" import * as timers from "../timers" @@ -24,17 +24,24 @@ async function cleanup() { export function createQueue( jobQueue: JobQueue, - opts: { removeStalledCb?: StalledFn } = {} + opts: { + removeStalledCb?: StalledFn + maxStalledCount?: number + jobOptions?: JobOptions + } = {} ): BullQueue.Queue { const redisOpts = getRedisOptions() const queueConfig: QueueOptions = { redis: redisOpts, settings: { - maxStalledCount: 0, + maxStalledCount: opts.maxStalledCount ? opts.maxStalledCount : 0, lockDuration: QUEUE_LOCK_MS, lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS, }, } + if (opts.jobOptions) { + queueConfig.defaultJobOptions = opts.jobOptions + } let queue: any if (!env.isTest()) { queue = new BullQueue(jobQueue, queueConfig) diff --git a/packages/builder/src/components/common/CodeEditor/index.js b/packages/builder/src/components/common/CodeEditor/index.js index 38d377b47a..0d71a475f0 100644 --- a/packages/builder/src/components/common/CodeEditor/index.js +++ b/packages/builder/src/components/common/CodeEditor/index.js @@ -286,7 +286,13 @@ export const hbInsert = (value, from, to, text) => { return parsedInsert } -export function jsInsert(value, from, to, text, { helper, disableWrapping }) { +export function jsInsert( + value, + from, + to, + text, + { helper, disableWrapping } = {} +) { let parsedInsert = "" const left = from ? value.substring(0, from) : "" diff --git a/packages/pro b/packages/pro index aaf7101cd1..9b14e5d518 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit aaf7101cd1493215155cc8f83124c70d53eb1be4 +Subproject commit 9b14e5d5182bf5e5ee98f717997e7352e5904799 diff --git a/packages/server/src/jsRunner/index.ts b/packages/server/src/jsRunner/index.ts index 9c54779567..20a48916c8 100644 --- a/packages/server/src/jsRunner/index.ts +++ b/packages/server/src/jsRunner/index.ts @@ -1,155 +1,28 @@ -import ivm from "isolated-vm" import env from "../environment" import { setJSRunner, JsErrorTimeout } from "@budibase/string-templates" -import { context } from "@budibase/backend-core" import tracer from "dd-trace" -import url from "url" -import crypto from "crypto" -import querystring from "querystring" -import { BundleType, loadBundle } from "./bundles" -class ExecutionTimeoutError extends Error { - constructor(message: string) { - super(message) - this.name = "ExecutionTimeoutError" - } -} +import { IsolatedVM } from "./vm" +import { context } from "@budibase/backend-core" export function init() { - const helpersSource = loadBundle(BundleType.HELPERS) setJSRunner((js: string, ctx: Record) => { return tracer.trace("runJS", {}, span => { try { const bbCtx = context.getCurrentContext()! - const isolateRefs = bbCtx.isolateRefs - if (!isolateRefs) { - const jsIsolate = new ivm.Isolate({ + let { vm } = bbCtx + if (!vm) { + vm = new IsolatedVM({ memoryLimit: env.JS_RUNNER_MEMORY_LIMIT, - }) - const jsContext = jsIsolate.createContextSync() + timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS, + perRequestLimit: env.JS_PER_REQUEST_TIME_LIMIT_MS, + }).withHelpers() - const injectedRequire = ` - const require = function(val){ - switch (val) { - case "url": - return { - resolve: (...params) => urlResolveCb(...params), - parse: (...params) => urlParseCb(...params), - } - case "querystring": - return { - escape: (...params) => querystringEscapeCb(...params), - } - } - };` - - const global = jsContext.global - global.setSync( - "urlResolveCb", - new ivm.Callback((...params: Parameters) => - url.resolve(...params) - ) - ) - - global.setSync( - "urlParseCb", - new ivm.Callback((...params: Parameters) => - url.parse(...params) - ) - ) - - global.setSync( - "querystringEscapeCb", - new ivm.Callback( - (...params: Parameters) => - querystring.escape(...params) - ) - ) - - global.setSync( - "helpersStripProtocol", - new ivm.Callback((str: string) => { - var parsed = url.parse(str) as any - parsed.protocol = "" - return parsed.format() - }) - ) - - const helpersModule = jsIsolate.compileModuleSync( - `${injectedRequire};${helpersSource}` - ) - - const cryptoModule = jsIsolate.compileModuleSync( - `export default { randomUUID: cryptoRandomUUIDCb }` - ) - cryptoModule.instantiateSync(jsContext, specifier => { - throw new Error(`No imports allowed. Required: ${specifier}`) - }) - - global.setSync( - "cryptoRandomUUIDCb", - new ivm.Callback( - (...params: Parameters) => { - return crypto.randomUUID(...params) - } - ) - ) - - helpersModule.instantiateSync(jsContext, specifier => { - if (specifier === "crypto") { - return cryptoModule - } - throw new Error(`No imports allowed. Required: ${specifier}`) - }) - - for (const [key, value] of Object.entries(ctx)) { - if (key === "helpers") { - // Can't copy the native helpers into the isolate. We just ignore them as they are handled properly from the helpersSource - continue - } - global.setSync(key, value) - } - - bbCtx.isolateRefs = { jsContext, jsIsolate, helpersModule } + bbCtx.vm = vm } - let { jsIsolate, jsContext, helpersModule } = bbCtx.isolateRefs! - - const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS - if (perRequestLimit) { - const cpuMs = Number(jsIsolate.cpuTime) / 1e6 - if (cpuMs > perRequestLimit) { - throw new ExecutionTimeoutError( - `CPU time limit exceeded (${cpuMs}ms > ${perRequestLimit}ms)` - ) - } - } - - const script = jsIsolate.compileModuleSync( - `import helpers from "compiled_module";const result=${js};cb(result)`, - {} - ) - - script.instantiateSync(jsContext, specifier => { - if (specifier === "compiled_module") { - return helpersModule - } - - throw new Error(`"${specifier}" import not allowed`) - }) - - let result - jsContext.global.setSync( - "cb", - new ivm.Callback((value: any) => { - result = value - }) - ) - - script.evaluateSync({ - timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS, - }) + const result = vm.execute(js) return result } catch (error: any) { diff --git a/packages/server/src/jsRunner/vm/index.ts b/packages/server/src/jsRunner/vm/index.ts new file mode 100644 index 0000000000..1e3a47e93e --- /dev/null +++ b/packages/server/src/jsRunner/vm/index.ts @@ -0,0 +1,240 @@ +import ivm from "isolated-vm" +import bson from "bson" + +import url from "url" +import crypto from "crypto" +import querystring from "querystring" + +import { BundleType, loadBundle } from "../bundles" +import { VM } from "@budibase/types" +import { context } from "@budibase/backend-core" + +class ExecutionTimeoutError extends Error { + constructor(message: string) { + super(message) + this.name = "ExecutionTimeoutError" + } +} + +export class IsolatedVM implements VM { + #isolate: ivm.Isolate + #vm: ivm.Context + #jail: ivm.Reference + #timeout: number + #perRequestLimit?: number + + #parseBson?: boolean + + #modules: { + import: string + moduleKey: string + module: ivm.Module + }[] = [] + + readonly #resultKey = "results" + + constructor({ + memoryLimit, + timeout, + perRequestLimit, + }: { + memoryLimit: number + timeout: number + perRequestLimit?: number + }) { + this.#isolate = new ivm.Isolate({ memoryLimit }) + this.#vm = this.#isolate.createContextSync() + this.#jail = this.#vm.global + this.#jail.setSync("global", this.#jail.derefInto()) + + this.#addToContext({ + [this.#resultKey]: { out: "" }, + }) + + this.#timeout = timeout + this.#perRequestLimit = perRequestLimit + } + + withHelpers() { + const urlModule = this.#registerCallbacks({ + resolve: url.resolve, + parse: url.parse, + }) + + const querystringModule = this.#registerCallbacks({ + escape: querystring.escape, + }) + + this.#addToContext({ + helpersStripProtocol: new ivm.Callback((str: string) => { + var parsed = url.parse(str) as any + parsed.protocol = "" + return parsed.format() + }), + }) + + const injectedRequire = `const require=function req(val) { + switch (val) { + case "url": return ${urlModule}; + case "querystring": return ${querystringModule}; + } + }` + const helpersSource = loadBundle(BundleType.HELPERS) + const helpersModule = this.#isolate.compileModuleSync( + `${injectedRequire};${helpersSource}` + ) + + helpersModule.instantiateSync(this.#vm, specifier => { + if (specifier === "crypto") { + const cryptoModule = this.#registerCallbacks({ + randomUUID: crypto.randomUUID, + }) + const module = this.#isolate.compileModuleSync( + `export default ${cryptoModule}` + ) + module.instantiateSync(this.#vm, specifier => { + throw new Error(`No imports allowed. Required: ${specifier}`) + }) + return module + } + throw new Error(`No imports allowed. Required: ${specifier}`) + }) + + this.#modules.push({ + import: "helpers", + moduleKey: `i${crypto.randomUUID().replace(/-/g, "")}`, + module: helpersModule, + }) + return this + } + + withContext(context: Record) { + this.#addToContext(context) + this.#handleBsonData() + return this + } + + withParsingBson() { + this.#parseBson = true + this.#handleBsonData() + + const bsonSource = loadBundle(BundleType.BSON) + const bsonModule = this.#isolate.compileModuleSync(bsonSource) + bsonModule.instantiateSync(this.#vm, specifier => { + throw new Error(`No imports allowed. Required: ${specifier}`) + }) + + this.#modules.push({ + import: "{deserialize, toJson}", + moduleKey: `i${crypto.randomUUID().replace(/-/g, "")}`, + module: bsonModule, + }) + + return this + } + + #handleBsonData() { + if (!this.#parseBson) { + return + } + + const data = this.#getFromContext("data") + if (!data) { + return + } + + this.#addToContext({ + data: bson.BSON.serialize({ data }), + }) + } + + execute(code: string): string { + const perRequestLimit = this.#perRequestLimit + + if (perRequestLimit) { + const cpuMs = Number(this.#isolate.cpuTime) / 1e6 + if (cpuMs > perRequestLimit) { + throw new ExecutionTimeoutError( + `CPU time limit exceeded (${cpuMs}ms > ${perRequestLimit}ms)` + ) + } + } + + if (this.#parseBson) { + // If we need to parse bson, we follow the next steps: + // 1. Serialise the data from potential BSON to buffer before passing it to the isolate + // 2. Deserialise the data within the isolate, to get the original data + // 3. Process script + // 4. Stringify the result in order to convert the result from BSON to json + code = `toJson( + (function(){ + data= deserialize(data).data; + return ${code}; + })() + );` + } + + code = [ + ...this.#modules.map(m => `import ${m.import} from "${m.moduleKey}"`), + `results.out=${code};`, + ].join(";") + + const script = this.#isolate.compileModuleSync(code) + + script.instantiateSync(this.#vm, specifier => { + const module = this.#modules.find(m => m.moduleKey === specifier) + if (module) { + return module.module + } + + throw new Error(`"${specifier}" import not allowed`) + }) + + script.evaluateSync({ timeout: this.#timeout }) + + const result = this.#getResult() + return result + } + + #registerCallbacks(functions: Record) { + const libId = crypto.randomUUID().replace(/-/g, "") + + const x: Record = {} + for (const [funcName, func] of Object.entries(functions)) { + const key = `f${libId}${funcName}cb` + x[funcName] = key + + this.#addToContext({ + [key]: new ivm.Callback((...params: any[]) => (func as any)(...params)), + }) + } + + const mod = + `{` + + Object.entries(x) + .map(([key, func]) => `${key}: ${func}`) + .join() + + "}" + return mod + } + + #addToContext(context: Record) { + for (let key in context) { + this.#jail.setSync( + key, + new ivm.ExternalCopy(context[key]).copyInto({ release: true }) + ) + } + } + + #getFromContext(key: string) { + return this.#jail.getSync(key) + } + + #getResult() { + const ref = this.#vm.global.getSync(this.#resultKey, { reference: true }) + const result = ref.copySync() + ref.release() + return result.out + } +} diff --git a/packages/server/src/utilities/scriptRunner.ts b/packages/server/src/utilities/scriptRunner.ts index 3a8c9a4fb4..3111ec1dc6 100644 --- a/packages/server/src/utilities/scriptRunner.ts +++ b/packages/server/src/utilities/scriptRunner.ts @@ -1,117 +1,139 @@ -import ivm from "isolated-vm" - import bson from "bson" -import { BundleType, loadBundle } from "../jsRunner/bundles" + import env from "../environment" +import { IsolatedVM } from "../jsRunner/vm" const JS_TIMEOUT_MS = 1000 - class ScriptRunner { - vm: IsolatedVM + #code: string + #vm: IsolatedVM constructor(script: string, context: any, { parseBson = false } = {}) { - this.vm = new IsolatedVM({ + this.#code = `(() => {${script}})();` + this.#vm = new IsolatedVM({ memoryLimit: env.JS_RUNNER_MEMORY_LIMIT, - parseBson, - context: { - ...context, - data: parseBson - ? bson.BSON.serialize({ data: context.data }) - : context.data, - }, - }) + timeout: JS_TIMEOUT_MS, + }).withContext(context) if (parseBson) { - // If we need to parse bson, we follow the next steps: - // 1. Serialise the data from potential BSON to buffer before passing it to the isolate - // 2. Deserialise the data within the isolate, to get the original data - // 3. Process script - // 4. Stringify the result in order to convert the result from BSON to json - script = `return toJson((function(){data=deserialize(data, { validation: { utf8: true } }).data;${script}})());` + this.#vm = this.#vm.withParsingBson() } - - this.vm.code = script } execute() { - this.vm.runScript() - const result = this.vm.getResult() + const result = this.#vm.execute(this.#code) return result } } -class IsolatedVM { - #isolate: ivm.Isolate - #vm: ivm.Context - #jail: ivm.Reference - #script: ivm.Module = undefined! - #bsonModule?: ivm.Module +// <<<<<<< HEAD +// constructor(script: string, context: any, { parseBson = false } = {}) { +// this.vm = new IsolatedVM({ +// memoryLimit: env.JS_RUNNER_MEMORY_LIMIT, +// parseBson, +// context: { +// ...context, +// data: parseBson +// ? bson.BSON.serialize({ data: context.data }) +// : context.data, +// }, +// }) - readonly #resultKey = "results" +// if (parseBson) { +// // If we need to parse bson, we follow the next steps: +// // 1. Serialise the data from potential BSON to buffer before passing it to the isolate +// // 2. Deserialise the data within the isolate, to get the original data +// // 3. Process script +// // 4. Stringify the result in order to convert the result from BSON to json +// script = `return toJson((function(){data=deserialize(data, { validation: { utf8: true } }).data;${script}})());` +// } - constructor({ - memoryLimit, - parseBson, - context, - }: { - memoryLimit: number - parseBson: boolean - context: Record - }) { - this.#isolate = new ivm.Isolate({ memoryLimit }) - this.#vm = this.#isolate.createContextSync() - this.#jail = this.#vm.global - this.#jail.setSync("global", this.#jail.derefInto()) +// this.vm.code = script +// } - this.#addToContext(context) - this.#addToContext({ - [this.#resultKey]: { out: "" }, - }) +// execute() { +// this.vm.runScript() +// const result = this.vm.getResult() +// return result +// } +// } - if (parseBson) { - const bsonSource = loadBundle(BundleType.BSON) - this.#bsonModule = this.#isolate.compileModuleSync(bsonSource) - this.#bsonModule.instantiateSync(this.#vm, specifier => { - throw new Error(`No imports allowed. Required: ${specifier}`) - }) - } - } +// class IsolatedVM { +// #isolate: ivm.Isolate +// #vm: ivm.Context +// #jail: ivm.Reference +// #script: ivm.Module = undefined! +// #bsonModule?: ivm.Module - getResult() { - const ref = this.#vm.global.getSync(this.#resultKey, { reference: true }) - const result = ref.copySync() - ref.release() - return result.out - } +// readonly #resultKey = "results" - #addToContext(context: Record) { - for (let key in context) { - this.#jail.setSync(key, this.#copyRefToVm(context[key])) - } - } +// constructor({ +// memoryLimit, +// parseBson, +// context, +// }: { +// memoryLimit: number +// parseBson: boolean +// context: Record +// }) { +// this.#isolate = new ivm.Isolate({ memoryLimit }) +// this.#vm = this.#isolate.createContextSync() +// this.#jail = this.#vm.global +// this.#jail.setSync("global", this.#jail.derefInto()) - set code(code: string) { - code = `const fn=function(){${code}};results.out=fn();` - if (this.#bsonModule) { - code = `import {deserialize, toJson} from "compiled_module";${code}` - } - this.#script = this.#isolate.compileModuleSync(code) - } +// this.#addToContext(context) +// this.#addToContext({ +// [this.#resultKey]: { out: "" }, +// }) - runScript(): void { - this.#script.instantiateSync(this.#vm, specifier => { - if (specifier === "compiled_module" && this.#bsonModule) { - return this.#bsonModule! - } +// if (parseBson) { +// const bsonSource = loadBundle(BundleType.BSON) +// this.#bsonModule = this.#isolate.compileModuleSync(bsonSource) +// this.#bsonModule.instantiateSync(this.#vm, specifier => { +// throw new Error(`No imports allowed. Required: ${specifier}`) +// }) +// } +// } - throw new Error(`"${specifier}" import not allowed`) - }) +// getResult() { +// const ref = this.#vm.global.getSync(this.#resultKey, { reference: true }) +// const result = ref.copySync() +// ref.release() +// return result.out +// } - this.#script.evaluateSync({ timeout: JS_TIMEOUT_MS }) - } +// #addToContext(context: Record) { +// for (let key in context) { +// this.#jail.setSync(key, this.#copyRefToVm(context[key])) +// } +// } + +// set code(code: string) { +// code = `const fn=function(){${code}};results.out=fn();` +// if (this.#bsonModule) { +// code = `import {deserialize, toJson} from "compiled_module";${code}` +// } +// this.#script = this.#isolate.compileModuleSync(code) +// } + +// runScript(): void { +// this.#script.instantiateSync(this.#vm, specifier => { +// if (specifier === "compiled_module" && this.#bsonModule) { +// return this.#bsonModule! +// } + +// throw new Error(`"${specifier}" import not allowed`) +// }) + +// this.#script.evaluateSync({ timeout: JS_TIMEOUT_MS }) +// } + +// #copyRefToVm(value: Object): ivm.Copy { +// return new ivm.ExternalCopy(value).copyInto({ release: true }) +// } +// ======= + +// >>>>>>> isolated-vm-wrapper +// } - #copyRefToVm(value: Object): ivm.Copy { - return new ivm.ExternalCopy(value).copyInto({ release: true }) - } -} export default ScriptRunner diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts index 0eab2ba556..36faaae9c3 100644 --- a/packages/types/src/sdk/index.ts +++ b/packages/types/src/sdk/index.ts @@ -20,3 +20,4 @@ export * from "./cli" export * from "./websocket" export * from "./permissions" export * from "./row" +export * from "./vm" diff --git a/packages/types/src/sdk/vm.ts b/packages/types/src/sdk/vm.ts new file mode 100644 index 0000000000..3abec4d39d --- /dev/null +++ b/packages/types/src/sdk/vm.ts @@ -0,0 +1,3 @@ +export interface VM { + execute(code: string): string +}