diff --git a/packages/server/src/api/controllers/script.ts b/packages/server/src/api/controllers/script.ts index d5b99d0733..f00383615e 100644 --- a/packages/server/src/api/controllers/script.ts +++ b/packages/server/src/api/controllers/script.ts @@ -1,10 +1,11 @@ -import ScriptRunner from "../../utilities/scriptRunner" import { Ctx } from "@budibase/types" +import { VM2 } from "../../jsRunner/vm" export async function execute(ctx: Ctx) { const { script, context } = ctx.request.body - const runner = new ScriptRunner(script, context) - ctx.body = runner.execute() + const runner = new VM2(context) + const result = runner.execute(script) + ctx.body = result } export async function save(ctx: Ctx) { diff --git a/packages/server/src/jsRunner/index.ts b/packages/server/src/jsRunner/index.ts index e39dab1313..c572c112c9 100644 --- a/packages/server/src/jsRunner/index.ts +++ b/packages/server/src/jsRunner/index.ts @@ -1,62 +1,16 @@ import vm from "vm" import env from "../environment" import { setJSRunner, setOnErrorLog } from "@budibase/string-templates" -import { context, logging, timers } from "@budibase/backend-core" +import { logging } from "@budibase/backend-core" import tracer from "dd-trace" import { serializeError } from "serialize-error" - -type TrackerFn = (f: () => T) => T +import { BuiltInVM } from "./vm" export function init() { setJSRunner((js: string, ctx: vm.Context) => { return tracer.trace("runJS", {}, span => { - const perRequestLimit = env.JS_PER_REQUEST_TIMEOUT_MS - let track: TrackerFn = f => f() - if (perRequestLimit) { - const bbCtx = tracer.trace("runJS.getCurrentContext", {}, span => - context.getCurrentContext() - ) - if (bbCtx) { - if (!bbCtx.jsExecutionTracker) { - span?.addTags({ - createdExecutionTracker: true, - }) - bbCtx.jsExecutionTracker = tracer.trace( - "runJS.createExecutionTimeTracker", - {}, - span => timers.ExecutionTimeTracker.withLimit(perRequestLimit) - ) - } - span?.addTags({ - js: { - limitMS: bbCtx.jsExecutionTracker.limitMs, - elapsedMS: bbCtx.jsExecutionTracker.elapsedMS, - }, - }) - // We call checkLimit() here to prevent paying the cost of creating - // a new VM context below when we don't need to. - tracer.trace("runJS.checkLimitAndBind", {}, span => { - bbCtx.jsExecutionTracker!.checkLimit() - track = bbCtx.jsExecutionTracker!.track.bind( - bbCtx.jsExecutionTracker - ) - }) - } - } - - ctx = { - ...ctx, - alert: undefined, - setInterval: undefined, - setTimeout: undefined, - } - - vm.createContext(ctx) - return track(() => - vm.runInNewContext(js, ctx, { - timeout: env.JS_PER_INVOCATION_TIMEOUT_MS, - }) - ) + const vm = new BuiltInVM(ctx, span) + return vm.execute(js) }) }) diff --git a/packages/server/src/jsRunner/vm/builtin-vm.ts b/packages/server/src/jsRunner/vm/builtin-vm.ts new file mode 100644 index 0000000000..b4c9f775f9 --- /dev/null +++ b/packages/server/src/jsRunner/vm/builtin-vm.ts @@ -0,0 +1,65 @@ +import vm from "vm" +import env from "../../environment" +import { context, timers } from "@budibase/backend-core" +import tracer, { Span } from "dd-trace" +import { VM } from "@budibase/types" + +type TrackerFn = (f: () => T) => T + +export class BuiltInVM implements VM { + private ctx: vm.Context + private span?: Span + + constructor(ctx: vm.Context, span?: Span) { + this.ctx = ctx + this.span = span + } + + execute(code: string) { + const perRequestLimit = env.JS_PER_REQUEST_TIMEOUT_MS + let track: TrackerFn = f => f() + if (perRequestLimit) { + const bbCtx = tracer.trace("runJS.getCurrentContext", {}, span => + context.getCurrentContext() + ) + if (bbCtx) { + if (!bbCtx.jsExecutionTracker) { + this.span?.addTags({ + createdExecutionTracker: true, + }) + bbCtx.jsExecutionTracker = tracer.trace( + "runJS.createExecutionTimeTracker", + {}, + span => timers.ExecutionTimeTracker.withLimit(perRequestLimit) + ) + } + this.span?.addTags({ + js: { + limitMS: bbCtx.jsExecutionTracker.limitMs, + elapsedMS: bbCtx.jsExecutionTracker.elapsedMS, + }, + }) + // We call checkLimit() here to prevent paying the cost of creating + // a new VM context below when we don't need to. + tracer.trace("runJS.checkLimitAndBind", {}, span => { + bbCtx.jsExecutionTracker!.checkLimit() + track = bbCtx.jsExecutionTracker!.track.bind(bbCtx.jsExecutionTracker) + }) + } + } + + this.ctx = { + ...this.ctx, + alert: undefined, + setInterval: undefined, + setTimeout: undefined, + } + + vm.createContext(this.ctx) + return track(() => + vm.runInNewContext(code, this.ctx, { + timeout: env.JS_PER_INVOCATION_TIMEOUT_MS, + }) + ) + } +} diff --git a/packages/server/src/jsRunner/vm/index.ts b/packages/server/src/jsRunner/vm/index.ts index ab26f3f6d1..01e0daa354 100644 --- a/packages/server/src/jsRunner/vm/index.ts +++ b/packages/server/src/jsRunner/vm/index.ts @@ -1,270 +1,3 @@ -import ivm from "isolated-vm" -import bson from "bson" - -import url from "url" -import crypto from "crypto" -import querystring from "querystring" - -import { BundleType, loadBundle } from "../bundles" -import { VM } from "@budibase/types" - -class ExecutionTimeoutError extends Error { - constructor(message: string) { - super(message) - this.name = "ExecutionTimeoutError" - } -} - -class ModuleHandler { - private modules: { - import: string - moduleKey: string - module: ivm.Module - }[] = [] - - private generateRandomKey = () => `i${crypto.randomUUID().replace(/-/g, "")}` - - registerModule(module: ivm.Module, imports: string) { - this.modules.push({ - moduleKey: this.generateRandomKey(), - import: imports, - module: module, - }) - } - - generateImports() { - return this.modules - .map(m => `import ${m.import} from "${m.moduleKey}"`) - .join(";") - } - - getModule(key: string) { - const module = this.modules.find(m => m.moduleKey === key) - return module?.module - } -} - -export class IsolatedVM implements VM { - private isolate: ivm.Isolate - private vm: ivm.Context - private jail: ivm.Reference - private invocationTimeout: number - private isolateAccumulatedTimeout?: number - - // By default the wrapper returns itself - private codeWrapper: (code: string) => string = code => code - - private moduleHandler = new ModuleHandler() - - private readonly resultKey = "results" - - constructor({ - memoryLimit, - invocationTimeout, - isolateAccumulatedTimeout, - }: { - memoryLimit: number - invocationTimeout: number - isolateAccumulatedTimeout?: number - }) { - this.isolate = new ivm.Isolate({ memoryLimit }) - this.vm = this.isolate.createContextSync() - this.jail = this.vm.global - this.jail.setSync("global", this.jail.derefInto()) - - this.addToContext({ - [this.resultKey]: { out: "" }, - }) - - this.invocationTimeout = invocationTimeout - this.isolateAccumulatedTimeout = isolateAccumulatedTimeout - } - - withHelpers() { - const urlModule = this.registerCallbacks({ - resolve: url.resolve, - parse: url.parse, - }) - - const querystringModule = this.registerCallbacks({ - escape: querystring.escape, - }) - - this.addToContext({ - helpersStripProtocol: new ivm.Callback((str: string) => { - var parsed = url.parse(str) as any - parsed.protocol = "" - return parsed.format() - }), - }) - - const injectedRequire = `const require=function req(val) { - switch (val) { - case "url": return ${urlModule}; - case "querystring": return ${querystringModule}; - } - }` - const helpersSource = loadBundle(BundleType.HELPERS) - const helpersModule = this.isolate.compileModuleSync( - `${injectedRequire};${helpersSource}` - ) - - helpersModule.instantiateSync(this.vm, specifier => { - if (specifier === "crypto") { - const cryptoModule = this.registerCallbacks({ - randomUUID: crypto.randomUUID, - }) - const module = this.isolate.compileModuleSync( - `export default ${cryptoModule}` - ) - module.instantiateSync(this.vm, specifier => { - throw new Error(`No imports allowed. Required: ${specifier}`) - }) - return module - } - throw new Error(`No imports allowed. Required: ${specifier}`) - }) - - this.moduleHandler.registerModule(helpersModule, "helpers") - return this - } - - withContext(context: Record) { - this.addToContext(context) - - return this - } - - withParsingBson(data: any) { - this.addToContext({ - bsonData: bson.BSON.serialize({ data }), - }) - - // If we need to parse bson, we follow the next steps: - // 1. Serialise the data from potential BSON to buffer before passing it to the isolate - // 2. Deserialise the data within the isolate, to get the original data - // 3. Process script - // 4. Stringify the result in order to convert the result from BSON to json - this.codeWrapper = code => - `(function(){ - const data = deserialize(bsonData, { validation: { utf8: false } }).data; - const result = ${code} - return toJson(result); - })();` - - const bsonSource = loadBundle(BundleType.BSON) - - this.addToContext({ - textDecoderCb: new ivm.Callback( - (args: { - constructorArgs: any - functionArgs: Parameters["decode"]> - }) => { - const result = new TextDecoder(...args.constructorArgs).decode( - ...args.functionArgs - ) - return result - } - ), - }) - - // "Polyfilling" text decoder. `bson.deserialize` requires decoding. We are creating a bridge function so we don't need to inject the full library - const textDecoderPolyfill = class TextDecoder { - constructorArgs - - constructor(...constructorArgs: any) { - this.constructorArgs = constructorArgs - } - - decode(...input: any) { - // @ts-ignore - return textDecoderCb({ - constructorArgs: this.constructorArgs, - functionArgs: input, - }) - } - }.toString() - const bsonModule = this.isolate.compileModuleSync( - `${textDecoderPolyfill};${bsonSource}` - ) - bsonModule.instantiateSync(this.vm, specifier => { - throw new Error(`No imports allowed. Required: ${specifier}`) - }) - - this.moduleHandler.registerModule(bsonModule, "{deserialize, toJson}") - - return this - } - - execute(code: string): any { - if (this.isolateAccumulatedTimeout) { - const cpuMs = Number(this.isolate.cpuTime) / 1e6 - if (cpuMs > this.isolateAccumulatedTimeout) { - throw new ExecutionTimeoutError( - `CPU time limit exceeded (${cpuMs}ms > ${this.isolateAccumulatedTimeout}ms)` - ) - } - } - - code = `${this.moduleHandler.generateImports()};results.out=${this.codeWrapper( - code - )};` - - const script = this.isolate.compileModuleSync(code) - - script.instantiateSync(this.vm, specifier => { - const module = this.moduleHandler.getModule(specifier) - if (module) { - return module - } - - throw new Error(`"${specifier}" import not allowed`) - }) - - script.evaluateSync({ timeout: this.invocationTimeout }) - - const result = this.getFromContext(this.resultKey) - return result.out - } - - private registerCallbacks(functions: Record) { - const libId = crypto.randomUUID().replace(/-/g, "") - - const x: Record = {} - for (const [funcName, func] of Object.entries(functions)) { - const key = `f${libId}${funcName}cb` - x[funcName] = key - - this.addToContext({ - [key]: new ivm.Callback((...params: any[]) => (func as any)(...params)), - }) - } - - const mod = - `{` + - Object.entries(x) - .map(([key, func]) => `${key}: ${func}`) - .join() + - "}" - return mod - } - - private addToContext(context: Record) { - for (let key in context) { - const value = context[key] - this.jail.setSync( - key, - typeof value === "function" - ? value - : new ivm.ExternalCopy(value).copyInto({ release: true }) - ) - } - } - - private getFromContext(key: string) { - const ref = this.vm.global.getSync(key, { reference: true }) - const result = ref.copySync() - ref.release() - return result - } -} +export * from "./isolated-vm" +export * from "./builtin-vm" +export * from "./vm2" diff --git a/packages/server/src/jsRunner/vm/isolated-vm.ts b/packages/server/src/jsRunner/vm/isolated-vm.ts new file mode 100644 index 0000000000..ab26f3f6d1 --- /dev/null +++ b/packages/server/src/jsRunner/vm/isolated-vm.ts @@ -0,0 +1,270 @@ +import ivm from "isolated-vm" +import bson from "bson" + +import url from "url" +import crypto from "crypto" +import querystring from "querystring" + +import { BundleType, loadBundle } from "../bundles" +import { VM } from "@budibase/types" + +class ExecutionTimeoutError extends Error { + constructor(message: string) { + super(message) + this.name = "ExecutionTimeoutError" + } +} + +class ModuleHandler { + private modules: { + import: string + moduleKey: string + module: ivm.Module + }[] = [] + + private generateRandomKey = () => `i${crypto.randomUUID().replace(/-/g, "")}` + + registerModule(module: ivm.Module, imports: string) { + this.modules.push({ + moduleKey: this.generateRandomKey(), + import: imports, + module: module, + }) + } + + generateImports() { + return this.modules + .map(m => `import ${m.import} from "${m.moduleKey}"`) + .join(";") + } + + getModule(key: string) { + const module = this.modules.find(m => m.moduleKey === key) + return module?.module + } +} + +export class IsolatedVM implements VM { + private isolate: ivm.Isolate + private vm: ivm.Context + private jail: ivm.Reference + private invocationTimeout: number + private isolateAccumulatedTimeout?: number + + // By default the wrapper returns itself + private codeWrapper: (code: string) => string = code => code + + private moduleHandler = new ModuleHandler() + + private readonly resultKey = "results" + + constructor({ + memoryLimit, + invocationTimeout, + isolateAccumulatedTimeout, + }: { + memoryLimit: number + invocationTimeout: number + isolateAccumulatedTimeout?: number + }) { + this.isolate = new ivm.Isolate({ memoryLimit }) + this.vm = this.isolate.createContextSync() + this.jail = this.vm.global + this.jail.setSync("global", this.jail.derefInto()) + + this.addToContext({ + [this.resultKey]: { out: "" }, + }) + + this.invocationTimeout = invocationTimeout + this.isolateAccumulatedTimeout = isolateAccumulatedTimeout + } + + withHelpers() { + const urlModule = this.registerCallbacks({ + resolve: url.resolve, + parse: url.parse, + }) + + const querystringModule = this.registerCallbacks({ + escape: querystring.escape, + }) + + this.addToContext({ + helpersStripProtocol: new ivm.Callback((str: string) => { + var parsed = url.parse(str) as any + parsed.protocol = "" + return parsed.format() + }), + }) + + const injectedRequire = `const require=function req(val) { + switch (val) { + case "url": return ${urlModule}; + case "querystring": return ${querystringModule}; + } + }` + const helpersSource = loadBundle(BundleType.HELPERS) + const helpersModule = this.isolate.compileModuleSync( + `${injectedRequire};${helpersSource}` + ) + + helpersModule.instantiateSync(this.vm, specifier => { + if (specifier === "crypto") { + const cryptoModule = this.registerCallbacks({ + randomUUID: crypto.randomUUID, + }) + const module = this.isolate.compileModuleSync( + `export default ${cryptoModule}` + ) + module.instantiateSync(this.vm, specifier => { + throw new Error(`No imports allowed. Required: ${specifier}`) + }) + return module + } + throw new Error(`No imports allowed. Required: ${specifier}`) + }) + + this.moduleHandler.registerModule(helpersModule, "helpers") + return this + } + + withContext(context: Record) { + this.addToContext(context) + + return this + } + + withParsingBson(data: any) { + this.addToContext({ + bsonData: bson.BSON.serialize({ data }), + }) + + // If we need to parse bson, we follow the next steps: + // 1. Serialise the data from potential BSON to buffer before passing it to the isolate + // 2. Deserialise the data within the isolate, to get the original data + // 3. Process script + // 4. Stringify the result in order to convert the result from BSON to json + this.codeWrapper = code => + `(function(){ + const data = deserialize(bsonData, { validation: { utf8: false } }).data; + const result = ${code} + return toJson(result); + })();` + + const bsonSource = loadBundle(BundleType.BSON) + + this.addToContext({ + textDecoderCb: new ivm.Callback( + (args: { + constructorArgs: any + functionArgs: Parameters["decode"]> + }) => { + const result = new TextDecoder(...args.constructorArgs).decode( + ...args.functionArgs + ) + return result + } + ), + }) + + // "Polyfilling" text decoder. `bson.deserialize` requires decoding. We are creating a bridge function so we don't need to inject the full library + const textDecoderPolyfill = class TextDecoder { + constructorArgs + + constructor(...constructorArgs: any) { + this.constructorArgs = constructorArgs + } + + decode(...input: any) { + // @ts-ignore + return textDecoderCb({ + constructorArgs: this.constructorArgs, + functionArgs: input, + }) + } + }.toString() + const bsonModule = this.isolate.compileModuleSync( + `${textDecoderPolyfill};${bsonSource}` + ) + bsonModule.instantiateSync(this.vm, specifier => { + throw new Error(`No imports allowed. Required: ${specifier}`) + }) + + this.moduleHandler.registerModule(bsonModule, "{deserialize, toJson}") + + return this + } + + execute(code: string): any { + if (this.isolateAccumulatedTimeout) { + const cpuMs = Number(this.isolate.cpuTime) / 1e6 + if (cpuMs > this.isolateAccumulatedTimeout) { + throw new ExecutionTimeoutError( + `CPU time limit exceeded (${cpuMs}ms > ${this.isolateAccumulatedTimeout}ms)` + ) + } + } + + code = `${this.moduleHandler.generateImports()};results.out=${this.codeWrapper( + code + )};` + + const script = this.isolate.compileModuleSync(code) + + script.instantiateSync(this.vm, specifier => { + const module = this.moduleHandler.getModule(specifier) + if (module) { + return module + } + + throw new Error(`"${specifier}" import not allowed`) + }) + + script.evaluateSync({ timeout: this.invocationTimeout }) + + const result = this.getFromContext(this.resultKey) + return result.out + } + + private registerCallbacks(functions: Record) { + const libId = crypto.randomUUID().replace(/-/g, "") + + const x: Record = {} + for (const [funcName, func] of Object.entries(functions)) { + const key = `f${libId}${funcName}cb` + x[funcName] = key + + this.addToContext({ + [key]: new ivm.Callback((...params: any[]) => (func as any)(...params)), + }) + } + + const mod = + `{` + + Object.entries(x) + .map(([key, func]) => `${key}: ${func}`) + .join() + + "}" + return mod + } + + private addToContext(context: Record) { + for (let key in context) { + const value = context[key] + this.jail.setSync( + key, + typeof value === "function" + ? value + : new ivm.ExternalCopy(value).copyInto({ release: true }) + ) + } + } + + private getFromContext(key: string) { + const ref = this.vm.global.getSync(key, { reference: true }) + const result = ref.copySync() + ref.release() + return result + } +} diff --git a/packages/server/src/utilities/scriptRunner.ts b/packages/server/src/jsRunner/vm/vm2.ts similarity index 54% rename from packages/server/src/utilities/scriptRunner.ts rename to packages/server/src/jsRunner/vm/vm2.ts index fee0215d2e..6d05943d25 100644 --- a/packages/server/src/utilities/scriptRunner.ts +++ b/packages/server/src/jsRunner/vm/vm2.ts @@ -1,29 +1,26 @@ -import fetch from "node-fetch" -import { VM, VMScript } from "vm2" +import vm2 from "vm2" +import { VM } from "@budibase/types" const JS_TIMEOUT_MS = 1000 -class ScriptRunner { - vm: VM +export class VM2 implements VM { + vm: vm2.VM results: { out: string } - script: VMScript - constructor(script: string, context: any) { - const code = `let fn = () => {\n${script}\n}; results.out = fn();` - this.vm = new VM({ + constructor(context: any) { + this.vm = new vm2.VM({ timeout: JS_TIMEOUT_MS, }) this.results = { out: "" } this.vm.setGlobals(context) this.vm.setGlobal("fetch", fetch) this.vm.setGlobal("results", this.results) - this.script = new VMScript(code) } - execute() { - this.vm.run(this.script) + execute(script: string) { + const code = `let fn = () => {\n${script}\n}; results.out = fn();` + const vmScript = new vm2.VMScript(code) + this.vm.run(vmScript) return this.results.out } } - -export default ScriptRunner diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts index 6b11ce4759..429d058ef7 100644 --- a/packages/server/src/threads/query.ts +++ b/packages/server/src/threads/query.ts @@ -7,7 +7,7 @@ import { QueryVariable, QueryResponse, } from "./definitions" -import ScriptRunner from "../utilities/scriptRunner" +import { VM2 } from "../jsRunner/vm" import { getIntegration } from "../integrations" import { processStringSync } from "@budibase/string-templates" import { context, cache, auth } from "@budibase/backend-core" @@ -26,7 +26,7 @@ class QueryRunner { fields: any parameters: any pagination: any - transformer: any + transformer: string cachedVariables: any[] ctx: any queryResponse: any @@ -127,11 +127,11 @@ class QueryRunner { // transform as required if (transformer) { - const runner = new ScriptRunner(transformer, { + const runner = new VM2({ data: rows, params: enrichedParameters, }) - rows = runner.execute() + rows = runner.execute(transformer) } // if the request fails we retry once, invalidating the cached value