@@ -42,7 +45,7 @@
on:change
>
-
+
{item.field}
diff --git a/packages/server/src/api/controllers/script.ts b/packages/server/src/api/controllers/script.ts
index d5b99d0733..f00383615e 100644
--- a/packages/server/src/api/controllers/script.ts
+++ b/packages/server/src/api/controllers/script.ts
@@ -1,10 +1,11 @@
-import ScriptRunner from "../../utilities/scriptRunner"
import { Ctx } from "@budibase/types"
+import { VM2 } from "../../jsRunner/vm"
export async function execute(ctx: Ctx) {
const { script, context } = ctx.request.body
- const runner = new ScriptRunner(script, context)
- ctx.body = runner.execute()
+ const runner = new VM2(context)
+ const result = runner.execute(script)
+ ctx.body = result
}
export async function save(ctx: Ctx) {
diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts
index 20142776b8..bc4b9eb35b 100644
--- a/packages/server/src/environment.ts
+++ b/packages/server/src/environment.ts
@@ -126,6 +126,10 @@ const environment = {
getDefaults: () => {
return DEFAULTS
},
+ useIsolatedVM: {
+ QUERY_TRANSFORMERS: !!process.env.QUERY_TRANSFORMERS_ISOLATEDVM,
+ JS_RUNNER: !!process.env.JS_RUNNER_ISOLATEDVM,
+ },
}
// clean up any environment variable edge cases
diff --git a/packages/server/src/jsRunner/index.ts b/packages/server/src/jsRunner/index.ts
index 93383b3955..5c863e2855 100644
--- a/packages/server/src/jsRunner/index.ts
+++ b/packages/server/src/jsRunner/index.ts
@@ -1,13 +1,17 @@
import env from "../environment"
-import { setJSRunner, JsErrorTimeout } from "@budibase/string-templates"
-import tracer from "dd-trace"
-
-import { IsolatedVM } from "./vm"
+import { JsErrorTimeout, setJSRunner } from "@budibase/string-templates"
import { context } from "@budibase/backend-core"
+import tracer from "dd-trace"
+import { BuiltInVM, IsolatedVM } from "./vm"
export function init() {
setJSRunner((js: string, ctx: Record
) => {
return tracer.trace("runJS", {}, span => {
+ if (!env.useIsolatedVM.JS_RUNNER) {
+ const vm = new BuiltInVM(ctx, span)
+ return vm.execute(js)
+ }
+
try {
const bbCtx = context.getCurrentContext()!
@@ -26,9 +30,7 @@ export function init() {
bbCtx.vm = vm
}
-
const result = vm.execute(js)
-
return result
} catch (error: any) {
if (error.message === "Script execution timed out.") {
diff --git a/packages/server/src/jsRunner/vm/builtin-vm.ts b/packages/server/src/jsRunner/vm/builtin-vm.ts
new file mode 100644
index 0000000000..b4c9f775f9
--- /dev/null
+++ b/packages/server/src/jsRunner/vm/builtin-vm.ts
@@ -0,0 +1,65 @@
+import vm from "vm"
+import env from "../../environment"
+import { context, timers } from "@budibase/backend-core"
+import tracer, { Span } from "dd-trace"
+import { VM } from "@budibase/types"
+
+type TrackerFn = (f: () => T) => T
+
+export class BuiltInVM implements VM {
+ private ctx: vm.Context
+ private span?: Span
+
+ constructor(ctx: vm.Context, span?: Span) {
+ this.ctx = ctx
+ this.span = span
+ }
+
+ execute(code: string) {
+ const perRequestLimit = env.JS_PER_REQUEST_TIMEOUT_MS
+ let track: TrackerFn = f => f()
+ if (perRequestLimit) {
+ const bbCtx = tracer.trace("runJS.getCurrentContext", {}, span =>
+ context.getCurrentContext()
+ )
+ if (bbCtx) {
+ if (!bbCtx.jsExecutionTracker) {
+ this.span?.addTags({
+ createdExecutionTracker: true,
+ })
+ bbCtx.jsExecutionTracker = tracer.trace(
+ "runJS.createExecutionTimeTracker",
+ {},
+ span => timers.ExecutionTimeTracker.withLimit(perRequestLimit)
+ )
+ }
+ this.span?.addTags({
+ js: {
+ limitMS: bbCtx.jsExecutionTracker.limitMs,
+ elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
+ },
+ })
+ // We call checkLimit() here to prevent paying the cost of creating
+ // a new VM context below when we don't need to.
+ tracer.trace("runJS.checkLimitAndBind", {}, span => {
+ bbCtx.jsExecutionTracker!.checkLimit()
+ track = bbCtx.jsExecutionTracker!.track.bind(bbCtx.jsExecutionTracker)
+ })
+ }
+ }
+
+ this.ctx = {
+ ...this.ctx,
+ alert: undefined,
+ setInterval: undefined,
+ setTimeout: undefined,
+ }
+
+ vm.createContext(this.ctx)
+ return track(() =>
+ vm.runInNewContext(code, this.ctx, {
+ timeout: env.JS_PER_INVOCATION_TIMEOUT_MS,
+ })
+ )
+ }
+}
diff --git a/packages/server/src/jsRunner/vm/index.ts b/packages/server/src/jsRunner/vm/index.ts
index 0285af8620..01e0daa354 100644
--- a/packages/server/src/jsRunner/vm/index.ts
+++ b/packages/server/src/jsRunner/vm/index.ts
@@ -1,232 +1,3 @@
-import ivm from "isolated-vm"
-import bson from "bson"
-
-import url from "url"
-import crypto from "crypto"
-import querystring from "querystring"
-
-import { BundleType, loadBundle } from "../bundles"
-import { VM } from "@budibase/types"
-
-class ExecutionTimeoutError extends Error {
- constructor(message: string) {
- super(message)
- this.name = "ExecutionTimeoutError"
- }
-}
-
-export class IsolatedVM implements VM {
- private isolate: ivm.Isolate
- private vm: ivm.Context
- private jail: ivm.Reference
- private invocationTimeout: number
- private isolateAccumulatedTimeout?: number
-
- // By default the wrapper returns itself
- private codeWrapper: (code: string) => string = code => code
-
- private readonly resultKey = "results"
- private runResultKey: string
-
- constructor({
- memoryLimit,
- invocationTimeout,
- isolateAccumulatedTimeout,
- }: {
- memoryLimit: number
- invocationTimeout: number
- isolateAccumulatedTimeout?: number
- }) {
- this.isolate = new ivm.Isolate({ memoryLimit })
- this.vm = this.isolate.createContextSync()
- this.jail = this.vm.global
- this.jail.setSync("global", this.jail.derefInto())
-
- this.runResultKey = crypto.randomUUID()
- this.addToContext({
- [this.resultKey]: { [this.runResultKey]: "" },
- })
-
- this.invocationTimeout = invocationTimeout
- this.isolateAccumulatedTimeout = isolateAccumulatedTimeout
- }
-
- withHelpers() {
- const urlModule = this.registerCallbacks({
- resolve: url.resolve,
- parse: url.parse,
- })
-
- const querystringModule = this.registerCallbacks({
- escape: querystring.escape,
- })
-
- const cryptoModule = this.registerCallbacks({
- randomUUID: crypto.randomUUID,
- })
-
- this.addToContext({
- helpersStripProtocol: new ivm.Callback((str: string) => {
- var parsed = url.parse(str) as any
- parsed.protocol = ""
- return parsed.format()
- }),
- })
-
- const injectedRequire = `require=function req(val) {
- switch (val) {
- case "url": return ${urlModule};
- case "querystring": return ${querystringModule};
- case "crypto": return ${cryptoModule};
- }
- }`
- const helpersSource = loadBundle(BundleType.HELPERS)
- const script = this.isolate.compileScriptSync(
- `${injectedRequire};${helpersSource};helpers=helpers.default`
- )
-
- script.runSync(this.vm, { timeout: this.invocationTimeout, release: false })
- new Promise(() => {
- script.release()
- })
-
- return this
- }
-
- withContext(context: Record) {
- this.addToContext(context)
-
- return this
- }
-
- withParsingBson(data: any) {
- this.addToContext({
- bsonData: bson.BSON.serialize({ data }),
- })
-
- // If we need to parse bson, we follow the next steps:
- // 1. Serialise the data from potential BSON to buffer before passing it to the isolate
- // 2. Deserialise the data within the isolate, to get the original data
- // 3. Process script
- // 4. Stringify the result in order to convert the result from BSON to json
- this.codeWrapper = code =>
- `(function(){
- const data = bson.deserialize(bsonData, { validation: { utf8: false } }).data;
- const result = ${code}
- return bson.toJson(result);
- })();`
-
- const bsonSource = loadBundle(BundleType.BSON)
-
- this.addToContext({
- textDecoderCb: new ivm.Callback(
- (args: {
- constructorArgs: any
- functionArgs: Parameters["decode"]>
- }) => {
- const result = new TextDecoder(...args.constructorArgs).decode(
- ...args.functionArgs
- )
- return result
- }
- ),
- })
-
- // "Polyfilling" text decoder. `bson.deserialize` requires decoding. We are creating a bridge function so we don't need to inject the full library
- const textDecoderPolyfill = class TextDecoderMock {
- constructorArgs
-
- constructor(...constructorArgs: any) {
- this.constructorArgs = constructorArgs
- }
-
- decode(...input: any) {
- // @ts-ignore
- return textDecoderCb({
- constructorArgs: this.constructorArgs,
- functionArgs: input,
- })
- }
- }
- .toString()
- .replace(/TextDecoderMock/, "TextDecoder")
-
- const script = this.isolate.compileScriptSync(
- `${textDecoderPolyfill};${bsonSource}`
- )
- script.runSync(this.vm, { timeout: this.invocationTimeout, release: false })
- new Promise(() => {
- script.release()
- })
-
- return this
- }
-
- execute(code: string): any {
- if (this.isolateAccumulatedTimeout) {
- const cpuMs = Number(this.isolate.cpuTime) / 1e6
- if (cpuMs > this.isolateAccumulatedTimeout) {
- throw new ExecutionTimeoutError(
- `CPU time limit exceeded (${cpuMs}ms > ${this.isolateAccumulatedTimeout}ms)`
- )
- }
- }
-
- code = `results['${this.runResultKey}']=${this.codeWrapper(code)}`
-
- const script = this.isolate.compileScriptSync(code)
-
- script.runSync(this.vm, { timeout: this.invocationTimeout, release: false })
- new Promise(() => {
- script.release()
- })
-
- // We can't rely on the script run result as it will not work for non-transferable values
- const result = this.getFromContext(this.resultKey)
- return result[this.runResultKey]
- }
-
- private registerCallbacks(functions: Record) {
- const libId = crypto.randomUUID().replace(/-/g, "")
-
- const x: Record = {}
- for (const [funcName, func] of Object.entries(functions)) {
- const key = `f${libId}${funcName}cb`
- x[funcName] = key
-
- this.addToContext({
- [key]: new ivm.Callback((...params: any[]) => (func as any)(...params)),
- })
- }
-
- const mod =
- `{` +
- Object.entries(x)
- .map(([key, func]) => `${key}: ${func}`)
- .join() +
- "}"
- return mod
- }
-
- private addToContext(context: Record) {
- for (let key in context) {
- const value = context[key]
- this.jail.setSync(
- key,
- typeof value === "function"
- ? value
- : new ivm.ExternalCopy(value).copyInto({ release: true })
- )
- }
- }
-
- private getFromContext(key: string) {
- const ref = this.vm.global.getSync(key, { reference: true })
- const result = ref.copySync()
-
- new Promise(() => {
- ref.release()
- })
- return result
- }
-}
+export * from "./isolated-vm"
+export * from "./builtin-vm"
+export * from "./vm2"
diff --git a/packages/server/src/jsRunner/vm/isolated-vm.ts b/packages/server/src/jsRunner/vm/isolated-vm.ts
new file mode 100644
index 0000000000..0285af8620
--- /dev/null
+++ b/packages/server/src/jsRunner/vm/isolated-vm.ts
@@ -0,0 +1,232 @@
+import ivm from "isolated-vm"
+import bson from "bson"
+
+import url from "url"
+import crypto from "crypto"
+import querystring from "querystring"
+
+import { BundleType, loadBundle } from "../bundles"
+import { VM } from "@budibase/types"
+
+class ExecutionTimeoutError extends Error {
+ constructor(message: string) {
+ super(message)
+ this.name = "ExecutionTimeoutError"
+ }
+}
+
+export class IsolatedVM implements VM {
+ private isolate: ivm.Isolate
+ private vm: ivm.Context
+ private jail: ivm.Reference
+ private invocationTimeout: number
+ private isolateAccumulatedTimeout?: number
+
+ // By default the wrapper returns itself
+ private codeWrapper: (code: string) => string = code => code
+
+ private readonly resultKey = "results"
+ private runResultKey: string
+
+ constructor({
+ memoryLimit,
+ invocationTimeout,
+ isolateAccumulatedTimeout,
+ }: {
+ memoryLimit: number
+ invocationTimeout: number
+ isolateAccumulatedTimeout?: number
+ }) {
+ this.isolate = new ivm.Isolate({ memoryLimit })
+ this.vm = this.isolate.createContextSync()
+ this.jail = this.vm.global
+ this.jail.setSync("global", this.jail.derefInto())
+
+ this.runResultKey = crypto.randomUUID()
+ this.addToContext({
+ [this.resultKey]: { [this.runResultKey]: "" },
+ })
+
+ this.invocationTimeout = invocationTimeout
+ this.isolateAccumulatedTimeout = isolateAccumulatedTimeout
+ }
+
+ withHelpers() {
+ const urlModule = this.registerCallbacks({
+ resolve: url.resolve,
+ parse: url.parse,
+ })
+
+ const querystringModule = this.registerCallbacks({
+ escape: querystring.escape,
+ })
+
+ const cryptoModule = this.registerCallbacks({
+ randomUUID: crypto.randomUUID,
+ })
+
+ this.addToContext({
+ helpersStripProtocol: new ivm.Callback((str: string) => {
+ var parsed = url.parse(str) as any
+ parsed.protocol = ""
+ return parsed.format()
+ }),
+ })
+
+ const injectedRequire = `require=function req(val) {
+ switch (val) {
+ case "url": return ${urlModule};
+ case "querystring": return ${querystringModule};
+ case "crypto": return ${cryptoModule};
+ }
+ }`
+ const helpersSource = loadBundle(BundleType.HELPERS)
+ const script = this.isolate.compileScriptSync(
+ `${injectedRequire};${helpersSource};helpers=helpers.default`
+ )
+
+ script.runSync(this.vm, { timeout: this.invocationTimeout, release: false })
+ new Promise(() => {
+ script.release()
+ })
+
+ return this
+ }
+
+ withContext(context: Record) {
+ this.addToContext(context)
+
+ return this
+ }
+
+ withParsingBson(data: any) {
+ this.addToContext({
+ bsonData: bson.BSON.serialize({ data }),
+ })
+
+ // If we need to parse bson, we follow the next steps:
+ // 1. Serialise the data from potential BSON to buffer before passing it to the isolate
+ // 2. Deserialise the data within the isolate, to get the original data
+ // 3. Process script
+ // 4. Stringify the result in order to convert the result from BSON to json
+ this.codeWrapper = code =>
+ `(function(){
+ const data = bson.deserialize(bsonData, { validation: { utf8: false } }).data;
+ const result = ${code}
+ return bson.toJson(result);
+ })();`
+
+ const bsonSource = loadBundle(BundleType.BSON)
+
+ this.addToContext({
+ textDecoderCb: new ivm.Callback(
+ (args: {
+ constructorArgs: any
+ functionArgs: Parameters["decode"]>
+ }) => {
+ const result = new TextDecoder(...args.constructorArgs).decode(
+ ...args.functionArgs
+ )
+ return result
+ }
+ ),
+ })
+
+ // "Polyfilling" text decoder. `bson.deserialize` requires decoding. We are creating a bridge function so we don't need to inject the full library
+ const textDecoderPolyfill = class TextDecoderMock {
+ constructorArgs
+
+ constructor(...constructorArgs: any) {
+ this.constructorArgs = constructorArgs
+ }
+
+ decode(...input: any) {
+ // @ts-ignore
+ return textDecoderCb({
+ constructorArgs: this.constructorArgs,
+ functionArgs: input,
+ })
+ }
+ }
+ .toString()
+ .replace(/TextDecoderMock/, "TextDecoder")
+
+ const script = this.isolate.compileScriptSync(
+ `${textDecoderPolyfill};${bsonSource}`
+ )
+ script.runSync(this.vm, { timeout: this.invocationTimeout, release: false })
+ new Promise(() => {
+ script.release()
+ })
+
+ return this
+ }
+
+ execute(code: string): any {
+ if (this.isolateAccumulatedTimeout) {
+ const cpuMs = Number(this.isolate.cpuTime) / 1e6
+ if (cpuMs > this.isolateAccumulatedTimeout) {
+ throw new ExecutionTimeoutError(
+ `CPU time limit exceeded (${cpuMs}ms > ${this.isolateAccumulatedTimeout}ms)`
+ )
+ }
+ }
+
+ code = `results['${this.runResultKey}']=${this.codeWrapper(code)}`
+
+ const script = this.isolate.compileScriptSync(code)
+
+ script.runSync(this.vm, { timeout: this.invocationTimeout, release: false })
+ new Promise(() => {
+ script.release()
+ })
+
+ // We can't rely on the script run result as it will not work for non-transferable values
+ const result = this.getFromContext(this.resultKey)
+ return result[this.runResultKey]
+ }
+
+ private registerCallbacks(functions: Record) {
+ const libId = crypto.randomUUID().replace(/-/g, "")
+
+ const x: Record = {}
+ for (const [funcName, func] of Object.entries(functions)) {
+ const key = `f${libId}${funcName}cb`
+ x[funcName] = key
+
+ this.addToContext({
+ [key]: new ivm.Callback((...params: any[]) => (func as any)(...params)),
+ })
+ }
+
+ const mod =
+ `{` +
+ Object.entries(x)
+ .map(([key, func]) => `${key}: ${func}`)
+ .join() +
+ "}"
+ return mod
+ }
+
+ private addToContext(context: Record) {
+ for (let key in context) {
+ const value = context[key]
+ this.jail.setSync(
+ key,
+ typeof value === "function"
+ ? value
+ : new ivm.ExternalCopy(value).copyInto({ release: true })
+ )
+ }
+ }
+
+ private getFromContext(key: string) {
+ const ref = this.vm.global.getSync(key, { reference: true })
+ const result = ref.copySync()
+
+ new Promise(() => {
+ ref.release()
+ })
+ return result
+ }
+}
diff --git a/packages/server/src/jsRunner/vm/vm2.ts b/packages/server/src/jsRunner/vm/vm2.ts
new file mode 100644
index 0000000000..6d05943d25
--- /dev/null
+++ b/packages/server/src/jsRunner/vm/vm2.ts
@@ -0,0 +1,26 @@
+import vm2 from "vm2"
+import { VM } from "@budibase/types"
+
+const JS_TIMEOUT_MS = 1000
+
+export class VM2 implements VM {
+ vm: vm2.VM
+ results: { out: string }
+
+ constructor(context: any) {
+ this.vm = new vm2.VM({
+ timeout: JS_TIMEOUT_MS,
+ })
+ this.results = { out: "" }
+ this.vm.setGlobals(context)
+ this.vm.setGlobal("fetch", fetch)
+ this.vm.setGlobal("results", this.results)
+ }
+
+ execute(script: string) {
+ const code = `let fn = () => {\n${script}\n}; results.out = fn();`
+ const vmScript = new vm2.VMScript(code)
+ this.vm.run(vmScript)
+ return this.results.out
+ }
+}
diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts
index b38dd7de6b..a8aa428b0a 100644
--- a/packages/server/src/threads/query.ts
+++ b/packages/server/src/threads/query.ts
@@ -7,17 +7,18 @@ import {
QueryVariable,
QueryResponse,
} from "./definitions"
-import ScriptRunner from "../utilities/scriptRunner"
+import { IsolatedVM, VM2 } from "../jsRunner/vm"
import { getIntegration } from "../integrations"
import { processStringSync } from "@budibase/string-templates"
import { context, cache, auth } from "@budibase/backend-core"
import { getGlobalIDFromUserMetadataID } from "../db/utils"
import sdk from "../sdk"
import { cloneDeep } from "lodash/fp"
-import { Datasource, Query, SourceName } from "@budibase/types"
+import { Datasource, Query, SourceName, VM } from "@budibase/types"
import { isSQL } from "../integrations/utils"
import { interpolateSQL } from "../integrations/queries/sql"
+import environment from "../environment"
class QueryRunner {
datasource: Datasource
@@ -26,7 +27,7 @@ class QueryRunner {
fields: any
parameters: any
pagination: any
- transformer: any
+ transformer: string
cachedVariables: any[]
ctx: any
queryResponse: any
@@ -127,17 +128,25 @@ class QueryRunner {
// transform as required
if (transformer) {
- const runner = new ScriptRunner(
- transformer,
- {
+ let runner: VM
+ if (!environment.useIsolatedVM.QUERY_TRANSFORMERS) {
+ runner = new VM2({
data: rows,
params: enrichedParameters,
- },
- {
- parseBson: datasource.source === SourceName.MONGODB,
+ })
+ } else {
+ let isolatedVm = new IsolatedVM().withContext({
+ data: rows,
+ params: enrichedParameters,
+ })
+ if (datasource.source === SourceName.MONGODB) {
+ isolatedVm = isolatedVm.withParsingBson(rows)
}
- )
- rows = runner.execute()
+
+ runner = isolatedVm
+ }
+
+ rows = runner.execute(transformer)
}
// if the request fails we retry once, invalidating the cached value
diff --git a/packages/server/src/utilities/scriptRunner.ts b/packages/server/src/utilities/scriptRunner.ts
deleted file mode 100644
index 72042a5791..0000000000
--- a/packages/server/src/utilities/scriptRunner.ts
+++ /dev/null
@@ -1,41 +0,0 @@
-import tracer, { Span } from "dd-trace"
-import env from "../environment"
-import { IsolatedVM } from "../jsRunner/vm"
-
-const JS_TIMEOUT_MS = 1000
-
-class ScriptRunner {
- private code: string
- private vm: IsolatedVM
-
- private tracerSpan: Span
-
- constructor(script: string, context: any, { parseBson = false } = {}) {
- this.tracerSpan = tracer.startSpan("scriptRunner", { tags: { parseBson } })
-
- this.code = `(() => {${script}})();`
- this.vm = new IsolatedVM({
- memoryLimit: env.JS_RUNNER_MEMORY_LIMIT,
- invocationTimeout: JS_TIMEOUT_MS,
- }).withContext(context)
-
- if (parseBson && context.data) {
- this.vm = this.vm.withParsingBson(context.data)
- }
- }
-
- execute() {
- const result = tracer.trace(
- "scriptRunner.execute",
- { childOf: this.tracerSpan },
- () => {
- const result = this.vm.execute(this.code)
- return result
- }
- )
- this.tracerSpan.finish()
- return result
- }
-}
-
-export default ScriptRunner