Merge branch 'wrap-existing-runners' into reenable-isolatedvm

This commit is contained in:
Adria Navarro 2024-02-19 16:52:59 +01:00
commit ce76329be8
9 changed files with 413 additions and 341 deletions

View File

@ -1,15 +1,25 @@
<script> <script>
import EditComponentPopover from "../EditComponentPopover.svelte" import EditComponentPopover from "../EditComponentPopover.svelte"
import { FieldTypeToComponentMap } from "../FieldConfiguration/utils"
import { Toggle, Icon } from "@budibase/bbui" import { Toggle, Icon } from "@budibase/bbui"
import { createEventDispatcher } from "svelte" import { createEventDispatcher } from "svelte"
import { cloneDeep } from "lodash/fp" import { cloneDeep } from "lodash/fp"
import { componentStore } from "stores/builder" import { FIELDS } from "constants/backend"
export let item export let item
export let anchor export let anchor
const dispatch = createEventDispatcher() const dispatch = createEventDispatcher()
$: fieldIconLookupMap = buildFieldIconLookupMap(FIELDS)
const buildFieldIconLookupMap = fields => {
let map = {}
Object.values(fields).forEach(fieldInfo => {
map[fieldInfo.type] = fieldInfo.icon
})
return map
}
const onToggle = item => { const onToggle = item => {
return e => { return e => {
item.active = e.detail item.active = e.detail
@ -24,13 +34,6 @@
return { ...setting, nested: true } return { ...setting, nested: true }
}) })
} }
const getIcon = () => {
const component = `@budibase/standard-components/${
FieldTypeToComponentMap[item.columnType]
}`
return componentStore.getDefinition(component).icon
}
</script> </script>
<div class="list-item-body"> <div class="list-item-body">
@ -42,7 +45,7 @@
on:change on:change
> >
<div slot="header" class="type-icon"> <div slot="header" class="type-icon">
<Icon name={getIcon()} /> <Icon name={fieldIconLookupMap[item.columnType]} />
<span>{item.field}</span> <span>{item.field}</span>
</div> </div>
</EditComponentPopover> </EditComponentPopover>

View File

@ -1,10 +1,11 @@
import ScriptRunner from "../../utilities/scriptRunner"
import { Ctx } from "@budibase/types" import { Ctx } from "@budibase/types"
import { VM2 } from "../../jsRunner/vm"
export async function execute(ctx: Ctx) { export async function execute(ctx: Ctx) {
const { script, context } = ctx.request.body const { script, context } = ctx.request.body
const runner = new ScriptRunner(script, context) const runner = new VM2(context)
ctx.body = runner.execute() const result = runner.execute(script)
ctx.body = result
} }
export async function save(ctx: Ctx) { export async function save(ctx: Ctx) {

View File

@ -1,13 +1,17 @@
import env from "../environment" import env from "../environment"
import { setJSRunner, JsErrorTimeout } from "@budibase/string-templates" import { JsErrorTimeout, setJSRunner } from "@budibase/string-templates"
import tracer from "dd-trace"
import { IsolatedVM } from "./vm"
import { context } from "@budibase/backend-core" import { context } from "@budibase/backend-core"
import tracer from "dd-trace"
import { BuiltInVM, IsolatedVM } from "./vm"
export function init() { export function init() {
setJSRunner((js: string, ctx: Record<string, any>) => { setJSRunner((js: string, ctx: Record<string, any>) => {
return tracer.trace("runJS", {}, span => { return tracer.trace("runJS", {}, span => {
if (!env.useIsolatedVM.JS_RUNNER) {
const vm = new BuiltInVM(ctx, span)
return vm.execute(js)
}
try { try {
const bbCtx = context.getCurrentContext()! const bbCtx = context.getCurrentContext()!
@ -26,9 +30,7 @@ export function init() {
bbCtx.vm = vm bbCtx.vm = vm
} }
const result = vm.execute(js) const result = vm.execute(js)
return result return result
} catch (error: any) { } catch (error: any) {
if (error.message === "Script execution timed out.") { if (error.message === "Script execution timed out.") {

View File

@ -0,0 +1,65 @@
import vm from "vm"
import env from "../../environment"
import { context, timers } from "@budibase/backend-core"
import tracer, { Span } from "dd-trace"
import { VM } from "@budibase/types"
type TrackerFn = <T>(f: () => T) => T
export class BuiltInVM implements VM {
private ctx: vm.Context
private span?: Span
constructor(ctx: vm.Context, span?: Span) {
this.ctx = ctx
this.span = span
}
execute(code: string) {
const perRequestLimit = env.JS_PER_REQUEST_TIMEOUT_MS
let track: TrackerFn = f => f()
if (perRequestLimit) {
const bbCtx = tracer.trace("runJS.getCurrentContext", {}, span =>
context.getCurrentContext()
)
if (bbCtx) {
if (!bbCtx.jsExecutionTracker) {
this.span?.addTags({
createdExecutionTracker: true,
})
bbCtx.jsExecutionTracker = tracer.trace(
"runJS.createExecutionTimeTracker",
{},
span => timers.ExecutionTimeTracker.withLimit(perRequestLimit)
)
}
this.span?.addTags({
js: {
limitMS: bbCtx.jsExecutionTracker.limitMs,
elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
},
})
// We call checkLimit() here to prevent paying the cost of creating
// a new VM context below when we don't need to.
tracer.trace("runJS.checkLimitAndBind", {}, span => {
bbCtx.jsExecutionTracker!.checkLimit()
track = bbCtx.jsExecutionTracker!.track.bind(bbCtx.jsExecutionTracker)
})
}
}
this.ctx = {
...this.ctx,
alert: undefined,
setInterval: undefined,
setTimeout: undefined,
}
vm.createContext(this.ctx)
return track(() =>
vm.runInNewContext(code, this.ctx, {
timeout: env.JS_PER_INVOCATION_TIMEOUT_MS,
})
)
}
}

View File

@ -1,270 +1,3 @@
import ivm from "isolated-vm" export * from "./isolated-vm"
import bson from "bson" export * from "./builtin-vm"
export * from "./vm2"
import url from "url"
import crypto from "crypto"
import querystring from "querystring"
import { BundleType, loadBundle } from "../bundles"
import { VM } from "@budibase/types"
class ExecutionTimeoutError extends Error {
constructor(message: string) {
super(message)
this.name = "ExecutionTimeoutError"
}
}
class ModuleHandler {
private modules: {
import: string
moduleKey: string
module: ivm.Module
}[] = []
private generateRandomKey = () => `i${crypto.randomUUID().replace(/-/g, "")}`
registerModule(module: ivm.Module, imports: string) {
this.modules.push({
moduleKey: this.generateRandomKey(),
import: imports,
module: module,
})
}
generateImports() {
return this.modules
.map(m => `import ${m.import} from "${m.moduleKey}"`)
.join(";")
}
getModule(key: string) {
const module = this.modules.find(m => m.moduleKey === key)
return module?.module
}
}
export class IsolatedVM implements VM {
private isolate: ivm.Isolate
private vm: ivm.Context
private jail: ivm.Reference
private invocationTimeout: number
private isolateAccumulatedTimeout?: number
// By default the wrapper returns itself
private codeWrapper: (code: string) => string = code => code
private moduleHandler = new ModuleHandler()
private readonly resultKey = "results"
constructor({
memoryLimit,
invocationTimeout,
isolateAccumulatedTimeout,
}: {
memoryLimit: number
invocationTimeout: number
isolateAccumulatedTimeout?: number
}) {
this.isolate = new ivm.Isolate({ memoryLimit })
this.vm = this.isolate.createContextSync()
this.jail = this.vm.global
this.jail.setSync("global", this.jail.derefInto())
this.addToContext({
[this.resultKey]: { out: "" },
})
this.invocationTimeout = invocationTimeout
this.isolateAccumulatedTimeout = isolateAccumulatedTimeout
}
withHelpers() {
const urlModule = this.registerCallbacks({
resolve: url.resolve,
parse: url.parse,
})
const querystringModule = this.registerCallbacks({
escape: querystring.escape,
})
this.addToContext({
helpersStripProtocol: new ivm.Callback((str: string) => {
var parsed = url.parse(str) as any
parsed.protocol = ""
return parsed.format()
}),
})
const injectedRequire = `const require=function req(val) {
switch (val) {
case "url": return ${urlModule};
case "querystring": return ${querystringModule};
}
}`
const helpersSource = loadBundle(BundleType.HELPERS)
const helpersModule = this.isolate.compileModuleSync(
`${injectedRequire};${helpersSource}`
)
helpersModule.instantiateSync(this.vm, specifier => {
if (specifier === "crypto") {
const cryptoModule = this.registerCallbacks({
randomUUID: crypto.randomUUID,
})
const module = this.isolate.compileModuleSync(
`export default ${cryptoModule}`
)
module.instantiateSync(this.vm, specifier => {
throw new Error(`No imports allowed. Required: ${specifier}`)
})
return module
}
throw new Error(`No imports allowed. Required: ${specifier}`)
})
this.moduleHandler.registerModule(helpersModule, "helpers")
return this
}
withContext(context: Record<string, any>) {
this.addToContext(context)
return this
}
withParsingBson(data: any) {
this.addToContext({
bsonData: bson.BSON.serialize({ data }),
})
// If we need to parse bson, we follow the next steps:
// 1. Serialise the data from potential BSON to buffer before passing it to the isolate
// 2. Deserialise the data within the isolate, to get the original data
// 3. Process script
// 4. Stringify the result in order to convert the result from BSON to json
this.codeWrapper = code =>
`(function(){
const data = deserialize(bsonData, { validation: { utf8: false } }).data;
const result = ${code}
return toJson(result);
})();`
const bsonSource = loadBundle(BundleType.BSON)
this.addToContext({
textDecoderCb: new ivm.Callback(
(args: {
constructorArgs: any
functionArgs: Parameters<InstanceType<typeof TextDecoder>["decode"]>
}) => {
const result = new TextDecoder(...args.constructorArgs).decode(
...args.functionArgs
)
return result
}
),
})
// "Polyfilling" text decoder. `bson.deserialize` requires decoding. We are creating a bridge function so we don't need to inject the full library
const textDecoderPolyfill = class TextDecoder {
constructorArgs
constructor(...constructorArgs: any) {
this.constructorArgs = constructorArgs
}
decode(...input: any) {
// @ts-ignore
return textDecoderCb({
constructorArgs: this.constructorArgs,
functionArgs: input,
})
}
}.toString()
const bsonModule = this.isolate.compileModuleSync(
`${textDecoderPolyfill};${bsonSource}`
)
bsonModule.instantiateSync(this.vm, specifier => {
throw new Error(`No imports allowed. Required: ${specifier}`)
})
this.moduleHandler.registerModule(bsonModule, "{deserialize, toJson}")
return this
}
execute(code: string): any {
if (this.isolateAccumulatedTimeout) {
const cpuMs = Number(this.isolate.cpuTime) / 1e6
if (cpuMs > this.isolateAccumulatedTimeout) {
throw new ExecutionTimeoutError(
`CPU time limit exceeded (${cpuMs}ms > ${this.isolateAccumulatedTimeout}ms)`
)
}
}
code = `${this.moduleHandler.generateImports()};results.out=${this.codeWrapper(
code
)};`
const script = this.isolate.compileModuleSync(code)
script.instantiateSync(this.vm, specifier => {
const module = this.moduleHandler.getModule(specifier)
if (module) {
return module
}
throw new Error(`"${specifier}" import not allowed`)
})
script.evaluateSync({ timeout: this.invocationTimeout })
const result = this.getFromContext(this.resultKey)
return result.out
}
private registerCallbacks(functions: Record<string, any>) {
const libId = crypto.randomUUID().replace(/-/g, "")
const x: Record<string, string> = {}
for (const [funcName, func] of Object.entries(functions)) {
const key = `f${libId}${funcName}cb`
x[funcName] = key
this.addToContext({
[key]: new ivm.Callback((...params: any[]) => (func as any)(...params)),
})
}
const mod =
`{` +
Object.entries(x)
.map(([key, func]) => `${key}: ${func}`)
.join() +
"}"
return mod
}
private addToContext(context: Record<string, any>) {
for (let key in context) {
const value = context[key]
this.jail.setSync(
key,
typeof value === "function"
? value
: new ivm.ExternalCopy(value).copyInto({ release: true })
)
}
}
private getFromContext(key: string) {
const ref = this.vm.global.getSync(key, { reference: true })
const result = ref.copySync()
ref.release()
return result
}
}

View File

@ -0,0 +1,274 @@
import ivm from "isolated-vm"
import bson from "bson"
import url from "url"
import crypto from "crypto"
import querystring from "querystring"
import { BundleType, loadBundle } from "../bundles"
import { VM } from "@budibase/types"
import environment from "../../environment"
class ExecutionTimeoutError extends Error {
constructor(message: string) {
super(message)
this.name = "ExecutionTimeoutError"
}
}
class ModuleHandler {
private modules: {
import: string
moduleKey: string
module: ivm.Module
}[] = []
private generateRandomKey = () => `i${crypto.randomUUID().replace(/-/g, "")}`
registerModule(module: ivm.Module, imports: string) {
this.modules.push({
moduleKey: this.generateRandomKey(),
import: imports,
module: module,
})
}
generateImports() {
return this.modules
.map(m => `import ${m.import} from "${m.moduleKey}"`)
.join(";")
}
getModule(key: string) {
const module = this.modules.find(m => m.moduleKey === key)
return module?.module
}
}
export class IsolatedVM implements VM {
private isolate: ivm.Isolate
private vm: ivm.Context
private jail: ivm.Reference
private invocationTimeout: number
private isolateAccumulatedTimeout?: number
// By default the wrapper returns itself
private codeWrapper: (code: string) => string = code => code
private moduleHandler = new ModuleHandler()
private readonly resultKey = "results"
constructor({
memoryLimit,
invocationTimeout,
isolateAccumulatedTimeout,
}: {
memoryLimit?: number
invocationTimeout?: number
isolateAccumulatedTimeout?: number
} = {}) {
memoryLimit = memoryLimit || environment.JS_RUNNER_MEMORY_LIMIT
invocationTimeout = memoryLimit || 1000
this.isolate = new ivm.Isolate({ memoryLimit })
this.vm = this.isolate.createContextSync()
this.jail = this.vm.global
this.jail.setSync("global", this.jail.derefInto())
this.addToContext({
[this.resultKey]: { out: "" },
})
this.invocationTimeout = invocationTimeout
this.isolateAccumulatedTimeout = isolateAccumulatedTimeout
}
withHelpers() {
const urlModule = this.registerCallbacks({
resolve: url.resolve,
parse: url.parse,
})
const querystringModule = this.registerCallbacks({
escape: querystring.escape,
})
this.addToContext({
helpersStripProtocol: new ivm.Callback((str: string) => {
var parsed = url.parse(str) as any
parsed.protocol = ""
return parsed.format()
}),
})
const injectedRequire = `const require=function req(val) {
switch (val) {
case "url": return ${urlModule};
case "querystring": return ${querystringModule};
}
}`
const helpersSource = loadBundle(BundleType.HELPERS)
const helpersModule = this.isolate.compileModuleSync(
`${injectedRequire};${helpersSource}`
)
helpersModule.instantiateSync(this.vm, specifier => {
if (specifier === "crypto") {
const cryptoModule = this.registerCallbacks({
randomUUID: crypto.randomUUID,
})
const module = this.isolate.compileModuleSync(
`export default ${cryptoModule}`
)
module.instantiateSync(this.vm, specifier => {
throw new Error(`No imports allowed. Required: ${specifier}`)
})
return module
}
throw new Error(`No imports allowed. Required: ${specifier}`)
})
this.moduleHandler.registerModule(helpersModule, "helpers")
return this
}
withContext(context: Record<string, any>) {
this.addToContext(context)
return this
}
withParsingBson(data: any) {
this.addToContext({
bsonData: bson.BSON.serialize({ data }),
})
// If we need to parse bson, we follow the next steps:
// 1. Serialise the data from potential BSON to buffer before passing it to the isolate
// 2. Deserialise the data within the isolate, to get the original data
// 3. Process script
// 4. Stringify the result in order to convert the result from BSON to json
this.codeWrapper = code =>
`(function(){
const data = deserialize(bsonData, { validation: { utf8: false } }).data;
const result = ${code}
return toJson(result);
})();`
const bsonSource = loadBundle(BundleType.BSON)
this.addToContext({
textDecoderCb: new ivm.Callback(
(args: {
constructorArgs: any
functionArgs: Parameters<InstanceType<typeof TextDecoder>["decode"]>
}) => {
const result = new TextDecoder(...args.constructorArgs).decode(
...args.functionArgs
)
return result
}
),
})
// "Polyfilling" text decoder. `bson.deserialize` requires decoding. We are creating a bridge function so we don't need to inject the full library
const textDecoderPolyfill = class TextDecoder {
constructorArgs
constructor(...constructorArgs: any) {
this.constructorArgs = constructorArgs
}
decode(...input: any) {
// @ts-ignore
return textDecoderCb({
constructorArgs: this.constructorArgs,
functionArgs: input,
})
}
}.toString()
const bsonModule = this.isolate.compileModuleSync(
`${textDecoderPolyfill};${bsonSource}`
)
bsonModule.instantiateSync(this.vm, specifier => {
throw new Error(`No imports allowed. Required: ${specifier}`)
})
this.moduleHandler.registerModule(bsonModule, "{deserialize, toJson}")
return this
}
execute(code: string): any {
if (this.isolateAccumulatedTimeout) {
const cpuMs = Number(this.isolate.cpuTime) / 1e6
if (cpuMs > this.isolateAccumulatedTimeout) {
throw new ExecutionTimeoutError(
`CPU time limit exceeded (${cpuMs}ms > ${this.isolateAccumulatedTimeout}ms)`
)
}
}
code = `${this.moduleHandler.generateImports()};results.out=${this.codeWrapper(
code
)};`
const script = this.isolate.compileModuleSync(code)
script.instantiateSync(this.vm, specifier => {
const module = this.moduleHandler.getModule(specifier)
if (module) {
return module
}
throw new Error(`"${specifier}" import not allowed`)
})
script.evaluateSync({ timeout: this.invocationTimeout })
const result = this.getFromContext(this.resultKey)
return result.out
}
private registerCallbacks(functions: Record<string, any>) {
const libId = crypto.randomUUID().replace(/-/g, "")
const x: Record<string, string> = {}
for (const [funcName, func] of Object.entries(functions)) {
const key = `f${libId}${funcName}cb`
x[funcName] = key
this.addToContext({
[key]: new ivm.Callback((...params: any[]) => (func as any)(...params)),
})
}
const mod =
`{` +
Object.entries(x)
.map(([key, func]) => `${key}: ${func}`)
.join() +
"}"
return mod
}
private addToContext(context: Record<string, any>) {
for (let key in context) {
const value = context[key]
this.jail.setSync(
key,
typeof value === "function"
? value
: new ivm.ExternalCopy(value).copyInto({ release: true })
)
}
}
private getFromContext(key: string) {
const ref = this.vm.global.getSync(key, { reference: true })
const result = ref.copySync()
ref.release()
return result
}
}

View File

@ -0,0 +1,26 @@
import vm2 from "vm2"
import { VM } from "@budibase/types"
const JS_TIMEOUT_MS = 1000
export class VM2 implements VM {
vm: vm2.VM
results: { out: string }
constructor(context: any) {
this.vm = new vm2.VM({
timeout: JS_TIMEOUT_MS,
})
this.results = { out: "" }
this.vm.setGlobals(context)
this.vm.setGlobal("fetch", fetch)
this.vm.setGlobal("results", this.results)
}
execute(script: string) {
const code = `let fn = () => {\n${script}\n}; results.out = fn();`
const vmScript = new vm2.VMScript(code)
this.vm.run(vmScript)
return this.results.out
}
}

View File

@ -7,17 +7,18 @@ import {
QueryVariable, QueryVariable,
QueryResponse, QueryResponse,
} from "./definitions" } from "./definitions"
import ScriptRunner from "../utilities/scriptRunner" import { IsolatedVM, VM2 } from "../jsRunner/vm"
import { getIntegration } from "../integrations" import { getIntegration } from "../integrations"
import { processStringSync } from "@budibase/string-templates" import { processStringSync } from "@budibase/string-templates"
import { context, cache, auth } from "@budibase/backend-core" import { context, cache, auth } from "@budibase/backend-core"
import { getGlobalIDFromUserMetadataID } from "../db/utils" import { getGlobalIDFromUserMetadataID } from "../db/utils"
import sdk from "../sdk" import sdk from "../sdk"
import { cloneDeep } from "lodash/fp" import { cloneDeep } from "lodash/fp"
import { Datasource, Query, SourceName } from "@budibase/types" import { Datasource, Query, SourceName, VM } from "@budibase/types"
import { isSQL } from "../integrations/utils" import { isSQL } from "../integrations/utils"
import { interpolateSQL } from "../integrations/queries/sql" import { interpolateSQL } from "../integrations/queries/sql"
import environment from "src/environment"
class QueryRunner { class QueryRunner {
datasource: Datasource datasource: Datasource
@ -26,7 +27,7 @@ class QueryRunner {
fields: any fields: any
parameters: any parameters: any
pagination: any pagination: any
transformer: any transformer: string
cachedVariables: any[] cachedVariables: any[]
ctx: any ctx: any
queryResponse: any queryResponse: any
@ -127,17 +128,25 @@ class QueryRunner {
// transform as required // transform as required
if (transformer) { if (transformer) {
const runner = new ScriptRunner( let runner: VM
transformer, if (!environment.useIsolatedVM.QUERY_TRANSFORMERS) {
{ runner = new VM2({
data: rows, data: rows,
params: enrichedParameters, params: enrichedParameters,
}, })
{ } else {
parseBson: datasource.source === SourceName.MONGODB, let isolatedVm = new IsolatedVM().withContext({
data: rows,
params: enrichedParameters,
})
if (datasource.source === SourceName.MONGODB) {
isolatedVm = isolatedVm.withParsingBson(rows)
} }
)
rows = runner.execute() runner = isolatedVm
}
rows = runner.execute(transformer)
} }
// if the request fails we retry once, invalidating the cached value // if the request fails we retry once, invalidating the cached value

View File

@ -1,41 +0,0 @@
import tracer, { Span } from "dd-trace"
import env from "../environment"
import { IsolatedVM } from "../jsRunner/vm"
const JS_TIMEOUT_MS = 1000
class ScriptRunner {
private code: string
private vm: IsolatedVM
private tracerSpan: Span
constructor(script: string, context: any, { parseBson = false } = {}) {
this.tracerSpan = tracer.startSpan("scriptRunner", { tags: { parseBson } })
this.code = `(() => {${script}})();`
this.vm = new IsolatedVM({
memoryLimit: env.JS_RUNNER_MEMORY_LIMIT,
invocationTimeout: JS_TIMEOUT_MS,
}).withContext(context)
if (parseBson && context.data) {
this.vm = this.vm.withParsingBson(context.data)
}
}
execute() {
const result = tracer.trace(
"scriptRunner.execute",
{ childOf: this.tracerSpan },
() => {
const result = this.vm.execute(this.code)
return result
}
)
this.tracerSpan.finish()
return result
}
}
export default ScriptRunner