MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge isolated-vm-wrapper

This commit is contained in:
Adria Navarro 2024-02-08 10:45:00 +01:00
parent 0ea7a515de
commit 7972f19cd1
10 changed files with 385 additions and 237 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
packages/backend-core/src/context/types.ts
View File

@ -1,5 +1,4 @@
import { IdentityContext } from "@budibase/types"
import { Isolate, Context, Module } from "isolated-vm"
import { IdentityContext, VM } from "@budibase/types"
// keep this out of Budibase types, don't want to expose context info
export type ContextMap = {
@ -10,9 +9,5 @@ export type ContextMap = {
isScim?: boolean
automationId?: string
isMigrating?: boolean
isolateRefs?: {
jsIsolate: Isolate
jsContext: Context
helpersModule: Module
}
vm?: VM
}

3
packages/backend-core/src/objectStore/objectStore.ts
View File

@ -255,7 +255,8 @@ export async function listAllObjects(bucketName: string, path: string) {
objects = objects.concat(response.Contents)
}
isTruncated = !!response.IsTruncated
} while (isTruncated)
token = response.NextContinuationToken
} while (isTruncated && token)
return objects
}

13
packages/backend-core/src/queue/queue.ts
View File

@ -2,7 +2,7 @@ import env from "../environment"
import { getRedisOptions } from "../redis/utils"
import { JobQueue } from "./constants"
import InMemoryQueue from "./inMemoryQueue"
import BullQueue, { QueueOptions } from "bull"
import BullQueue, { QueueOptions, JobOptions } from "bull"
import { addListeners, StalledFn } from "./listeners"
import { Duration } from "../utils"
import * as timers from "../timers"
@ -24,17 +24,24 @@ async function cleanup() {
export function createQueue<T>(
jobQueue: JobQueue,
opts: { removeStalledCb?: StalledFn } = {}
opts: {
removeStalledCb?: StalledFn
maxStalledCount?: number
jobOptions?: JobOptions
} = {}
): BullQueue.Queue<T> {
const redisOpts = getRedisOptions()
const queueConfig: QueueOptions = {
redis: redisOpts,
settings: {
maxStalledCount: 0,
maxStalledCount: opts.maxStalledCount ? opts.maxStalledCount : 0,
lockDuration: QUEUE_LOCK_MS,
lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS,
},
}
if (opts.jobOptions) {
queueConfig.defaultJobOptions = opts.jobOptions
}
let queue: any
if (!env.isTest()) {
queue = new BullQueue(jobQueue, queueConfig)

8
packages/builder/src/components/common/CodeEditor/index.js
View File

@ -286,7 +286,13 @@ export const hbInsert = (value, from, to, text) => {
return parsedInsert
}
export function jsInsert(value, from, to, text, { helper, disableWrapping }) {
export function jsInsert(
value,
from,
to,
text,
{ helper, disableWrapping } = {}
) {
let parsedInsert = ""
const left = from ? value.substring(0, from) : ""

2
packages/pro

@ -1 +1 @@
Subproject commit aaf7101cd1493215155cc8f83124c70d53eb1be4
Subproject commit 9b14e5d5182bf5e5ee98f717997e7352e5904799

151
packages/server/src/jsRunner/index.ts
View File

@ -1,155 +1,28 @@
import ivm from "isolated-vm"
import env from "../environment"
import { setJSRunner, JsErrorTimeout } from "@budibase/string-templates"
import { context } from "@budibase/backend-core"
import tracer from "dd-trace"
import url from "url"
import crypto from "crypto"
import querystring from "querystring"
import { BundleType, loadBundle } from "./bundles"
class ExecutionTimeoutError extends Error {
constructor(message: string) {
super(message)
this.name = "ExecutionTimeoutError"
}
}
import { IsolatedVM } from "./vm"
import { context } from "@budibase/backend-core"
export function init() {
const helpersSource = loadBundle(BundleType.HELPERS)
setJSRunner((js: string, ctx: Record<string, any>) => {
return tracer.trace("runJS", {}, span => {
try {
const bbCtx = context.getCurrentContext()!
const isolateRefs = bbCtx.isolateRefs
if (!isolateRefs) {
const jsIsolate = new ivm.Isolate({
let { vm } = bbCtx
if (!vm) {
vm = new IsolatedVM({
memoryLimit: env.JS_RUNNER_MEMORY_LIMIT,
})
const jsContext = jsIsolate.createContextSync()
const injectedRequire = `
const require = function(val){
switch (val) {
case "url":
return {
resolve: (...params) => urlResolveCb(...params),
parse: (...params) => urlParseCb(...params),
}
case "querystring":
return {
escape: (...params) => querystringEscapeCb(...params),
}
}
};`
const global = jsContext.global
global.setSync(
"urlResolveCb",
new ivm.Callback((...params: Parameters<typeof url.resolve>) =>
url.resolve(...params)
)
)
global.setSync(
"urlParseCb",
new ivm.Callback((...params: Parameters<typeof url.parse>) =>
url.parse(...params)
)
)
global.setSync(
"querystringEscapeCb",
new ivm.Callback(
(...params: Parameters<typeof querystring.escape>) =>
querystring.escape(...params)
)
)
global.setSync(
"helpersStripProtocol",
new ivm.Callback((str: string) => {
var parsed = url.parse(str) as any
parsed.protocol = ""
return parsed.format()
})
)
const helpersModule = jsIsolate.compileModuleSync(
`${injectedRequire};${helpersSource}`
)
const cryptoModule = jsIsolate.compileModuleSync(
`export default { randomUUID: cryptoRandomUUIDCb }`
)
cryptoModule.instantiateSync(jsContext, specifier => {
throw new Error(`No imports allowed. Required: ${specifier}`)
})
global.setSync(
"cryptoRandomUUIDCb",
new ivm.Callback(
(...params: Parameters<typeof crypto.randomUUID>) => {
return crypto.randomUUID(...params)
}
)
)
helpersModule.instantiateSync(jsContext, specifier => {
if (specifier === "crypto") {
return cryptoModule
}
throw new Error(`No imports allowed. Required: ${specifier}`)
})
for (const [key, value] of Object.entries(ctx)) {
if (key === "helpers") {
// Can't copy the native helpers into the isolate. We just ignore them as they are handled properly from the helpersSource
continue
}
global.setSync(key, value)
}
bbCtx.isolateRefs = { jsContext, jsIsolate, helpersModule }
}
let { jsIsolate, jsContext, helpersModule } = bbCtx.isolateRefs!
const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
if (perRequestLimit) {
const cpuMs = Number(jsIsolate.cpuTime) / 1e6
if (cpuMs > perRequestLimit) {
throw new ExecutionTimeoutError(
`CPU time limit exceeded (${cpuMs}ms > ${perRequestLimit}ms)`
)
}
}
const script = jsIsolate.compileModuleSync(
`import helpers from "compiled_module";const result=${js};cb(result)`,
{}
)
script.instantiateSync(jsContext, specifier => {
if (specifier === "compiled_module") {
return helpersModule
}
throw new Error(`"${specifier}" import not allowed`)
})
let result
jsContext.global.setSync(
"cb",
new ivm.Callback((value: any) => {
result = value
})
)
script.evaluateSync({
timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
})
perRequestLimit: env.JS_PER_REQUEST_TIME_LIMIT_MS,
}).withHelpers()
bbCtx.vm = vm
}
const result = vm.execute(js)
return result
} catch (error: any) {

240
packages/server/src/jsRunner/vm/index.ts Normal file
View File

@ -0,0 +1,240 @@
import ivm from "isolated-vm"
import bson from "bson"
import url from "url"
import crypto from "crypto"
import querystring from "querystring"
import { BundleType, loadBundle } from "../bundles"
import { VM } from "@budibase/types"
import { context } from "@budibase/backend-core"
class ExecutionTimeoutError extends Error {
constructor(message: string) {
super(message)
this.name = "ExecutionTimeoutError"
}
}
export class IsolatedVM implements VM {
#isolate: ivm.Isolate
#vm: ivm.Context
#jail: ivm.Reference
#timeout: number
#perRequestLimit?: number
#parseBson?: boolean
#modules: {
import: string
moduleKey: string
module: ivm.Module
}[] = []
readonly #resultKey = "results"
constructor({
memoryLimit,
timeout,
perRequestLimit,
}: {
memoryLimit: number
timeout: number
perRequestLimit?: number
}) {
this.#isolate = new ivm.Isolate({ memoryLimit })
this.#vm = this.#isolate.createContextSync()
this.#jail = this.#vm.global
this.#jail.setSync("global", this.#jail.derefInto())
this.#addToContext({
[this.#resultKey]: { out: "" },
})
this.#timeout = timeout
this.#perRequestLimit = perRequestLimit
}
withHelpers() {
const urlModule = this.#registerCallbacks({
resolve: url.resolve,
parse: url.parse,
})
const querystringModule = this.#registerCallbacks({
escape: querystring.escape,
})
this.#addToContext({
helpersStripProtocol: new ivm.Callback((str: string) => {
var parsed = url.parse(str) as any
parsed.protocol = ""
return parsed.format()
}),
})
const injectedRequire = `const require=function req(val) {
switch (val) {
case "url": return ${urlModule};
case "querystring": return ${querystringModule};
}
}`
const helpersSource = loadBundle(BundleType.HELPERS)
const helpersModule = this.#isolate.compileModuleSync(
`${injectedRequire};${helpersSource}`
)
helpersModule.instantiateSync(this.#vm, specifier => {
if (specifier === "crypto") {
const cryptoModule = this.#registerCallbacks({
randomUUID: crypto.randomUUID,
})
const module = this.#isolate.compileModuleSync(
`export default ${cryptoModule}`
)
module.instantiateSync(this.#vm, specifier => {
throw new Error(`No imports allowed. Required: ${specifier}`)
})
return module
}
throw new Error(`No imports allowed. Required: ${specifier}`)
})
this.#modules.push({
import: "helpers",
moduleKey: `i${crypto.randomUUID().replace(/-/g, "")}`,
module: helpersModule,
})
return this
}
withContext(context: Record<string, any>) {
this.#addToContext(context)
this.#handleBsonData()
return this
}
withParsingBson() {
this.#parseBson = true
this.#handleBsonData()
const bsonSource = loadBundle(BundleType.BSON)
const bsonModule = this.#isolate.compileModuleSync(bsonSource)
bsonModule.instantiateSync(this.#vm, specifier => {
throw new Error(`No imports allowed. Required: ${specifier}`)
})
this.#modules.push({
import: "{deserialize, toJson}",
moduleKey: `i${crypto.randomUUID().replace(/-/g, "")}`,
module: bsonModule,
})
return this
}
#handleBsonData() {
if (!this.#parseBson) {
return
}
const data = this.#getFromContext("data")
if (!data) {
return
}
this.#addToContext({
data: bson.BSON.serialize({ data }),
})
}
execute(code: string): string {
const perRequestLimit = this.#perRequestLimit
if (perRequestLimit) {
const cpuMs = Number(this.#isolate.cpuTime) / 1e6
if (cpuMs > perRequestLimit) {
throw new ExecutionTimeoutError(
`CPU time limit exceeded (${cpuMs}ms > ${perRequestLimit}ms)`
)
}
}
if (this.#parseBson) {
// If we need to parse bson, we follow the next steps:
// 1. Serialise the data from potential BSON to buffer before passing it to the isolate
// 2. Deserialise the data within the isolate, to get the original data
// 3. Process script
// 4. Stringify the result in order to convert the result from BSON to json
code = `toJson(
(function(){
data= deserialize(data).data;
return ${code};
})()
);`
}
code = [
...this.#modules.map(m => `import ${m.import} from "${m.moduleKey}"`),
`results.out=${code};`,
].join(";")
const script = this.#isolate.compileModuleSync(code)
script.instantiateSync(this.#vm, specifier => {
const module = this.#modules.find(m => m.moduleKey === specifier)
if (module) {
return module.module
}
throw new Error(`"${specifier}" import not allowed`)
})
script.evaluateSync({ timeout: this.#timeout })
const result = this.#getResult()
return result
}
#registerCallbacks(functions: Record<string, any>) {
const libId = crypto.randomUUID().replace(/-/g, "")
const x: Record<string, string> = {}
for (const [funcName, func] of Object.entries(functions)) {
const key = `f${libId}${funcName}cb`
x[funcName] = key
this.#addToContext({
[key]: new ivm.Callback((...params: any[]) => (func as any)(...params)),
})
}
const mod =
`{` +
Object.entries(x)
.map(([key, func]) => `${key}: ${func}`)
.join() +
"}"
return mod
}
#addToContext(context: Record<string, any>) {
for (let key in context) {
this.#jail.setSync(
key,
new ivm.ExternalCopy(context[key]).copyInto({ release: true })
)
}
}
#getFromContext(key: string) {
return this.#jail.getSync(key)
}
#getResult() {
const ref = this.#vm.global.getSync(this.#resultKey, { reference: true })
const result = ref.copySync()
ref.release()
return result.out
}
}

196
packages/server/src/utilities/scriptRunner.ts
View File

@ -1,117 +1,139 @@
import ivm from "isolated-vm"
import bson from "bson"
import { BundleType, loadBundle } from "../jsRunner/bundles"
import env from "../environment"
import { IsolatedVM } from "../jsRunner/vm"
const JS_TIMEOUT_MS = 1000
class ScriptRunner {
vm: IsolatedVM
#code: string
#vm: IsolatedVM
constructor(script: string, context: any, { parseBson = false } = {}) {
this.vm = new IsolatedVM({
this.#code = `(() => {${script}})();`
this.#vm = new IsolatedVM({
memoryLimit: env.JS_RUNNER_MEMORY_LIMIT,
parseBson,
context: {
...context,
data: parseBson
? bson.BSON.serialize({ data: context.data })
: context.data,
},
})
timeout: JS_TIMEOUT_MS,
}).withContext(context)
if (parseBson) {
// If we need to parse bson, we follow the next steps:
// 1. Serialise the data from potential BSON to buffer before passing it to the isolate
// 2. Deserialise the data within the isolate, to get the original data
// 3. Process script
// 4. Stringify the result in order to convert the result from BSON to json
script = `return toJson((function(){data=deserialize(data, { validation: { utf8: true } }).data;${script}})());`
this.#vm = this.#vm.withParsingBson()
}
this.vm.code = script
}
execute() {
this.vm.runScript()
const result = this.vm.getResult()
const result = this.#vm.execute(this.#code)
return result
}
}
class IsolatedVM {
#isolate: ivm.Isolate
#vm: ivm.Context
#jail: ivm.Reference
#script: ivm.Module = undefined!
#bsonModule?: ivm.Module
// <<<<<<< HEAD
// constructor(script: string, context: any, { parseBson = false } = {}) {
// this.vm = new IsolatedVM({
// memoryLimit: env.JS_RUNNER_MEMORY_LIMIT,
// parseBson,
// context: {
// ...context,
// data: parseBson
// ? bson.BSON.serialize({ data: context.data })
// : context.data,
// },
// })
readonly #resultKey = "results"
// if (parseBson) {
// // If we need to parse bson, we follow the next steps:
// // 1. Serialise the data from potential BSON to buffer before passing it to the isolate
// // 2. Deserialise the data within the isolate, to get the original data
// // 3. Process script
// // 4. Stringify the result in order to convert the result from BSON to json
// script = `return toJson((function(){data=deserialize(data, { validation: { utf8: true } }).data;${script}})());`
// }
constructor({
memoryLimit,
parseBson,
context,
}: {
memoryLimit: number
parseBson: boolean
context: Record<string, any>
}) {
this.#isolate = new ivm.Isolate({ memoryLimit })
this.#vm = this.#isolate.createContextSync()
this.#jail = this.#vm.global
this.#jail.setSync("global", this.#jail.derefInto())
// this.vm.code = script
// }
this.#addToContext(context)
this.#addToContext({
[this.#resultKey]: { out: "" },
})
// execute() {
// this.vm.runScript()
// const result = this.vm.getResult()
// return result
// }
// }
if (parseBson) {
const bsonSource = loadBundle(BundleType.BSON)
this.#bsonModule = this.#isolate.compileModuleSync(bsonSource)
this.#bsonModule.instantiateSync(this.#vm, specifier => {
throw new Error(`No imports allowed. Required: ${specifier}`)
})
}
}
// class IsolatedVM {
// #isolate: ivm.Isolate
// #vm: ivm.Context
// #jail: ivm.Reference
// #script: ivm.Module = undefined!
// #bsonModule?: ivm.Module
getResult() {
const ref = this.#vm.global.getSync(this.#resultKey, { reference: true })
const result = ref.copySync()
ref.release()
return result.out
}
// readonly #resultKey = "results"
#addToContext(context: Record<string, any>) {
for (let key in context) {
this.#jail.setSync(key, this.#copyRefToVm(context[key]))
}
}
// constructor({
// memoryLimit,
// parseBson,
// context,
// }: {
// memoryLimit: number
// parseBson: boolean
// context: Record<string, any>
// }) {
// this.#isolate = new ivm.Isolate({ memoryLimit })
// this.#vm = this.#isolate.createContextSync()
// this.#jail = this.#vm.global
// this.#jail.setSync("global", this.#jail.derefInto())
set code(code: string) {
code = `const fn=function(){${code}};results.out=fn();`
if (this.#bsonModule) {
code = `import {deserialize, toJson} from "compiled_module";${code}`
}
this.#script = this.#isolate.compileModuleSync(code)
}
// this.#addToContext(context)
// this.#addToContext({
// [this.#resultKey]: { out: "" },
// })
runScript(): void {
this.#script.instantiateSync(this.#vm, specifier => {
if (specifier === "compiled_module" && this.#bsonModule) {
return this.#bsonModule!
}
// if (parseBson) {
// const bsonSource = loadBundle(BundleType.BSON)
// this.#bsonModule = this.#isolate.compileModuleSync(bsonSource)
// this.#bsonModule.instantiateSync(this.#vm, specifier => {
// throw new Error(`No imports allowed. Required: ${specifier}`)
// })
// }
// }
throw new Error(`"${specifier}" import not allowed`)
})
// getResult() {
// const ref = this.#vm.global.getSync(this.#resultKey, { reference: true })
// const result = ref.copySync()
// ref.release()
// return result.out
// }
this.#script.evaluateSync({ timeout: JS_TIMEOUT_MS })
}
// #addToContext(context: Record<string, any>) {
// for (let key in context) {
// this.#jail.setSync(key, this.#copyRefToVm(context[key]))
// }
// }
// set code(code: string) {
// code = `const fn=function(){${code}};results.out=fn();`
// if (this.#bsonModule) {
// code = `import {deserialize, toJson} from "compiled_module";${code}`
// }
// this.#script = this.#isolate.compileModuleSync(code)
// }
// runScript(): void {
// this.#script.instantiateSync(this.#vm, specifier => {
// if (specifier === "compiled_module" && this.#bsonModule) {
// return this.#bsonModule!
// }
// throw new Error(`"${specifier}" import not allowed`)
// })
// this.#script.evaluateSync({ timeout: JS_TIMEOUT_MS })
// }
// #copyRefToVm(value: Object): ivm.Copy<Object> {
// return new ivm.ExternalCopy(value).copyInto({ release: true })
// }
// =======
// >>>>>>> isolated-vm-wrapper
// }
#copyRefToVm(value: Object): ivm.Copy<Object> {
return new ivm.ExternalCopy(value).copyInto({ release: true })
}
}
export default ScriptRunner

1
packages/types/src/sdk/index.ts
View File

@ -20,3 +20,4 @@ export * from "./cli"
export * from "./websocket"
export * from "./permissions"
export * from "./row"
export * from "./vm"

3
packages/types/src/sdk/vm.ts Normal file
View File

@ -0,0 +1,3 @@
export interface VM {
execute(code: string): string
}