Merge isolated-vm-wrapper

packages/backend-core/src/context/types.ts

@@ -1,5 +1,4 @@
-import { IdentityContext } from "@budibase/types"
+import { IdentityContext, VM } from "@budibase/types"
-import { Isolate, Context, Module } from "isolated-vm"
 
 // keep this out of Budibase types, don't want to expose context info
 export type ContextMap = {
@@ -10,9 +9,5 @@ export type ContextMap = {
   isScim?: boolean
   automationId?: string
   isMigrating?: boolean
-  isolateRefs?: {
+  vm?: VM
-    jsIsolate: Isolate
-    jsContext: Context
-    helpersModule: Module
-  }
 }

packages/backend-core/src/objectStore/objectStore.ts

@@ -255,7 +255,8 @@ export async function listAllObjects(bucketName: string, path: string) {
       objects = objects.concat(response.Contents)
     }
     isTruncated = !!response.IsTruncated
-  } while (isTruncated)
+    token = response.NextContinuationToken
+  } while (isTruncated && token)
   return objects
 }
 

packages/backend-core/src/queue/queue.ts

@@ -2,7 +2,7 @@ import env from "../environment"
 import { getRedisOptions } from "../redis/utils"
 import { JobQueue } from "./constants"
 import InMemoryQueue from "./inMemoryQueue"
-import BullQueue, { QueueOptions } from "bull"
+import BullQueue, { QueueOptions, JobOptions } from "bull"
 import { addListeners, StalledFn } from "./listeners"
 import { Duration } from "../utils"
 import * as timers from "../timers"
@@ -24,17 +24,24 @@ async function cleanup() {
 
 export function createQueue<T>(
   jobQueue: JobQueue,
-  opts: { removeStalledCb?: StalledFn } = {}
+  opts: {
+    removeStalledCb?: StalledFn
+    maxStalledCount?: number
+    jobOptions?: JobOptions
+  } = {}
 ): BullQueue.Queue<T> {
   const redisOpts = getRedisOptions()
   const queueConfig: QueueOptions = {
     redis: redisOpts,
     settings: {
-      maxStalledCount: 0,
+      maxStalledCount: opts.maxStalledCount ? opts.maxStalledCount : 0,
       lockDuration: QUEUE_LOCK_MS,
       lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS,
     },
   }
+  if (opts.jobOptions) {
+    queueConfig.defaultJobOptions = opts.jobOptions
+  }
   let queue: any
   if (!env.isTest()) {
     queue = new BullQueue(jobQueue, queueConfig)

packages/builder/src/components/common/CodeEditor/index.js

@@ -286,7 +286,13 @@ export const hbInsert = (value, from, to, text) => {
   return parsedInsert
 }
 
-export function jsInsert(value, from, to, text, { helper, disableWrapping }) {
+export function jsInsert(
+  value,
+  from,
+  to,
+  text,
+  { helper, disableWrapping } = {}
+) {
   let parsedInsert = ""
 
   const left = from ? value.substring(0, from) : ""

packages/pro

@@ -1 +1 @@
-Subproject commit aaf7101cd1493215155cc8f83124c70d53eb1be4
+Subproject commit 9b14e5d5182bf5e5ee98f717997e7352e5904799

packages/server/src/jsRunner/index.ts

@@ -1,155 +1,28 @@
-import ivm from "isolated-vm"
 import env from "../environment"
 import { setJSRunner, JsErrorTimeout } from "@budibase/string-templates"
-import { context } from "@budibase/backend-core"
 import tracer from "dd-trace"
-import url from "url"
-import crypto from "crypto"
-import querystring from "querystring"
-import { BundleType, loadBundle } from "./bundles"
 
-class ExecutionTimeoutError extends Error {
+import { IsolatedVM } from "./vm"
-  constructor(message: string) {
+import { context } from "@budibase/backend-core"
-    super(message)
-    this.name = "ExecutionTimeoutError"
-  }
-}
 
 export function init() {
-  const helpersSource = loadBundle(BundleType.HELPERS)
   setJSRunner((js: string, ctx: Record<string, any>) => {
     return tracer.trace("runJS", {}, span => {
       try {
         const bbCtx = context.getCurrentContext()!
 
-        const isolateRefs = bbCtx.isolateRefs
+        let { vm } = bbCtx
-        if (!isolateRefs) {
+        if (!vm) {
-          const jsIsolate = new ivm.Isolate({
+          vm = new IsolatedVM({
             memoryLimit: env.JS_RUNNER_MEMORY_LIMIT,
-          })
+            timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
-          const jsContext = jsIsolate.createContextSync()
+            perRequestLimit: env.JS_PER_REQUEST_TIME_LIMIT_MS,
+          }).withHelpers()
 
-          const injectedRequire = `
+          bbCtx.vm = vm
-            const require = function(val){
-              switch (val) {
-                case "url": 
-                  return {
-                    resolve: (...params) => urlResolveCb(...params),
-                    parse: (...params) => urlParseCb(...params),
-                  }
-                case "querystring":
-                  return {
-                    escape: (...params) => querystringEscapeCb(...params),
-                  }
-              }
-            };`
-
-          const global = jsContext.global
-          global.setSync(
-            "urlResolveCb",
-            new ivm.Callback((...params: Parameters<typeof url.resolve>) =>
-              url.resolve(...params)
-            )
-          )
-
-          global.setSync(
-            "urlParseCb",
-            new ivm.Callback((...params: Parameters<typeof url.parse>) =>
-              url.parse(...params)
-            )
-          )
-
-          global.setSync(
-            "querystringEscapeCb",
-            new ivm.Callback(
-              (...params: Parameters<typeof querystring.escape>) =>
-                querystring.escape(...params)
-            )
-          )
-
-          global.setSync(
-            "helpersStripProtocol",
-            new ivm.Callback((str: string) => {
-              var parsed = url.parse(str) as any
-              parsed.protocol = ""
-              return parsed.format()
-            })
-          )
-
-          const helpersModule = jsIsolate.compileModuleSync(
-            `${injectedRequire};${helpersSource}`
-          )
-
-          const cryptoModule = jsIsolate.compileModuleSync(
-            `export default { randomUUID: cryptoRandomUUIDCb }`
-          )
-          cryptoModule.instantiateSync(jsContext, specifier => {
-            throw new Error(`No imports allowed. Required: ${specifier}`)
-          })
-
-          global.setSync(
-            "cryptoRandomUUIDCb",
-            new ivm.Callback(
-              (...params: Parameters<typeof crypto.randomUUID>) => {
-                return crypto.randomUUID(...params)
-              }
-            )
-          )
-
-          helpersModule.instantiateSync(jsContext, specifier => {
-            if (specifier === "crypto") {
-              return cryptoModule
-            }
-            throw new Error(`No imports allowed. Required: ${specifier}`)
-          })
-
-          for (const [key, value] of Object.entries(ctx)) {
-            if (key === "helpers") {
-              // Can't copy the native helpers into the isolate. We just ignore them as they are handled properly from the helpersSource
-              continue
-            }
-            global.setSync(key, value)
-          }
-
-          bbCtx.isolateRefs = { jsContext, jsIsolate, helpersModule }
         }
 
-        let { jsIsolate, jsContext, helpersModule } = bbCtx.isolateRefs!
+        const result = vm.execute(js)
-
-        const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
-        if (perRequestLimit) {
-          const cpuMs = Number(jsIsolate.cpuTime) / 1e6
-          if (cpuMs > perRequestLimit) {
-            throw new ExecutionTimeoutError(
-              `CPU time limit exceeded (${cpuMs}ms > ${perRequestLimit}ms)`
-            )
-          }
-        }
-
-        const script = jsIsolate.compileModuleSync(
-          `import helpers from "compiled_module";const result=${js};cb(result)`,
-          {}
-        )
-
-        script.instantiateSync(jsContext, specifier => {
-          if (specifier === "compiled_module") {
-            return helpersModule
-          }
-
-          throw new Error(`"${specifier}" import not allowed`)
-        })
-
-        let result
-        jsContext.global.setSync(
-          "cb",
-          new ivm.Callback((value: any) => {
-            result = value
-          })
-        )
-
-        script.evaluateSync({
-          timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
-        })
 
         return result
       } catch (error: any) {

packages/server/src/jsRunner/vm/index.ts

@@ -0,0 +1,240 @@
+import ivm from "isolated-vm"
+import bson from "bson"
+
+import url from "url"
+import crypto from "crypto"
+import querystring from "querystring"
+
+import { BundleType, loadBundle } from "../bundles"
+import { VM } from "@budibase/types"
+import { context } from "@budibase/backend-core"
+
+class ExecutionTimeoutError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = "ExecutionTimeoutError"
+  }
+}
+
+export class IsolatedVM implements VM {
+  #isolate: ivm.Isolate
+  #vm: ivm.Context
+  #jail: ivm.Reference
+  #timeout: number
+  #perRequestLimit?: number
+
+  #parseBson?: boolean
+
+  #modules: {
+    import: string
+    moduleKey: string
+    module: ivm.Module
+  }[] = []
+
+  readonly #resultKey = "results"
+
+  constructor({
+    memoryLimit,
+    timeout,
+    perRequestLimit,
+  }: {
+    memoryLimit: number
+    timeout: number
+    perRequestLimit?: number
+  }) {
+    this.#isolate = new ivm.Isolate({ memoryLimit })
+    this.#vm = this.#isolate.createContextSync()
+    this.#jail = this.#vm.global
+    this.#jail.setSync("global", this.#jail.derefInto())
+
+    this.#addToContext({
+      [this.#resultKey]: { out: "" },
+    })
+
+    this.#timeout = timeout
+    this.#perRequestLimit = perRequestLimit
+  }
+
+  withHelpers() {
+    const urlModule = this.#registerCallbacks({
+      resolve: url.resolve,
+      parse: url.parse,
+    })
+
+    const querystringModule = this.#registerCallbacks({
+      escape: querystring.escape,
+    })
+
+    this.#addToContext({
+      helpersStripProtocol: new ivm.Callback((str: string) => {
+        var parsed = url.parse(str) as any
+        parsed.protocol = ""
+        return parsed.format()
+      }),
+    })
+
+    const injectedRequire = `const require=function req(val) {
+        switch (val) {
+            case "url": return ${urlModule};
+            case "querystring": return ${querystringModule};
+        }
+      }`
+    const helpersSource = loadBundle(BundleType.HELPERS)
+    const helpersModule = this.#isolate.compileModuleSync(
+      `${injectedRequire};${helpersSource}`
+    )
+
+    helpersModule.instantiateSync(this.#vm, specifier => {
+      if (specifier === "crypto") {
+        const cryptoModule = this.#registerCallbacks({
+          randomUUID: crypto.randomUUID,
+        })
+        const module = this.#isolate.compileModuleSync(
+          `export default ${cryptoModule}`
+        )
+        module.instantiateSync(this.#vm, specifier => {
+          throw new Error(`No imports allowed. Required: ${specifier}`)
+        })
+        return module
+      }
+      throw new Error(`No imports allowed. Required: ${specifier}`)
+    })
+
+    this.#modules.push({
+      import: "helpers",
+      moduleKey: `i${crypto.randomUUID().replace(/-/g, "")}`,
+      module: helpersModule,
+    })
+    return this
+  }
+
+  withContext(context: Record<string, any>) {
+    this.#addToContext(context)
+    this.#handleBsonData()
+    return this
+  }
+
+  withParsingBson() {
+    this.#parseBson = true
+    this.#handleBsonData()
+
+    const bsonSource = loadBundle(BundleType.BSON)
+    const bsonModule = this.#isolate.compileModuleSync(bsonSource)
+    bsonModule.instantiateSync(this.#vm, specifier => {
+      throw new Error(`No imports allowed. Required: ${specifier}`)
+    })
+
+    this.#modules.push({
+      import: "{deserialize, toJson}",
+      moduleKey: `i${crypto.randomUUID().replace(/-/g, "")}`,
+      module: bsonModule,
+    })
+
+    return this
+  }
+
+  #handleBsonData() {
+    if (!this.#parseBson) {
+      return
+    }
+
+    const data = this.#getFromContext("data")
+    if (!data) {
+      return
+    }
+
+    this.#addToContext({
+      data: bson.BSON.serialize({ data }),
+    })
+  }
+
+  execute(code: string): string {
+    const perRequestLimit = this.#perRequestLimit
+
+    if (perRequestLimit) {
+      const cpuMs = Number(this.#isolate.cpuTime) / 1e6
+      if (cpuMs > perRequestLimit) {
+        throw new ExecutionTimeoutError(
+          `CPU time limit exceeded (${cpuMs}ms > ${perRequestLimit}ms)`
+        )
+      }
+    }
+
+    if (this.#parseBson) {
+      // If we need to parse bson, we follow the next steps:
+      // 1. Serialise the data from potential BSON to buffer before passing it to the isolate
+      // 2. Deserialise the data within the isolate, to get the original data
+      // 3. Process script
+      // 4. Stringify the result in order to convert the result from BSON to json
+      code = `toJson(
+                (function(){
+                    data= deserialize(data).data;
+                    return ${code};
+                })()
+            );`
+    }
+
+    code = [
+      ...this.#modules.map(m => `import ${m.import} from "${m.moduleKey}"`),
+      `results.out=${code};`,
+    ].join(";")
+
+    const script = this.#isolate.compileModuleSync(code)
+
+    script.instantiateSync(this.#vm, specifier => {
+      const module = this.#modules.find(m => m.moduleKey === specifier)
+      if (module) {
+        return module.module
+      }
+
+      throw new Error(`"${specifier}" import not allowed`)
+    })
+
+    script.evaluateSync({ timeout: this.#timeout })
+
+    const result = this.#getResult()
+    return result
+  }
+
+  #registerCallbacks(functions: Record<string, any>) {
+    const libId = crypto.randomUUID().replace(/-/g, "")
+
+    const x: Record<string, string> = {}
+    for (const [funcName, func] of Object.entries(functions)) {
+      const key = `f${libId}${funcName}cb`
+      x[funcName] = key
+
+      this.#addToContext({
+        [key]: new ivm.Callback((...params: any[]) => (func as any)(...params)),
+      })
+    }
+
+    const mod =
+      `{` +
+      Object.entries(x)
+        .map(([key, func]) => `${key}: ${func}`)
+        .join() +
+      "}"
+    return mod
+  }
+
+  #addToContext(context: Record<string, any>) {
+    for (let key in context) {
+      this.#jail.setSync(
+        key,
+        new ivm.ExternalCopy(context[key]).copyInto({ release: true })
+      )
+    }
+  }
+
+  #getFromContext(key: string) {
+    return this.#jail.getSync(key)
+  }
+
+  #getResult() {
+    const ref = this.#vm.global.getSync(this.#resultKey, { reference: true })
+    const result = ref.copySync()
+    ref.release()
+    return result.out
+  }
+}

packages/server/src/utilities/scriptRunner.ts

@@ -1,117 +1,139 @@
-import ivm from "isolated-vm"
-
 import bson from "bson"
-import { BundleType, loadBundle } from "../jsRunner/bundles"
+
 import env from "../environment"
+import { IsolatedVM } from "../jsRunner/vm"
 
 const JS_TIMEOUT_MS = 1000
-
 class ScriptRunner {
-  vm: IsolatedVM
+  #code: string
+  #vm: IsolatedVM
 
   constructor(script: string, context: any, { parseBson = false } = {}) {
-    this.vm = new IsolatedVM({
+    this.#code = `(() => {${script}})();`
+    this.#vm = new IsolatedVM({
       memoryLimit: env.JS_RUNNER_MEMORY_LIMIT,
-      parseBson,
+      timeout: JS_TIMEOUT_MS,
-      context: {
+    }).withContext(context)
-        ...context,
-        data: parseBson
-          ? bson.BSON.serialize({ data: context.data })
-          : context.data,
-      },
-    })
 
     if (parseBson) {
-      // If we need to parse bson, we follow the next steps:
+      this.#vm = this.#vm.withParsingBson()
-      // 1. Serialise the data from potential BSON to buffer before passing it to the isolate
-      // 2. Deserialise the data within the isolate, to get the original data
-      // 3. Process script
-      // 4. Stringify the result in order to convert the result from BSON to json
-      script = `return toJson((function(){data=deserialize(data, { validation: { utf8: true } }).data;${script}})());`
     }
-
-    this.vm.code = script
   }
 
   execute() {
-    this.vm.runScript()
+    const result = this.#vm.execute(this.#code)
-    const result = this.vm.getResult()
     return result
   }
 }
 
-class IsolatedVM {
+// <<<<<<< HEAD
-  #isolate: ivm.Isolate
+//   constructor(script: string, context: any, { parseBson = false } = {}) {
-  #vm: ivm.Context
+//     this.vm = new IsolatedVM({
-  #jail: ivm.Reference
+//       memoryLimit: env.JS_RUNNER_MEMORY_LIMIT,
-  #script: ivm.Module = undefined!
+//       parseBson,
-  #bsonModule?: ivm.Module
+//       context: {
+//         ...context,
+//         data: parseBson
+//           ? bson.BSON.serialize({ data: context.data })
+//           : context.data,
+//       },
+//     })
 
-  readonly #resultKey = "results"
+//     if (parseBson) {
+//       // If we need to parse bson, we follow the next steps:
+//       // 1. Serialise the data from potential BSON to buffer before passing it to the isolate
+//       // 2. Deserialise the data within the isolate, to get the original data
+//       // 3. Process script
+//       // 4. Stringify the result in order to convert the result from BSON to json
+//       script = `return toJson((function(){data=deserialize(data, { validation: { utf8: true } }).data;${script}})());`
+//     }
 
-  constructor({
+//     this.vm.code = script
-    memoryLimit,
+//   }
-    parseBson,
-    context,
-  }: {
-    memoryLimit: number
-    parseBson: boolean
-    context: Record<string, any>
-  }) {
-    this.#isolate = new ivm.Isolate({ memoryLimit })
-    this.#vm = this.#isolate.createContextSync()
-    this.#jail = this.#vm.global
-    this.#jail.setSync("global", this.#jail.derefInto())
 
-    this.#addToContext(context)
+//   execute() {
-    this.#addToContext({
+//     this.vm.runScript()
-      [this.#resultKey]: { out: "" },
+//     const result = this.vm.getResult()
-    })
+//     return result
+//   }
+// }
 
-    if (parseBson) {
+// class IsolatedVM {
-      const bsonSource = loadBundle(BundleType.BSON)
+//   #isolate: ivm.Isolate
-      this.#bsonModule = this.#isolate.compileModuleSync(bsonSource)
+//   #vm: ivm.Context
-      this.#bsonModule.instantiateSync(this.#vm, specifier => {
+//   #jail: ivm.Reference
-        throw new Error(`No imports allowed. Required: ${specifier}`)
+//   #script: ivm.Module = undefined!
-      })
+//   #bsonModule?: ivm.Module
-    }
-  }
 
-  getResult() {
+//   readonly #resultKey = "results"
-    const ref = this.#vm.global.getSync(this.#resultKey, { reference: true })
-    const result = ref.copySync()
-    ref.release()
-    return result.out
-  }
 
-  #addToContext(context: Record<string, any>) {
+//   constructor({
-    for (let key in context) {
+//     memoryLimit,
-      this.#jail.setSync(key, this.#copyRefToVm(context[key]))
+//     parseBson,
-    }
+//     context,
-  }
+//   }: {
+//     memoryLimit: number
+//     parseBson: boolean
+//     context: Record<string, any>
+//   }) {
+//     this.#isolate = new ivm.Isolate({ memoryLimit })
+//     this.#vm = this.#isolate.createContextSync()
+//     this.#jail = this.#vm.global
+//     this.#jail.setSync("global", this.#jail.derefInto())
 
-  set code(code: string) {
+//     this.#addToContext(context)
-    code = `const fn=function(){${code}};results.out=fn();`
+//     this.#addToContext({
-    if (this.#bsonModule) {
+//       [this.#resultKey]: { out: "" },
-      code = `import {deserialize, toJson} from "compiled_module";${code}`
+//     })
-    }
-    this.#script = this.#isolate.compileModuleSync(code)
-  }
 
-  runScript(): void {
+//     if (parseBson) {
-    this.#script.instantiateSync(this.#vm, specifier => {
+//       const bsonSource = loadBundle(BundleType.BSON)
-      if (specifier === "compiled_module" && this.#bsonModule) {
+//       this.#bsonModule = this.#isolate.compileModuleSync(bsonSource)
-        return this.#bsonModule!
+//       this.#bsonModule.instantiateSync(this.#vm, specifier => {
-      }
+//         throw new Error(`No imports allowed. Required: ${specifier}`)
+//       })
+//     }
+//   }
 
-      throw new Error(`"${specifier}" import not allowed`)
+//   getResult() {
-    })
+//     const ref = this.#vm.global.getSync(this.#resultKey, { reference: true })
+//     const result = ref.copySync()
+//     ref.release()
+//     return result.out
+//   }
 
-    this.#script.evaluateSync({ timeout: JS_TIMEOUT_MS })
+//   #addToContext(context: Record<string, any>) {
-  }
+//     for (let key in context) {
+//       this.#jail.setSync(key, this.#copyRefToVm(context[key]))
+//     }
+//   }
+
+//   set code(code: string) {
+//     code = `const fn=function(){${code}};results.out=fn();`
+//     if (this.#bsonModule) {
+//       code = `import {deserialize, toJson} from "compiled_module";${code}`
+//     }
+//     this.#script = this.#isolate.compileModuleSync(code)
+//   }
+
+//   runScript(): void {
+//     this.#script.instantiateSync(this.#vm, specifier => {
+//       if (specifier === "compiled_module" && this.#bsonModule) {
+//         return this.#bsonModule!
+//       }
+
+//       throw new Error(`"${specifier}" import not allowed`)
+//     })
+
+//     this.#script.evaluateSync({ timeout: JS_TIMEOUT_MS })
+//   }
+
+//   #copyRefToVm(value: Object): ivm.Copy<Object> {
+//     return new ivm.ExternalCopy(value).copyInto({ release: true })
+//   }
+// =======
+
+// >>>>>>> isolated-vm-wrapper
+// }
 
-  #copyRefToVm(value: Object): ivm.Copy<Object> {
-    return new ivm.ExternalCopy(value).copyInto({ release: true })
-  }
-}
 export default ScriptRunner

packages/types/src/sdk/index.ts

@@ -20,3 +20,4 @@ export * from "./cli"
 export * from "./websocket"
 export * from "./permissions"
 export * from "./row"
+export * from "./vm"

packages/types/src/sdk/vm.ts

@@ -0,0 +1,3 @@
+export interface VM {
+  execute(code: string): string
+}