Merge branch 'master' into fix-arm-image-with-isolated-vm

packages/builder/src/components/design/settings/controls/GridColumnConfiguration/FieldSetting.svelte

@@ -1,15 +1,25 @@
 <script>
   import EditComponentPopover from "../EditComponentPopover.svelte"
-  import { FieldTypeToComponentMap } from "../FieldConfiguration/utils"
   import { Toggle, Icon } from "@budibase/bbui"
   import { createEventDispatcher } from "svelte"
   import { cloneDeep } from "lodash/fp"
-  import { componentStore } from "stores/builder"
+  import { FIELDS } from "constants/backend"
 
   export let item
   export let anchor
 
   const dispatch = createEventDispatcher()
+
+  $: fieldIconLookupMap = buildFieldIconLookupMap(FIELDS)
+
+  const buildFieldIconLookupMap = fields => {
+    let map = {}
+    Object.values(fields).forEach(fieldInfo => {
+      map[fieldInfo.type] = fieldInfo.icon
+    })
+    return map
+  }
+
   const onToggle = item => {
     return e => {
       item.active = e.detail
@@ -24,13 +34,6 @@
         return { ...setting, nested: true }
       })
   }
-
-  const getIcon = () => {
-    const component = `@budibase/standard-components/${
-      FieldTypeToComponentMap[item.columnType]
-    }`
-    return componentStore.getDefinition(component).icon
-  }
 </script>
 
 <div class="list-item-body">
@@ -42,7 +45,7 @@
       on:change
     >
       <div slot="header" class="type-icon">
-        <Icon name={getIcon()} />
+        <Icon name={fieldIconLookupMap[item.columnType]} />
         <span>{item.field}</span>
       </div>
     </EditComponentPopover>

packages/server/package.json

@@ -13,8 +13,8 @@
     "build": "node ./scripts/build.js",
     "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../yarn.lock ./dist/",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null",
-    "build:isolated-vm-lib:string-templates": "esbuild --minify --bundle src/jsRunner/bundles/index-helpers.ts --outfile=src/jsRunner/bundles/index-helpers.ivm.bundle.js --platform=node --format=esm --external:handlebars",
+    "build:isolated-vm-lib:string-templates": "esbuild --minify --bundle src/jsRunner/bundles/index-helpers.ts --outfile=src/jsRunner/bundles/index-helpers.ivm.bundle.js --platform=node --format=iife --external:handlebars --global-name=helpers",
-    "build:isolated-vm-lib:bson": "esbuild --minify --bundle src/jsRunner/bundles/bsonPackage.ts --outfile=src/jsRunner/bundles/bson.ivm.bundle.js --platform=node --format=esm",
+    "build:isolated-vm-lib:bson": "esbuild --minify --bundle src/jsRunner/bundles/bsonPackage.ts --outfile=src/jsRunner/bundles/bson.ivm.bundle.js --platform=node --format=iife --global-name=bson",
     "build:isolated-vm-libs": "yarn build:isolated-vm-lib:string-templates && yarn build:isolated-vm-lib:bson",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",

packages/server/src/api/controllers/script.ts

@@ -1,10 +1,12 @@
-import ScriptRunner from "../../utilities/scriptRunner"
 import { Ctx } from "@budibase/types"
+import { IsolatedVM } from "../../jsRunner/vm"
 
 export async function execute(ctx: Ctx) {
   const { script, context } = ctx.request.body
-  const runner = new ScriptRunner(script, context)
+  const runner = new IsolatedVM().withContext(context)
-  ctx.body = runner.execute()
+
+  const result = runner.execute(`(function(){\n${script}\n})();`)
+  ctx.body = result
 }
 
 export async function save(ctx: Ctx) {

packages/server/src/environment.ts

@@ -113,6 +113,7 @@ const environment = {
     process.env[key] = value
     // @ts-ignore
     environment[key] = value
+    cleanVariables()
   },
   isTest: coreEnv.isTest,
   isJest: coreEnv.isJest,
@@ -128,18 +129,22 @@ const environment = {
   },
 }
 
-// clean up any environment variable edge cases
+function cleanVariables() {
-for (let [key, value] of Object.entries(environment)) {
+  // clean up any environment variable edge cases
-  // handle the edge case of "0" to disable an environment variable
+  for (let [key, value] of Object.entries(environment)) {
-  if (value === "0") {
+    // handle the edge case of "0" to disable an environment variable
-    // @ts-ignore
+    if (value === "0") {
-    environment[key] = 0
+      // @ts-ignore
-  }
+      environment[key] = 0
-  // handle the edge case of "false" to disable an environment variable
+    }
-  if (value === "false") {
+    // handle the edge case of "false" to disable an environment variable
-    // @ts-ignore
+    if (value === "false") {
-    environment[key] = 0
+      // @ts-ignore
+      environment[key] = 0
+    }
   }
 }
 
+cleanVariables()
+
 export default environment

packages/server/src/jsRunner/bundles/bsonPackage.ts

@@ -1,4 +1,2 @@
-import { EJSON } from "bson"
+export const deserialize = require("bson").deserialize
-
+export const toJson = require("bson").EJSON.deserialize
-export { deserialize } from "bson"
-export const toJson = EJSON.deserialize

packages/server/src/jsRunner/bundles/index-helpers.ts

@@ -2,9 +2,8 @@ const {
   getJsHelperList,
 } = require("../../../../string-templates/src/helpers/list.js")
 
-const helpers = getJsHelperList()
 export default {
-  ...helpers,
+  ...getJsHelperList(),
   // pointing stripProtocol to a unexisting function to be able to declare it on isolated-vm
   // @ts-ignore
   // eslint-disable-next-line no-undef

packages/server/src/jsRunner/bundles/index.ts

@@ -7,22 +7,19 @@ export const enum BundleType {
   BSON = "bson",
 }
 
-const bundleSourceCode = {
+const bundleSourceFile: Record<BundleType, string> = {
-  [BundleType.HELPERS]: "../bundles/index-helpers.ivm.bundle.js",
+  [BundleType.HELPERS]: "./index-helpers.ivm.bundle.js",
-  [BundleType.BSON]: "../bundles/bson.ivm.bundle.js",
+  [BundleType.BSON]: "./bson.ivm.bundle.js",
 }
+const bundleSourceCode: Partial<Record<BundleType, string>> = {}
 
 export function loadBundle(type: BundleType) {
-  if (environment.isJest()) {
+  let sourceCode = bundleSourceCode[type]
-    return fs.readFileSync(require.resolve(bundleSourceCode[type]), "utf-8")
+  if (sourceCode) {
+    return sourceCode
   }
 
-  switch (type) {
+  sourceCode = fs.readFileSync(require.resolve(bundleSourceFile[type]), "utf-8")
-    case BundleType.HELPERS:
+  bundleSourceCode[type] = sourceCode
-      return require("../bundles/index-helpers.ivm.bundle.js")
+  return sourceCode
-    case BundleType.BSON:
-      return require("../bundles/bson.ivm.bundle.js")
-    default:
-      utils.unreachable(type)
-  }
 }

packages/server/src/jsRunner/index.ts

@@ -1,68 +1,58 @@
-import vm from "vm"
-import env from "../environment"
-import { setJSRunner, setOnErrorLog } from "@budibase/string-templates"
-import { context, logging, timers } from "@budibase/backend-core"
-import tracer from "dd-trace"
 import { serializeError } from "serialize-error"
+import env from "../environment"
+import {
+  JsErrorTimeout,
+  setJSRunner,
+  setOnErrorLog,
+} from "@budibase/string-templates"
+import { context, logging } from "@budibase/backend-core"
+import tracer from "dd-trace"
 
-type TrackerFn = <T>(f: () => T) => T
+import { BuiltInVM, IsolatedVM } from "./vm"
+
+const USE_ISOLATED_VM = true
 
 export function init() {
-  setJSRunner((js: string, ctx: vm.Context) => {
+  setJSRunner((js: string, ctx: Record<string, any>) => {
     return tracer.trace("runJS", {}, span => {
-      const perRequestLimit = env.JS_PER_REQUEST_TIMEOUT_MS
+      if (!USE_ISOLATED_VM) {
-      let track: TrackerFn = f => f()
+        const vm = new BuiltInVM(ctx, span)
-      if (perRequestLimit) {
+        return vm.execute(js)
-        const bbCtx = tracer.trace("runJS.getCurrentContext", {}, span =>
+      }
-          context.getCurrentContext()
+
-        )
+      try {
-        if (bbCtx) {
+        const bbCtx = context.getCurrentContext()!
-          if (!bbCtx.jsExecutionTracker) {
+
-            span?.addTags({
+        let { vm } = bbCtx
-              createdExecutionTracker: true,
+        if (!vm) {
-            })
+          // Can't copy the native helpers into the isolate. We just ignore them as they are handled properly from the helpersSource
-            bbCtx.jsExecutionTracker = tracer.trace(
+          const { helpers, ...ctxToPass } = ctx
-              "runJS.createExecutionTimeTracker",
+
-              {},
+          vm = new IsolatedVM({
-              span => timers.ExecutionTimeTracker.withLimit(perRequestLimit)
+            memoryLimit: env.JS_RUNNER_MEMORY_LIMIT,
-            )
+            invocationTimeout: env.JS_PER_INVOCATION_TIMEOUT_MS,
-          }
+            isolateAccumulatedTimeout: env.JS_PER_REQUEST_TIMEOUT_MS,
-          span?.addTags({
-            js: {
-              limitMS: bbCtx.jsExecutionTracker.limitMs,
-              elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
-            },
-          })
-          // We call checkLimit() here to prevent paying the cost of creating
-          // a new VM context below when we don't need to.
-          tracer.trace("runJS.checkLimitAndBind", {}, span => {
-            bbCtx.jsExecutionTracker!.checkLimit()
-            track = bbCtx.jsExecutionTracker!.track.bind(
-              bbCtx.jsExecutionTracker
-            )
           })
+            .withContext(ctxToPass)
+            .withHelpers()
+
+          bbCtx.vm = vm
         }
+        return vm.execute(js)
+      } catch (error: any) {
+        if (error.message === "Script execution timed out.") {
+          throw new JsErrorTimeout()
+        }
+        throw error
       }
-
-      ctx = {
-        ...ctx,
-        alert: undefined,
-        setInterval: undefined,
-        setTimeout: undefined,
-      }
-
-      vm.createContext(ctx)
-      return track(() =>
-        vm.runInNewContext(js, ctx, {
-          timeout: env.JS_PER_INVOCATION_TIMEOUT_MS,
-        })
-      )
     })
   })
 
   if (env.LOG_JS_ERRORS) {
     setOnErrorLog((error: Error) => {
-      logging.logWarn(JSON.stringify(serializeError(error)))
+      logging.logWarn(
+        `Error while executing js: ${JSON.stringify(serializeError(error))}`
+      )
     })
   }
 }

packages/server/src/jsRunner/tests/jsRunner.spec.ts

@@ -1,22 +1,4 @@
-// import { validate as isValidUUID } from "uuid"
+import { validate as isValidUUID } from "uuid"
-
-jest.mock("@budibase/handlebars-helpers/lib/math", () => {
-  const actual = jest.requireActual("@budibase/handlebars-helpers/lib/math")
-
-  return {
-    ...actual,
-    random: () => 10,
-  }
-})
-jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
-  const actual = jest.requireActual("@budibase/handlebars-helpers/lib/uuid")
-
-  return {
-    ...actual,
-    uuid: () => "f34ebc66-93bd-4f7c-b79b-92b5569138bc",
-  }
-})
-
 import { processStringSync, encodeJSBinding } from "@budibase/string-templates"
 
 const { runJsHelpersTests } = require("@budibase/string-templates/test/utils")
@@ -27,7 +9,7 @@ import TestConfiguration from "../../tests/utilities/TestConfiguration"
 
 tk.freeze("2021-01-21T12:00:00")
 
-describe("jsRunner", () => {
+describe("jsRunner (using isolated-vm)", () => {
   const config = new TestConfiguration()
 
   beforeAll(async () => {
@@ -36,6 +18,10 @@ describe("jsRunner", () => {
     await config.init()
   })
 
+  afterAll(() => {
+    config.end()
+  })
+
   const processJS = (js: string, context?: object) => {
     return config.doInContext(config.getAppId(), async () =>
       processStringSync(encodeJSBinding(js), context || {})
@@ -47,10 +33,14 @@ describe("jsRunner", () => {
     expect(output).toBe(3)
   })
 
-  // TODO This should be reenabled when running on isolated-vm
+  it("it can execute sloppy javascript", async () => {
-  it.skip("should prevent sandbox escape", async () => {
+    const output = await processJS(`a=2;b=3;return a + b`)
+    expect(output).toBe(5)
+  })
+
+  it("should prevent sandbox escape", async () => {
     const output = await processJS(
-      `return this.constructor.constructor("return process")()`
+      `return this.constructor.constructor("return process.env")()`
     )
     expect(output).toBe("Error while executing JS")
   })
@@ -58,26 +48,26 @@ describe("jsRunner", () => {
   describe("helpers", () => {
     runJsHelpersTests({
       funcWrap: (func: any) => config.doInContext(config.getAppId(), func),
-      // testsToSkip: ["random", "uuid"],
+      testsToSkip: ["random", "uuid"],
     })
 
-    // describe("uuid", () => {
+    describe("uuid", () => {
-    //   it("uuid helper returns a valid uuid", async () => {
+      it("uuid helper returns a valid uuid", async () => {
-    //     const result = await processJS("return helpers.uuid()")
+        const result = await processJS("return helpers.uuid()")
-    //     expect(result).toBeDefined()
+        expect(result).toBeDefined()
-    //     expect(isValidUUID(result)).toBe(true)
+        expect(isValidUUID(result)).toBe(true)
-    //   })
+      })
-    // })
+    })
 
-    // describe("random", () => {
+    describe("random", () => {
-    //   it("random helper returns a valid number", async () => {
+      it("random helper returns a valid number", async () => {
-    //     const min = 1
+        const min = 1
-    //     const max = 8
+        const max = 8
-    //     const result = await processJS(`return helpers.random(${min}, ${max})`)
+        const result = await processJS(`return helpers.random(${min}, ${max})`)
-    //     expect(result).toBeDefined()
+        expect(result).toBeDefined()
-    //     expect(result).toBeGreaterThanOrEqual(min)
+        expect(result).toBeGreaterThanOrEqual(min)
-    //     expect(result).toBeLessThanOrEqual(max)
+        expect(result).toBeLessThanOrEqual(max)
-    //   })
+      })
-    // })
+    })
   })
 })

packages/server/src/jsRunner/vm/builtin-vm.ts

@@ -0,0 +1,65 @@
+import vm from "vm"
+import env from "../../environment"
+import { context, timers } from "@budibase/backend-core"
+import tracer, { Span } from "dd-trace"
+import { VM } from "@budibase/types"
+
+type TrackerFn = <T>(f: () => T) => T
+
+export class BuiltInVM implements VM {
+  private ctx: vm.Context
+  private span?: Span
+
+  constructor(ctx: vm.Context, span?: Span) {
+    this.ctx = ctx
+    this.span = span
+  }
+
+  execute(code: string) {
+    const perRequestLimit = env.JS_PER_REQUEST_TIMEOUT_MS
+    let track: TrackerFn = f => f()
+    if (perRequestLimit) {
+      const bbCtx = tracer.trace("runJS.getCurrentContext", {}, span =>
+        context.getCurrentContext()
+      )
+      if (bbCtx) {
+        if (!bbCtx.jsExecutionTracker) {
+          this.span?.addTags({
+            createdExecutionTracker: true,
+          })
+          bbCtx.jsExecutionTracker = tracer.trace(
+            "runJS.createExecutionTimeTracker",
+            {},
+            span => timers.ExecutionTimeTracker.withLimit(perRequestLimit)
+          )
+        }
+        this.span?.addTags({
+          js: {
+            limitMS: bbCtx.jsExecutionTracker.limitMs,
+            elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
+          },
+        })
+        // We call checkLimit() here to prevent paying the cost of creating
+        // a new VM context below when we don't need to.
+        tracer.trace("runJS.checkLimitAndBind", {}, span => {
+          bbCtx.jsExecutionTracker!.checkLimit()
+          track = bbCtx.jsExecutionTracker!.track.bind(bbCtx.jsExecutionTracker)
+        })
+      }
+    }
+
+    this.ctx = {
+      ...this.ctx,
+      alert: undefined,
+      setInterval: undefined,
+      setTimeout: undefined,
+    }
+
+    vm.createContext(this.ctx)
+    return track(() =>
+      vm.runInNewContext(code, this.ctx, {
+        timeout: env.JS_PER_INVOCATION_TIMEOUT_MS,
+      })
+    )
+  }
+}

packages/server/src/jsRunner/vm/index.ts

@@ -1,270 +1,3 @@
-import ivm from "isolated-vm"
+export * from "./isolated-vm"
-import bson from "bson"
+export * from "./builtin-vm"
-
+export * from "./vm2"
-import url from "url"
-import crypto from "crypto"
-import querystring from "querystring"
-
-import { BundleType, loadBundle } from "../bundles"
-import { VM } from "@budibase/types"
-
-class ExecutionTimeoutError extends Error {
-  constructor(message: string) {
-    super(message)
-    this.name = "ExecutionTimeoutError"
-  }
-}
-
-class ModuleHandler {
-  private modules: {
-    import: string
-    moduleKey: string
-    module: ivm.Module
-  }[] = []
-
-  private generateRandomKey = () => `i${crypto.randomUUID().replace(/-/g, "")}`
-
-  registerModule(module: ivm.Module, imports: string) {
-    this.modules.push({
-      moduleKey: this.generateRandomKey(),
-      import: imports,
-      module: module,
-    })
-  }
-
-  generateImports() {
-    return this.modules
-      .map(m => `import ${m.import} from "${m.moduleKey}"`)
-      .join(";")
-  }
-
-  getModule(key: string) {
-    const module = this.modules.find(m => m.moduleKey === key)
-    return module?.module
-  }
-}
-
-export class IsolatedVM implements VM {
-  private isolate: ivm.Isolate
-  private vm: ivm.Context
-  private jail: ivm.Reference
-  private invocationTimeout: number
-  private isolateAccumulatedTimeout?: number
-
-  // By default the wrapper returns itself
-  private codeWrapper: (code: string) => string = code => code
-
-  private moduleHandler = new ModuleHandler()
-
-  private readonly resultKey = "results"
-
-  constructor({
-    memoryLimit,
-    invocationTimeout,
-    isolateAccumulatedTimeout,
-  }: {
-    memoryLimit: number
-    invocationTimeout: number
-    isolateAccumulatedTimeout?: number
-  }) {
-    this.isolate = new ivm.Isolate({ memoryLimit })
-    this.vm = this.isolate.createContextSync()
-    this.jail = this.vm.global
-    this.jail.setSync("global", this.jail.derefInto())
-
-    this.addToContext({
-      [this.resultKey]: { out: "" },
-    })
-
-    this.invocationTimeout = invocationTimeout
-    this.isolateAccumulatedTimeout = isolateAccumulatedTimeout
-  }
-
-  withHelpers() {
-    const urlModule = this.registerCallbacks({
-      resolve: url.resolve,
-      parse: url.parse,
-    })
-
-    const querystringModule = this.registerCallbacks({
-      escape: querystring.escape,
-    })
-
-    this.addToContext({
-      helpersStripProtocol: new ivm.Callback((str: string) => {
-        var parsed = url.parse(str) as any
-        parsed.protocol = ""
-        return parsed.format()
-      }),
-    })
-
-    const injectedRequire = `const require=function req(val) {
-        switch (val) {
-            case "url": return ${urlModule};
-            case "querystring": return ${querystringModule};
-        }
-      }`
-    const helpersSource = loadBundle(BundleType.HELPERS)
-    const helpersModule = this.isolate.compileModuleSync(
-      `${injectedRequire};${helpersSource}`
-    )
-
-    helpersModule.instantiateSync(this.vm, specifier => {
-      if (specifier === "crypto") {
-        const cryptoModule = this.registerCallbacks({
-          randomUUID: crypto.randomUUID,
-        })
-        const module = this.isolate.compileModuleSync(
-          `export default ${cryptoModule}`
-        )
-        module.instantiateSync(this.vm, specifier => {
-          throw new Error(`No imports allowed. Required: ${specifier}`)
-        })
-        return module
-      }
-      throw new Error(`No imports allowed. Required: ${specifier}`)
-    })
-
-    this.moduleHandler.registerModule(helpersModule, "helpers")
-    return this
-  }
-
-  withContext(context: Record<string, any>) {
-    this.addToContext(context)
-
-    return this
-  }
-
-  withParsingBson(data: any) {
-    this.addToContext({
-      bsonData: bson.BSON.serialize({ data }),
-    })
-
-    // If we need to parse bson, we follow the next steps:
-    // 1. Serialise the data from potential BSON to buffer before passing it to the isolate
-    // 2. Deserialise the data within the isolate, to get the original data
-    // 3. Process script
-    // 4. Stringify the result in order to convert the result from BSON to json
-    this.codeWrapper = code =>
-      `(function(){
-            const data = deserialize(bsonData, { validation: { utf8: false } }).data;
-            const result = ${code}
-            return toJson(result);
-        })();`
-
-    const bsonSource = loadBundle(BundleType.BSON)
-
-    this.addToContext({
-      textDecoderCb: new ivm.Callback(
-        (args: {
-          constructorArgs: any
-          functionArgs: Parameters<InstanceType<typeof TextDecoder>["decode"]>
-        }) => {
-          const result = new TextDecoder(...args.constructorArgs).decode(
-            ...args.functionArgs
-          )
-          return result
-        }
-      ),
-    })
-
-    // "Polyfilling" text decoder. `bson.deserialize` requires decoding. We are creating a bridge function so we don't need to inject the full library
-    const textDecoderPolyfill = class TextDecoder {
-      constructorArgs
-
-      constructor(...constructorArgs: any) {
-        this.constructorArgs = constructorArgs
-      }
-
-      decode(...input: any) {
-        // @ts-ignore
-        return textDecoderCb({
-          constructorArgs: this.constructorArgs,
-          functionArgs: input,
-        })
-      }
-    }.toString()
-    const bsonModule = this.isolate.compileModuleSync(
-      `${textDecoderPolyfill};${bsonSource}`
-    )
-    bsonModule.instantiateSync(this.vm, specifier => {
-      throw new Error(`No imports allowed. Required: ${specifier}`)
-    })
-
-    this.moduleHandler.registerModule(bsonModule, "{deserialize, toJson}")
-
-    return this
-  }
-
-  execute(code: string): any {
-    if (this.isolateAccumulatedTimeout) {
-      const cpuMs = Number(this.isolate.cpuTime) / 1e6
-      if (cpuMs > this.isolateAccumulatedTimeout) {
-        throw new ExecutionTimeoutError(
-          `CPU time limit exceeded (${cpuMs}ms > ${this.isolateAccumulatedTimeout}ms)`
-        )
-      }
-    }
-
-    code = `${this.moduleHandler.generateImports()};results.out=${this.codeWrapper(
-      code
-    )};`
-
-    const script = this.isolate.compileModuleSync(code)
-
-    script.instantiateSync(this.vm, specifier => {
-      const module = this.moduleHandler.getModule(specifier)
-      if (module) {
-        return module
-      }
-
-      throw new Error(`"${specifier}" import not allowed`)
-    })
-
-    script.evaluateSync({ timeout: this.invocationTimeout })
-
-    const result = this.getFromContext(this.resultKey)
-    return result.out
-  }
-
-  private registerCallbacks(functions: Record<string, any>) {
-    const libId = crypto.randomUUID().replace(/-/g, "")
-
-    const x: Record<string, string> = {}
-    for (const [funcName, func] of Object.entries(functions)) {
-      const key = `f${libId}${funcName}cb`
-      x[funcName] = key
-
-      this.addToContext({
-        [key]: new ivm.Callback((...params: any[]) => (func as any)(...params)),
-      })
-    }
-
-    const mod =
-      `{` +
-      Object.entries(x)
-        .map(([key, func]) => `${key}: ${func}`)
-        .join() +
-      "}"
-    return mod
-  }
-
-  private addToContext(context: Record<string, any>) {
-    for (let key in context) {
-      const value = context[key]
-      this.jail.setSync(
-        key,
-        typeof value === "function"
-          ? value
-          : new ivm.ExternalCopy(value).copyInto({ release: true })
-      )
-    }
-  }
-
-  private getFromContext(key: string) {
-    const ref = this.vm.global.getSync(key, { reference: true })
-    const result = ref.copySync()
-    ref.release()
-    return result
-  }
-}

packages/server/src/jsRunner/vm/isolated-vm.ts

@@ -0,0 +1,236 @@
+import ivm from "isolated-vm"
+import bson from "bson"
+
+import url from "url"
+import crypto from "crypto"
+import querystring from "querystring"
+
+import { BundleType, loadBundle } from "../bundles"
+import { VM } from "@budibase/types"
+import environment from "../../environment"
+
+class ExecutionTimeoutError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = "ExecutionTimeoutError"
+  }
+}
+
+export class IsolatedVM implements VM {
+  private isolate: ivm.Isolate
+  private vm: ivm.Context
+  private jail: ivm.Reference
+  private invocationTimeout: number
+  private isolateAccumulatedTimeout?: number
+
+  // By default the wrapper returns itself
+  private codeWrapper: (code: string) => string = code => code
+
+  private readonly resultKey = "results"
+  private runResultKey: string
+
+  constructor({
+    memoryLimit,
+    invocationTimeout,
+    isolateAccumulatedTimeout,
+  }: {
+    memoryLimit?: number
+    invocationTimeout?: number
+    isolateAccumulatedTimeout?: number
+  } = {}) {
+    memoryLimit = memoryLimit || environment.JS_RUNNER_MEMORY_LIMIT
+    invocationTimeout = memoryLimit || 1000
+
+    this.isolate = new ivm.Isolate({ memoryLimit })
+    this.vm = this.isolate.createContextSync()
+    this.jail = this.vm.global
+    this.jail.setSync("global", this.jail.derefInto())
+
+    this.runResultKey = crypto.randomUUID()
+    this.addToContext({
+      [this.resultKey]: { [this.runResultKey]: "" },
+    })
+
+    this.invocationTimeout = invocationTimeout
+    this.isolateAccumulatedTimeout = isolateAccumulatedTimeout
+  }
+
+  withHelpers() {
+    const urlModule = this.registerCallbacks({
+      resolve: url.resolve,
+      parse: url.parse,
+    })
+
+    const querystringModule = this.registerCallbacks({
+      escape: querystring.escape,
+    })
+
+    const cryptoModule = this.registerCallbacks({
+      randomUUID: crypto.randomUUID,
+    })
+
+    this.addToContext({
+      helpersStripProtocol: new ivm.Callback((str: string) => {
+        var parsed = url.parse(str) as any
+        parsed.protocol = ""
+        return parsed.format()
+      }),
+    })
+
+    const injectedRequire = `require=function req(val) {
+        switch (val) {
+            case "url": return ${urlModule};
+            case "querystring": return ${querystringModule};
+            case "crypto": return ${cryptoModule};
+        }
+      }`
+    const helpersSource = loadBundle(BundleType.HELPERS)
+    const script = this.isolate.compileScriptSync(
+      `${injectedRequire};${helpersSource};helpers=helpers.default`
+    )
+
+    script.runSync(this.vm, { timeout: this.invocationTimeout, release: false })
+    new Promise(() => {
+      script.release()
+    })
+
+    return this
+  }
+
+  withContext(context: Record<string, any>) {
+    this.addToContext(context)
+
+    return this
+  }
+
+  withParsingBson(data: any) {
+    this.addToContext({
+      bsonData: bson.BSON.serialize({ data }),
+    })
+
+    // If we need to parse bson, we follow the next steps:
+    // 1. Serialise the data from potential BSON to buffer before passing it to the isolate
+    // 2. Deserialise the data within the isolate, to get the original data
+    // 3. Process script
+    // 4. Stringify the result in order to convert the result from BSON to json
+    this.codeWrapper = code =>
+      `(function(){
+            const data = bson.deserialize(bsonData, { validation: { utf8: false } }).data;
+            const result = ${code}
+            return bson.toJson(result);
+        })();`
+
+    const bsonSource = loadBundle(BundleType.BSON)
+
+    this.addToContext({
+      textDecoderCb: new ivm.Callback(
+        (args: {
+          constructorArgs: any
+          functionArgs: Parameters<InstanceType<typeof TextDecoder>["decode"]>
+        }) => {
+          const result = new TextDecoder(...args.constructorArgs).decode(
+            ...args.functionArgs
+          )
+          return result
+        }
+      ),
+    })
+
+    // "Polyfilling" text decoder. `bson.deserialize` requires decoding. We are creating a bridge function so we don't need to inject the full library
+    const textDecoderPolyfill = class TextDecoderMock {
+      constructorArgs
+
+      constructor(...constructorArgs: any) {
+        this.constructorArgs = constructorArgs
+      }
+
+      decode(...input: any) {
+        // @ts-ignore
+        return textDecoderCb({
+          constructorArgs: this.constructorArgs,
+          functionArgs: input,
+        })
+      }
+    }
+      .toString()
+      .replace(/TextDecoderMock/, "TextDecoder")
+
+    const script = this.isolate.compileScriptSync(
+      `${textDecoderPolyfill};${bsonSource}`
+    )
+    script.runSync(this.vm, { timeout: this.invocationTimeout, release: false })
+    new Promise(() => {
+      script.release()
+    })
+
+    return this
+  }
+
+  execute(code: string): any {
+    if (this.isolateAccumulatedTimeout) {
+      const cpuMs = Number(this.isolate.cpuTime) / 1e6
+      if (cpuMs > this.isolateAccumulatedTimeout) {
+        throw new ExecutionTimeoutError(
+          `CPU time limit exceeded (${cpuMs}ms > ${this.isolateAccumulatedTimeout}ms)`
+        )
+      }
+    }
+
+    code = `results['${this.runResultKey}']=${this.codeWrapper(code)}`
+
+    const script = this.isolate.compileScriptSync(code)
+
+    script.runSync(this.vm, { timeout: this.invocationTimeout, release: false })
+    new Promise(() => {
+      script.release()
+    })
+
+    // We can't rely on the script run result as it will not work for non-transferable values
+    const result = this.getFromContext(this.resultKey)
+    return result[this.runResultKey]
+  }
+
+  private registerCallbacks(functions: Record<string, any>) {
+    const libId = crypto.randomUUID().replace(/-/g, "")
+
+    const x: Record<string, string> = {}
+    for (const [funcName, func] of Object.entries(functions)) {
+      const key = `f${libId}${funcName}cb`
+      x[funcName] = key
+
+      this.addToContext({
+        [key]: new ivm.Callback((...params: any[]) => (func as any)(...params)),
+      })
+    }
+
+    const mod =
+      `{` +
+      Object.entries(x)
+        .map(([key, func]) => `${key}: ${func}`)
+        .join() +
+      "}"
+    return mod
+  }
+
+  private addToContext(context: Record<string, any>) {
+    for (let key in context) {
+      const value = context[key]
+      this.jail.setSync(
+        key,
+        typeof value === "function"
+          ? value
+          : new ivm.ExternalCopy(value).copyInto({ release: true })
+      )
+    }
+  }
+
+  private getFromContext(key: string) {
+    const ref = this.vm.global.getSync(key, { reference: true })
+    const result = ref.copySync()
+
+    new Promise(() => {
+      ref.release()
+    })
+    return result
+  }
+}

packages/server/src/jsRunner/vm/vm2.ts

@@ -1,29 +1,26 @@
-import fetch from "node-fetch"
+import vm2 from "vm2"
-import { VM, VMScript } from "vm2"
+import { VM } from "@budibase/types"
 
 const JS_TIMEOUT_MS = 1000
 
-class ScriptRunner {
+export class VM2 implements VM {
-  vm: VM
+  vm: vm2.VM
   results: { out: string }
-  script: VMScript
 
-  constructor(script: string, context: any) {
+  constructor(context: any) {
-    const code = `let fn = () => {\n${script}\n}; results.out = fn();`
+    this.vm = new vm2.VM({
-    this.vm = new VM({
       timeout: JS_TIMEOUT_MS,
     })
     this.results = { out: "" }
     this.vm.setGlobals(context)
     this.vm.setGlobal("fetch", fetch)
     this.vm.setGlobal("results", this.results)
-    this.script = new VMScript(code)
   }
 
-  execute() {
+  execute(script: string) {
-    this.vm.run(this.script)
+    const code = `let fn = () => {\n${script}\n}; results.out = fn();`
+    const vmScript = new vm2.VMScript(code)
+    this.vm.run(vmScript)
     return this.results.out
   }
 }
-
-export default ScriptRunner

packages/server/src/threads/query.ts

@@ -7,18 +7,20 @@ import {
   QueryVariable,
   QueryResponse,
 } from "./definitions"
-import ScriptRunner from "../utilities/scriptRunner"
+import { IsolatedVM, VM2 } from "../jsRunner/vm"
 import { getIntegration } from "../integrations"
 import { processStringSync } from "@budibase/string-templates"
 import { context, cache, auth } from "@budibase/backend-core"
 import { getGlobalIDFromUserMetadataID } from "../db/utils"
 import sdk from "../sdk"
 import { cloneDeep } from "lodash/fp"
-import { Datasource, Query, SourceName } from "@budibase/types"
+import { Datasource, Query, SourceName, VM } from "@budibase/types"
 
 import { isSQL } from "../integrations/utils"
 import { interpolateSQL } from "../integrations/queries/sql"
 
+const USE_ISOLATED_VM = true
+
 class QueryRunner {
   datasource: Datasource
   queryVerb: string
@@ -26,7 +28,7 @@ class QueryRunner {
   fields: any
   parameters: any
   pagination: any
-  transformer: any
+  transformer: string
   cachedVariables: any[]
   ctx: any
   queryResponse: any
@@ -127,11 +129,26 @@ class QueryRunner {
 
     // transform as required
     if (transformer) {
-      const runner = new ScriptRunner(transformer, {
+      let runner: VM
-        data: rows,
+      if (!USE_ISOLATED_VM) {
-        params: enrichedParameters,
+        runner = new VM2({
-      })
+          data: rows,
-      rows = runner.execute()
+          params: enrichedParameters,
+        })
+      } else {
+        transformer = `(function(){\n${transformer}\n})();`
+        let isolatedVm = new IsolatedVM().withContext({
+          data: rows,
+          params: enrichedParameters,
+        })
+        if (datasource.source === SourceName.MONGODB) {
+          isolatedVm = isolatedVm.withParsingBson(rows)
+        }
+
+        runner = isolatedVm
+      }
+
+      rows = runner.execute(transformer)
     }
 
     // if the request fails we retry once, invalidating the cached value

packages/server/src/utilities/rowProcessor/attachments.ts

@@ -72,6 +72,9 @@ export class AttachmentCleanup {
           continue
         }
         rows.forEach(row => {
+          if (!Array.isArray(row[key])) {
+            return
+          }
           files = files.concat(
             row[key].map((attachment: any) => attachment.key)
           )

packages/server/src/utilities/rowProcessor/tests/attachments.spec.ts

@@ -103,6 +103,14 @@ describe("attachment cleanup", () => {
     expect(mockedDeleteFiles).toBeCalledWith(BUCKET, [FILE_NAME])
   })
 
+  it("should handle row deletion and not throw when attachments are undefined", async () => {
+    await AttachmentCleanup.rowDelete(table(), [
+      {
+        attach: undefined,
+      },
+    ])
+  })
+
   it("shouldn't cleanup attachments if row not updated", async () => {
     await AttachmentCleanup.rowUpdate(table(), { row: row(), oldRow: row() })
     expect(mockedDeleteFiles).not.toBeCalled()

scripts/build.js

@@ -49,7 +49,6 @@ function runBuild(entry, outfile) {
     preserveSymlinks: true,
     loader: {
       ".svelte": "copy",
-      ".ivm.bundle.js": "text",
     },
     metafile: true,
     external: [
@@ -70,7 +69,7 @@ function runBuild(entry, outfile) {
     platform: "node",
     outfile,
   }).then(result => {
-    glob(`${process.cwd()}/src/**/*.hbs`, {}, (err, files) => {
+    glob(`${process.cwd()}/src/**/*.{hbs,ivm.bundle.js}`, {}, (err, files) => {
       for (const file of files) {
         fs.copyFileSync(file, `${process.cwd()}/dist/${path.basename(file)}`)
       }

yarn.lock

@@ -10763,7 +10763,7 @@ fetch-cookie@0.11.0:
   dependencies:
     tough-cookie "^2.3.3 || ^3.0.1 || ^4.0.0"
 
-fflate@^0.4.1, fflate@^0.4.8:
+fflate@^0.4.1:
   version "0.4.8"
   resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.4.8.tgz#f90b82aefbd8ac174213abb338bd7ef848f0f5ae"
   integrity sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA==