Merge branch 'master' into remove-vm2-refs

packages/server/package.json

@@ -13,8 +13,8 @@
     "build": "node ./scripts/build.js",
     "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../yarn.lock ./dist/",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null",
-    "build:isolated-vm-lib:string-templates": "esbuild --minify --bundle src/jsRunner/bundles/index-helpers.ts --outfile=src/jsRunner/bundles/index-helpers.ivm.bundle.js --platform=node --format=esm --external:handlebars",
-    "build:isolated-vm-lib:bson": "esbuild --minify --bundle src/jsRunner/bundles/bsonPackage.ts --outfile=src/jsRunner/bundles/bson.ivm.bundle.js --platform=node --format=esm",
+    "build:isolated-vm-lib:string-templates": "esbuild --minify --bundle src/jsRunner/bundles/index-helpers.ts --outfile=src/jsRunner/bundles/index-helpers.ivm.bundle.js --platform=node --format=iife --external:handlebars --global-name=helpers",
+    "build:isolated-vm-lib:bson": "esbuild --minify --bundle src/jsRunner/bundles/bsonPackage.ts --outfile=src/jsRunner/bundles/bson.ivm.bundle.js --platform=node --format=iife --global-name=bson",
     "build:isolated-vm-libs": "yarn build:isolated-vm-lib:string-templates && yarn build:isolated-vm-lib:bson",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",

packages/server/src/environment.ts

@@ -113,6 +113,7 @@ const environment = {
     process.env[key] = value
     // @ts-ignore
     environment[key] = value
+    cleanVariables()
   },
   isTest: coreEnv.isTest,
   isJest: coreEnv.isJest,
@@ -128,18 +129,22 @@ const environment = {
   },
 }
 
-// clean up any environment variable edge cases
-for (let [key, value] of Object.entries(environment)) {
-  // handle the edge case of "0" to disable an environment variable
-  if (value === "0") {
-    // @ts-ignore
-    environment[key] = 0
-  }
-  // handle the edge case of "false" to disable an environment variable
-  if (value === "false") {
-    // @ts-ignore
-    environment[key] = 0
+function cleanVariables() {
+  // clean up any environment variable edge cases
+  for (let [key, value] of Object.entries(environment)) {
+    // handle the edge case of "0" to disable an environment variable
+    if (value === "0") {
+      // @ts-ignore
+      environment[key] = 0
+    }
+    // handle the edge case of "false" to disable an environment variable
+    if (value === "false") {
+      // @ts-ignore
+      environment[key] = 0
+    }
   }
 }
 
+cleanVariables()
+
 export default environment

packages/server/src/jsRunner/bundles/bsonPackage.ts

@@ -1,4 +1,2 @@
-import { EJSON } from "bson"
-
-export { deserialize } from "bson"
-export const toJson = EJSON.deserialize
+export const deserialize = require("bson").deserialize
+export const toJson = require("bson").EJSON.deserialize

packages/server/src/jsRunner/bundles/index-helpers.ts

@@ -2,9 +2,8 @@ const {
   getJsHelperList,
 } = require("../../../../string-templates/src/helpers/list.js")
 
-const helpers = getJsHelperList()
 export default {
-  ...helpers,
+  ...getJsHelperList(),
   // pointing stripProtocol to a unexisting function to be able to declare it on isolated-vm
   // @ts-ignore
   // eslint-disable-next-line no-undef

packages/server/src/jsRunner/bundles/index.ts

@@ -7,22 +7,19 @@ export const enum BundleType {
   BSON = "bson",
 }
 
-const bundleSourceCode = {
-  [BundleType.HELPERS]: "../bundles/index-helpers.ivm.bundle.js",
-  [BundleType.BSON]: "../bundles/bson.ivm.bundle.js",
+const bundleSourceFile: Record<BundleType, string> = {
+  [BundleType.HELPERS]: "./index-helpers.ivm.bundle.js",
+  [BundleType.BSON]: "./bson.ivm.bundle.js",
 }
+const bundleSourceCode: Partial<Record<BundleType, string>> = {}
 
 export function loadBundle(type: BundleType) {
-  if (environment.isJest()) {
-    return fs.readFileSync(require.resolve(bundleSourceCode[type]), "utf-8")
+  let sourceCode = bundleSourceCode[type]
+  if (sourceCode) {
+    return sourceCode
   }
 
-  switch (type) {
-    case BundleType.HELPERS:
-      return require("../bundles/index-helpers.ivm.bundle.js")
-    case BundleType.BSON:
-      return require("../bundles/bson.ivm.bundle.js")
-    default:
-      utils.unreachable(type)
-  }
+  sourceCode = fs.readFileSync(require.resolve(bundleSourceFile[type]), "utf-8")
+  bundleSourceCode[type] = sourceCode
+  return sourceCode
 }

packages/server/src/jsRunner/index.ts

@@ -1,22 +1,44 @@
-import vm from "vm"
 import env from "../environment"
-import { setJSRunner, setOnErrorLog } from "@budibase/string-templates"
-import { logging } from "@budibase/backend-core"
+import { JsErrorTimeout, setJSRunner } from "@budibase/string-templates"
+import { context } from "@budibase/backend-core"
 import tracer from "dd-trace"
-import { serializeError } from "serialize-error"
-import { BuiltInVM } from "./vm"
+import { BuiltInVM, IsolatedVM } from "./vm"
+
+const USE_ISOLATED_VM = true
 
 export function init() {
-  setJSRunner((js: string, ctx: vm.Context) => {
+  setJSRunner((js: string, ctx: Record<string, any>) => {
     return tracer.trace("runJS", {}, span => {
-      const vm = new BuiltInVM(ctx, span)
-      return vm.execute(js)
+      if (!USE_ISOLATED_VM) {
+        const vm = new BuiltInVM(ctx, span)
+        return vm.execute(js)
+      }
+
+      try {
+        const bbCtx = context.getCurrentContext()!
+
+        let { vm } = bbCtx
+        if (!vm) {
+          // Can't copy the native helpers into the isolate. We just ignore them as they are handled properly from the helpersSource
+          const { helpers, ...ctxToPass } = ctx
+
+          vm = new IsolatedVM({
+            memoryLimit: env.JS_RUNNER_MEMORY_LIMIT,
+            invocationTimeout: env.JS_PER_INVOCATION_TIMEOUT_MS,
+            isolateAccumulatedTimeout: env.JS_PER_REQUEST_TIMEOUT_MS,
+          })
+            .withContext(ctxToPass)
+            .withHelpers()
+
+          bbCtx.vm = vm
+        }
+        return vm.execute(js)
+      } catch (error: any) {
+        if (error.message === "Script execution timed out.") {
+          throw new JsErrorTimeout()
+        }
+        throw error
+      }
     })
   })
-
-  if (env.LOG_JS_ERRORS) {
-    setOnErrorLog((error: Error) => {
-      logging.logWarn(JSON.stringify(serializeError(error)))
-    })
-  }
 }

packages/server/src/jsRunner/tests/jsRunner.spec.ts

@@ -1,22 +1,4 @@
-// import { validate as isValidUUID } from "uuid"
-
-jest.mock("@budibase/handlebars-helpers/lib/math", () => {
-  const actual = jest.requireActual("@budibase/handlebars-helpers/lib/math")
-
-  return {
-    ...actual,
-    random: () => 10,
-  }
-})
-jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
-  const actual = jest.requireActual("@budibase/handlebars-helpers/lib/uuid")
-
-  return {
-    ...actual,
-    uuid: () => "f34ebc66-93bd-4f7c-b79b-92b5569138bc",
-  }
-})
-
+import { validate as isValidUUID } from "uuid"
 import { processStringSync, encodeJSBinding } from "@budibase/string-templates"
 
 const { runJsHelpersTests } = require("@budibase/string-templates/test/utils")
@@ -27,7 +9,7 @@ import TestConfiguration from "../../tests/utilities/TestConfiguration"
 
 tk.freeze("2021-01-21T12:00:00")
 
-describe("jsRunner", () => {
+describe("jsRunner (using isolated-vm)", () => {
   const config = new TestConfiguration()
 
   beforeAll(async () => {
@@ -36,6 +18,10 @@ describe("jsRunner", () => {
     await config.init()
   })
 
+  afterAll(() => {
+    config.end()
+  })
+
   const processJS = (js: string, context?: object) => {
     return config.doInContext(config.getAppId(), async () =>
       processStringSync(encodeJSBinding(js), context || {})
@@ -47,10 +33,14 @@ describe("jsRunner", () => {
     expect(output).toBe(3)
   })
 
-  // TODO This should be reenabled when running on isolated-vm
-  it.skip("should prevent sandbox escape", async () => {
+  it("it can execute sloppy javascript", async () => {
+    const output = await processJS(`a=2;b=3;return a + b`)
+    expect(output).toBe(5)
+  })
+
+  it("should prevent sandbox escape", async () => {
     const output = await processJS(
-      `return this.constructor.constructor("return process")()`
+      `return this.constructor.constructor("return process.env")()`
     )
     expect(output).toBe("Error while executing JS")
   })
@@ -58,26 +48,26 @@ describe("jsRunner", () => {
   describe("helpers", () => {
     runJsHelpersTests({
       funcWrap: (func: any) => config.doInContext(config.getAppId(), func),
-      // testsToSkip: ["random", "uuid"],
+      testsToSkip: ["random", "uuid"],
     })
 
-    // describe("uuid", () => {
-    //   it("uuid helper returns a valid uuid", async () => {
-    //     const result = await processJS("return helpers.uuid()")
-    //     expect(result).toBeDefined()
-    //     expect(isValidUUID(result)).toBe(true)
-    //   })
-    // })
+    describe("uuid", () => {
+      it("uuid helper returns a valid uuid", async () => {
+        const result = await processJS("return helpers.uuid()")
+        expect(result).toBeDefined()
+        expect(isValidUUID(result)).toBe(true)
+      })
+    })
 
-    // describe("random", () => {
-    //   it("random helper returns a valid number", async () => {
-    //     const min = 1
-    //     const max = 8
-    //     const result = await processJS(`return helpers.random(${min}, ${max})`)
-    //     expect(result).toBeDefined()
-    //     expect(result).toBeGreaterThanOrEqual(min)
-    //     expect(result).toBeLessThanOrEqual(max)
-    //   })
-    // })
+    describe("random", () => {
+      it("random helper returns a valid number", async () => {
+        const min = 1
+        const max = 8
+        const result = await processJS(`return helpers.random(${min}, ${max})`)
+        expect(result).toBeDefined()
+        expect(result).toBeGreaterThanOrEqual(min)
+        expect(result).toBeLessThanOrEqual(max)
+      })
+    })
   })
 })

packages/server/src/jsRunner/vm/isolated-vm.ts

@@ -7,6 +7,7 @@ import querystring from "querystring"
 
 import { BundleType, loadBundle } from "../bundles"
 import { VM } from "@budibase/types"
+import environment from "../../environment"
 
 class ExecutionTimeoutError extends Error {
   constructor(message: string) {
@@ -15,35 +16,6 @@ class ExecutionTimeoutError extends Error {
   }
 }
 
-class ModuleHandler {
-  private modules: {
-    import: string
-    moduleKey: string
-    module: ivm.Module
-  }[] = []
-
-  private generateRandomKey = () => `i${crypto.randomUUID().replace(/-/g, "")}`
-
-  registerModule(module: ivm.Module, imports: string) {
-    this.modules.push({
-      moduleKey: this.generateRandomKey(),
-      import: imports,
-      module: module,
-    })
-  }
-
-  generateImports() {
-    return this.modules
-      .map(m => `import ${m.import} from "${m.moduleKey}"`)
-      .join(";")
-  }
-
-  getModule(key: string) {
-    const module = this.modules.find(m => m.moduleKey === key)
-    return module?.module
-  }
-}
-
 export class IsolatedVM implements VM {
   private isolate: ivm.Isolate
   private vm: ivm.Context
@@ -54,26 +26,29 @@ export class IsolatedVM implements VM {
   // By default the wrapper returns itself
   private codeWrapper: (code: string) => string = code => code
 
-  private moduleHandler = new ModuleHandler()
-
   private readonly resultKey = "results"
+  private runResultKey: string
 
   constructor({
     memoryLimit,
     invocationTimeout,
     isolateAccumulatedTimeout,
   }: {
-    memoryLimit: number
-    invocationTimeout: number
+    memoryLimit?: number
+    invocationTimeout?: number
     isolateAccumulatedTimeout?: number
-  }) {
+  } = {}) {
+    memoryLimit = memoryLimit || environment.JS_RUNNER_MEMORY_LIMIT
+    invocationTimeout = memoryLimit || 1000
+
     this.isolate = new ivm.Isolate({ memoryLimit })
     this.vm = this.isolate.createContextSync()
     this.jail = this.vm.global
     this.jail.setSync("global", this.jail.derefInto())
 
+    this.runResultKey = crypto.randomUUID()
     this.addToContext({
-      [this.resultKey]: { out: "" },
+      [this.resultKey]: { [this.runResultKey]: "" },
     })
 
     this.invocationTimeout = invocationTimeout
@@ -90,6 +65,10 @@ export class IsolatedVM implements VM {
       escape: querystring.escape,
     })
 
+    const cryptoModule = this.registerCallbacks({
+      randomUUID: crypto.randomUUID,
+    })
+
     this.addToContext({
       helpersStripProtocol: new ivm.Callback((str: string) => {
         var parsed = url.parse(str) as any
@@ -98,34 +77,23 @@ export class IsolatedVM implements VM {
       }),
     })
 
-    const injectedRequire = `const require=function req(val) {
+    const injectedRequire = `require=function req(val) {
         switch (val) {
             case "url": return ${urlModule};
             case "querystring": return ${querystringModule};
+            case "crypto": return ${cryptoModule};
         }
       }`
     const helpersSource = loadBundle(BundleType.HELPERS)
-    const helpersModule = this.isolate.compileModuleSync(
-      `${injectedRequire};${helpersSource}`
+    const script = this.isolate.compileScriptSync(
+      `${injectedRequire};${helpersSource};helpers=helpers.default`
     )
 
-    helpersModule.instantiateSync(this.vm, specifier => {
-      if (specifier === "crypto") {
-        const cryptoModule = this.registerCallbacks({
-          randomUUID: crypto.randomUUID,
-        })
-        const module = this.isolate.compileModuleSync(
-          `export default ${cryptoModule}`
-        )
-        module.instantiateSync(this.vm, specifier => {
-          throw new Error(`No imports allowed. Required: ${specifier}`)
-        })
-        return module
-      }
-      throw new Error(`No imports allowed. Required: ${specifier}`)
+    script.runSync(this.vm, { timeout: this.invocationTimeout, release: false })
+    new Promise(() => {
+      script.release()
     })
 
-    this.moduleHandler.registerModule(helpersModule, "helpers")
     return this
   }
 
@@ -147,9 +115,9 @@ export class IsolatedVM implements VM {
     // 4. Stringify the result in order to convert the result from BSON to json
     this.codeWrapper = code =>
       `(function(){
-            const data = deserialize(bsonData, { validation: { utf8: false } }).data;
+            const data = bson.deserialize(bsonData, { validation: { utf8: false } }).data;
             const result = ${code}
-            return toJson(result);
+            return bson.toJson(result);
         })();`
 
     const bsonSource = loadBundle(BundleType.BSON)
@@ -169,7 +137,7 @@ export class IsolatedVM implements VM {
     })
 
     // "Polyfilling" text decoder. `bson.deserialize` requires decoding. We are creating a bridge function so we don't need to inject the full library
-    const textDecoderPolyfill = class TextDecoder {
+    const textDecoderPolyfill = class TextDecoderMock {
       constructorArgs
 
       constructor(...constructorArgs: any) {
@@ -183,16 +151,18 @@ export class IsolatedVM implements VM {
           functionArgs: input,
         })
       }
-    }.toString()
-    const bsonModule = this.isolate.compileModuleSync(
+    }
+      .toString()
+      .replace(/TextDecoderMock/, "TextDecoder")
+
+    const script = this.isolate.compileScriptSync(
       `${textDecoderPolyfill};${bsonSource}`
     )
-    bsonModule.instantiateSync(this.vm, specifier => {
-      throw new Error(`No imports allowed. Required: ${specifier}`)
+    script.runSync(this.vm, { timeout: this.invocationTimeout, release: false })
+    new Promise(() => {
+      script.release()
     })
 
-    this.moduleHandler.registerModule(bsonModule, "{deserialize, toJson}")
-
     return this
   }
 
@@ -206,25 +176,18 @@ export class IsolatedVM implements VM {
       }
     }
 
-    code = `${this.moduleHandler.generateImports()};results.out=${this.codeWrapper(
-      code
-    )};`
+    code = `results['${this.runResultKey}']=${this.codeWrapper(code)}`
 
-    const script = this.isolate.compileModuleSync(code)
+    const script = this.isolate.compileScriptSync(code)
 
-    script.instantiateSync(this.vm, specifier => {
-      const module = this.moduleHandler.getModule(specifier)
-      if (module) {
-        return module
-      }
-
-      throw new Error(`"${specifier}" import not allowed`)
+    script.runSync(this.vm, { timeout: this.invocationTimeout, release: false })
+    new Promise(() => {
+      script.release()
     })
 
-    script.evaluateSync({ timeout: this.invocationTimeout })
-
+    // We can't rely on the script run result as it will not work for non-transferable values
     const result = this.getFromContext(this.resultKey)
-    return result.out
+    return result[this.runResultKey]
   }
 
   private registerCallbacks(functions: Record<string, any>) {
@@ -264,7 +227,10 @@ export class IsolatedVM implements VM {
   private getFromContext(key: string) {
     const ref = this.vm.global.getSync(key, { reference: true })
     const result = ref.copySync()
-    ref.release()
+
+    new Promise(() => {
+      ref.release()
+    })
     return result
   }
 }

packages/server/src/threads/query.ts

@@ -7,18 +7,20 @@ import {
   QueryVariable,
   QueryResponse,
 } from "./definitions"
-import { VM2 } from "../jsRunner/vm"
+import { IsolatedVM, VM2 } from "../jsRunner/vm"
 import { getIntegration } from "../integrations"
 import { processStringSync } from "@budibase/string-templates"
 import { context, cache, auth } from "@budibase/backend-core"
 import { getGlobalIDFromUserMetadataID } from "../db/utils"
 import sdk from "../sdk"
 import { cloneDeep } from "lodash/fp"
-import { Datasource, Query, SourceName } from "@budibase/types"
+import { Datasource, Query, SourceName, VM } from "@budibase/types"
 
 import { isSQL } from "../integrations/utils"
 import { interpolateSQL } from "../integrations/queries/sql"
 
+const USE_ISOLATED_VM = true
+
 class QueryRunner {
   datasource: Datasource
   queryVerb: string
@@ -127,10 +129,25 @@ class QueryRunner {
 
     // transform as required
     if (transformer) {
-      const runner = new VM2({
-        data: rows,
-        params: enrichedParameters,
-      })
+      let runner: VM
+      if (!USE_ISOLATED_VM) {
+        runner = new VM2({
+          data: rows,
+          params: enrichedParameters,
+        })
+      } else {
+        transformer = `(function(){\n${transformer}\n})();`
+        let isolatedVm = new IsolatedVM().withContext({
+          data: rows,
+          params: enrichedParameters,
+        })
+        if (datasource.source === SourceName.MONGODB) {
+          isolatedVm = isolatedVm.withParsingBson(rows)
+        }
+
+        runner = isolatedVm
+      }
+
       rows = runner.execute(transformer)
     }
 

scripts/build.js

@@ -49,7 +49,6 @@ function runBuild(entry, outfile) {
     preserveSymlinks: true,
     loader: {
       ".svelte": "copy",
-      ".ivm.bundle.js": "text",
     },
     metafile: true,
     external: [
@@ -70,7 +69,7 @@ function runBuild(entry, outfile) {
     platform: "node",
     outfile,
   }).then(result => {
-    glob(`${process.cwd()}/src/**/*.hbs`, {}, (err, files) => {
+    glob(`${process.cwd()}/src/**/*.{hbs,ivm.bundle.js}`, {}, (err, files) => {
       for (const file of files) {
         fs.copyFileSync(file, `${process.cwd()}/dist/${path.basename(file)}`)
       }