Merge pull request #12938 from Budibase/revert-12934-revert-12930-revert-12929-revert-12769-isolated-vm

Migrate from `vm` to `isolated-vm`

.eslintignore

@@ -10,4 +10,5 @@ packages/builder/.routify
 packages/sdk/sdk
 packages/account-portal/packages/server/build
 packages/account-portal/packages/ui/.routify
-packages/account-portal/packages/ui/build
+packages/account-portal/packages/ui/build
+**/*.ivm.bundle.js

.prettierignore

@@ -11,4 +11,5 @@ packages/sdk/sdk
 packages/pro/coverage
 packages/account-portal/packages/ui/build
 packages/account-portal/packages/ui/.routify
-packages/account-portal/packages/server/build
+packages/account-portal/packages/server/build
+**/*.ivm.bundle.js

hosting/single/Dockerfile

@@ -124,6 +124,8 @@ HEALTHCHECK --interval=15s --timeout=15s --start-period=45s CMD "/healthcheck.sh
 
 # must set this just before running
 ENV NODE_ENV=production
+# this is required for isolated-vm to work on Node 20+
+ENV NODE_OPTIONS="--no-node-snapshot"
 WORKDIR /
 
 CMD ["./runner.sh"]

package.json

@@ -97,7 +97,17 @@
     "@budibase/backend-core": "0.0.0",
     "@budibase/shared-core": "0.0.0",
     "@budibase/string-templates": "0.0.0",
-    "@budibase/types": "0.0.0"
+    "@budibase/types": "0.0.0",
+    "tough-cookie": "4.1.3",
+    "node-fetch": "2.6.7",
+    "semver": "7.5.3",
+    "http-cache-semantics": "4.1.1",
+    "msgpackr": "1.10.1",
+    "axios": "1.6.3",
+    "xml2js": "0.6.2",
+    "unset-value": "2.0.1",
+    "got": "13.0.0",
+    "passport": "0.6.0"
   },
   "engines": {
     "node": ">=20.0.0 <21.0.0"

packages/backend-core/package.json

@@ -25,7 +25,7 @@
     "@budibase/pouchdb-replication-stream": "1.2.10",
     "@budibase/shared-core": "0.0.0",
     "@budibase/types": "0.0.0",
-    "@techpass/passport-openidconnect": "0.3.2",
+    "@govtechsg/passport-openidconnect": "^1.0.2",
     "aws-cloudfront-sign": "3.0.2",
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.1.0",
@@ -37,7 +37,7 @@
     "ioredis": "5.3.2",
     "joi": "17.6.0",
     "jsonwebtoken": "9.0.2",
-    "koa-passport": "4.1.4",
+    "koa-passport": "^6.0.0",
     "koa-pino-logger": "4.0.0",
     "lodash": "4.17.21",
     "node-fetch": "2.6.7",
@@ -52,9 +52,9 @@
     "redlock": "4.2.0",
     "rotating-file-stream": "3.1.0",
     "sanitize-s3-objectkey": "0.0.1",
-    "semver": "7.3.7",
+    "semver": "^7.5.4",
     "tar-fs": "2.1.1",
-    "uuid": "8.3.2"
+    "uuid": "^8.3.2"
   },
   "devDependencies": {
     "@shopify/jest-koa-mocks": "5.1.1",

packages/backend-core/src/context/types.ts

@@ -1,5 +1,4 @@
-import { IdentityContext } from "@budibase/types"
+import { IdentityContext, VM } from "@budibase/types"
-import { ExecutionTimeTracker } from "../timers"
 
 // keep this out of Budibase types, don't want to expose context info
 export type ContextMap = {
@@ -10,5 +9,5 @@ export type ContextMap = {
   isScim?: boolean
   automationId?: string
   isMigrating?: boolean
-  jsExecutionTracker?: ExecutionTimeTracker
+  vm?: VM
 }

packages/backend-core/src/timers/timers.ts

@@ -20,41 +20,3 @@ export function cleanup() {
   }
   intervals = []
 }
-
-export class ExecutionTimeoutError extends Error {
-  public readonly name = "ExecutionTimeoutError"
-}
-
-export class ExecutionTimeTracker {
-  static withLimit(limitMs: number) {
-    return new ExecutionTimeTracker(limitMs)
-  }
-
-  constructor(readonly limitMs: number) {}
-
-  private totalTimeMs = 0
-
-  track<T>(f: () => T): T {
-    this.checkLimit()
-    const start = process.hrtime.bigint()
-    try {
-      return f()
-    } finally {
-      const end = process.hrtime.bigint()
-      this.totalTimeMs += Number(end - start) / 1e6
-      this.checkLimit()
-    }
-  }
-
-  get elapsedMS() {
-    return this.totalTimeMs
-  }
-
-  checkLimit() {
-    if (this.totalTimeMs > this.limitMs) {
-      throw new ExecutionTimeoutError(
-        `Execution time limit of ${this.limitMs}ms exceeded: ${this.totalTimeMs}ms`
-      )
-    }
-  }
-}

packages/bbui/src/Form/Core/EnvDropdown.svelte

@@ -184,7 +184,7 @@
       {#if environmentVariablesEnabled}
         <div on:click={() => showModal()} class="add-variable">
           <svg
-            class="spectrum-Icon spectrum-Icon--sizeS "
+            class="spectrum-Icon spectrum-Icon--sizeS"
             focusable="false"
             aria-hidden="true"
           >
@@ -195,7 +195,7 @@
       {:else}
         <div on:click={() => handleUpgradePanel()} class="add-variable">
           <svg
-            class="spectrum-Icon spectrum-Icon--sizeS "
+            class="spectrum-Icon spectrum-Icon--sizeS"
             focusable="false"
             aria-hidden="true"
           >

packages/builder/src/components/integration/QueryViewer.svelte

@@ -41,6 +41,7 @@
 
   let autoSchema = {}
   let rows = []
+  let keys = {}
 
   const parseQuery = query => {
     modified = false
@@ -143,8 +144,20 @@
   const handleScroll = e => {
     scrolling = e.target.scrollTop !== 0
   }
+
+  async function handleKeyDown(evt) {
+    keys[evt.key] = true
+    if ((keys["Meta"] || keys["Control"]) && keys["Enter"]) {
+      await runQuery({ suppressErrors: false })
+    }
+  }
+
+  function handleKeyUp(evt) {
+    delete keys[evt.key]
+  }
 </script>
 
+<svelte:window on:keydown={handleKeyDown} on:keyup={handleKeyUp} />
 <QueryViewerSavePromptModal
   checkIsModified={() => checkIsModified(newQuery)}
   attemptSave={() => runQuery({ suppressErrors: false }).then(saveQuery)}

packages/builder/src/components/integration/QueryViewerSidePanel/JSONPanel.svelte

@@ -1,7 +1,20 @@
 <script>
   export let data
+  export let maxRowsToDisplay = 5
 
-  $: string = JSON.stringify(data || {}, null, 2)
+  let string
+  $: {
+    string = JSON.stringify(data || {}, null, 2)
+    if (Array.isArray(data) && data.length > maxRowsToDisplay) {
+      string = JSON.stringify(data.slice(0, maxRowsToDisplay) || {}, null, 2)
+
+      // Display '...' at the end of the array
+      string = string.replace(
+        /(}\n])/,
+        `},\n  ...${data.length - maxRowsToDisplay} further items\n]`
+      )
+    }
+  }
 </script>
 
 <textarea class="json" disabled value={string} />

packages/builder/src/components/integration/QueryViewerSidePanel/PreviewPanel.svelte

@@ -4,13 +4,17 @@
 
   export let schema = {}
   export let rows = []
+  export let maxRowsToDisplay = 5
 
-  $: rowsCopy = cloneDeep(rows)
+  let rowsToDisplay
+  $: rowsToDisplay = [...cloneDeep(rows).slice(0, maxRowsToDisplay)]
+
+  $: additionalRows = rows.length - maxRowsToDisplay
 
   // Cast field in query preview response to number if specified by schema
   $: {
-    for (let i = 0; i < rowsCopy.length; i++) {
+    for (let i = 0; i < rowsToDisplay.length; i++) {
-      let row = rowsCopy[i]
+      let row = rowsToDisplay[i]
       for (let fieldName of Object.keys(schema)) {
         if (schema[fieldName] === "number" && !isNaN(Number(row[fieldName]))) {
           row[fieldName] = Number(row[fieldName])
@@ -23,11 +27,27 @@
 </script>
 
 <div class="table">
-  <Table {schema} data={rowsCopy} allowEditing={false} />
+  <Table {schema} data={rowsToDisplay} allowEditing={false} />
+  {#if additionalRows > 0}
+    <div class="show-more">
+      ...{additionalRows} further items
+    </div>
+  {/if}
 </div>
 
 <style>
-  .table :global(.spectrum-Table-cell) {
+  .table :global(.spectrum-Table-cell),
+  .show-more {
     min-width: 100px;
   }
+
+  .show-more {
+    display: flex;
+    padding: 16px;
+    justify-content: center;
+
+    background-color: var(--spectrum-global-color-gray-50);
+    border: 1px solid var(--spectrum-alias-border-color-mid);
+    border-top: 0;
+  }
 </style>

packages/builder/src/components/integration/QueryViewerSidePanel/index.svelte

@@ -41,7 +41,7 @@
   </div>
   <div class="content">
     {#if activeTab === "JSON"}
-      <JSONPanel data={rows[0] || {}} />
+      <JSONPanel data={rows?.length === 1 ? rows[0] : rows || {}} />
     {:else if activeTab === "Schema"}
       <SchemaPanel {onSchemaChange} {schema} />
     {:else}

packages/server/Dockerfile

@@ -82,6 +82,8 @@ EXPOSE 4001
 # due to this causing yarn to stop installing dev dependencies
 # which are actually needed to get this environment up and running
 ENV NODE_ENV=production
+# this is required for isolated-vm to work on Node 20+
+ENV NODE_OPTIONS="--no-node-snapshot"
 ENV CLUSTER_MODE=${CLUSTER_MODE}
 ENV TOP_LEVEL_PATH=/app
 

packages/server/nodemon.json

@@ -8,6 +8,6 @@
     "../string-templates"
   ],
   "ext": "js,ts,json,svelte",
-  "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../*/dist/**/*"],
+  "ignore": ["**/*.spec.ts", "**/*.spec.js", "../*/dist/**/*"],
-  "exec": "yarn build && node ./dist/index.js"
+  "exec": "yarn build && node --no-node-snapshot ./dist/index.js"
 }

packages/server/package.json

@@ -13,8 +13,12 @@
     "build": "node ./scripts/build.js",
     "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../yarn.lock ./dist/",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null",
+    "build:isolated-vm-lib:string-templates": "esbuild --minify --bundle src/jsRunner/bundles/index-helpers.ts --outfile=src/jsRunner/bundles/index-helpers.ivm.bundle.js --platform=node --format=esm --external:handlebars",
+    "build:isolated-vm-lib:bson": "esbuild --minify --bundle src/jsRunner/bundles/bsonPackage.ts --outfile=src/jsRunner/bundles/bson.ivm.bundle.js --platform=node --format=esm",
+    "build:isolated-vm-libs": "yarn build:isolated-vm-lib:string-templates && yarn build:isolated-vm-lib:bson",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
+    "jest": "NODE_OPTIONS=\"--no-node-snapshot $NODE_OPTIONS\" jest",
     "test": "bash scripts/test.sh",
     "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
     "test:watch": "jest --watch",
@@ -49,8 +53,8 @@
     "@budibase/shared-core": "0.0.0",
     "@budibase/string-templates": "0.0.0",
     "@budibase/types": "0.0.0",
-    "@bull-board/api": "3.7.0",
+    "@bull-board/api": "5.10.2",
-    "@bull-board/koa": "3.9.4",
+    "@bull-board/koa": "5.10.2",
     "@elastic/elasticsearch": "7.10.0",
     "@google-cloud/firestore": "6.8.0",
     "@koa/router": "8.0.8",
@@ -72,7 +76,8 @@
     "google-auth-library": "7.12.0",
     "google-spreadsheet": "3.2.0",
     "ioredis": "5.3.2",
-    "jimp": "0.16.1",
+    "isolated-vm": "^4.7.2",
+    "jimp": "0.22.10",
     "joi": "17.6.0",
     "js-yaml": "4.1.0",
     "jsonschema": "1.4.0",
@@ -85,7 +90,7 @@
     "koa2-ratelimit": "1.1.1",
     "lodash": "4.17.21",
     "memorystream": "0.3.1",
-    "mongodb": "5.7",
+    "mongodb": "^6.3.0",
     "mssql": "10.0.1",
     "mysql2": "3.5.2",
     "node-fetch": "2.6.7",
@@ -104,9 +109,10 @@
     "svelte": "^3.49.0",
     "tar": "6.1.15",
     "to-json-schema": "0.2.5",
-    "uuid": "3.3.2",
+    "undici": "^6.0.1",
+    "undici-types": "^6.0.1",
+    "uuid": "^8.3.2",
     "validate.js": "0.13.1",
-    "vm2": "^3.9.19",
     "worker-farm": "1.7.0",
     "xml2js": "0.5.0"
   },
@@ -129,6 +135,7 @@
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.14",
     "@types/tar": "6.1.5",
+    "@types/uuid": "8.3.4",
     "apidoc": "0.50.4",
     "copyfiles": "2.4.1",
     "docker-compose": "0.23.17",

packages/server/scripts/integrations/mongo/docker-compose.yaml

@@ -0,0 +1,24 @@
+# Use root/example as user/password credentials
+version: "3.1"
+
+services:
+  mongo:
+    image: mongo
+    restart: always
+    ports:
+      - 27017:27017
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root
+      MONGO_INITDB_ROOT_PASSWORD: example
+
+  mongo-express:
+    image: mongo-express
+    restart: always
+    ports:
+      - 8081:8081
+    environment:
+      ME_CONFIG_MONGODB_ADMINUSERNAME: root
+      ME_CONFIG_MONGODB_ADMINPASSWORD: example
+      ME_CONFIG_MONGODB_AUTH_USERNAME: admin
+      ME_CONFIG_MONGODB_AUTH_PASSWORD: pass
+      ME_CONFIG_MONGODB_URL: mongodb://root:example@mongo:27017/

packages/server/scripts/test.sh

@@ -3,12 +3,12 @@ set -e
 
 if [[ -n $CI ]]
 then
-  # Running in ci, where resources are limited
+  export NODE_OPTIONS="--max-old-space-size=4096 --no-node-snapshot"
-  export NODE_OPTIONS="--max-old-space-size=4096"
   echo "jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail $@"
   jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail $@
 else
   # --maxWorkers performs better in development
+  export NODE_OPTIONS="--no-node-snapshot"
   echo "jest --coverage --maxWorkers=2 --forceExit $@"
   jest --coverage --maxWorkers=2 --forceExit $@
 fi

packages/server/src/api/controllers/script.ts

@@ -1,12 +1,12 @@
 import ScriptRunner from "../../utilities/scriptRunner"
-import { BBContext } from "@budibase/types"
+import { Ctx } from "@budibase/types"
 
-export async function execute(ctx: BBContext) {
+export async function execute(ctx: Ctx) {
   const { script, context } = ctx.request.body
   const runner = new ScriptRunner(script, context)
   ctx.body = runner.execute()
 }
 
-export async function save(ctx: BBContext) {
+export async function save(ctx: Ctx) {
   ctx.throw(501, "Not currently implemented")
 }

packages/server/src/api/controllers/static/index.ts

@@ -3,7 +3,7 @@ import { InvalidFileExtensions } from "@budibase/shared-core"
 require("svelte/register")
 
 import { join } from "../../../utilities/centralPath"
-import uuid from "uuid"
+import * as uuid from "uuid"
 import { ObjectStoreBuckets } from "../../../constants"
 import { processString } from "@budibase/string-templates"
 import {

packages/server/src/api/routes/tests/row.spec.ts

@@ -2031,7 +2031,7 @@ describe.each([
 
   describe("Formula JS protection", () => {
     it("should time out JS execution if a single cell takes too long", async () => {
-      await config.withEnv({ JS_PER_EXECUTION_TIME_LIMIT_MS: 20 }, async () => {
+      await config.withEnv({ JS_PER_INVOCATION_TIMEOUT_MS: 20 }, async () => {
         const js = Buffer.from(
           `
               let i = 0;
@@ -2071,8 +2071,8 @@ describe.each([
     it("should time out JS execution if a multiple cells take too long", async () => {
       await config.withEnv(
         {
-          JS_PER_EXECUTION_TIME_LIMIT_MS: 20,
+          JS_PER_INVOCATION_TIMEOUT_MS: 20,
-          JS_PER_REQUEST_TIME_LIMIT_MS: 40,
+          JS_PER_REQUEST_TIMEOUT_MS: 40,
         },
         async () => {
           const js = Buffer.from(

packages/server/src/api/routes/tests/table.spec.ts

@@ -16,7 +16,7 @@ import {
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 import sdk from "../../../sdk"
-import uuid from "uuid"
+import * as uuid from "uuid"
 
 const { basicTable } = setup.structures
 

packages/server/src/automations/tests/executeScript.spec.js

@@ -9,34 +9,40 @@ describe("test the execute script action", () => {
   afterAll(setup.afterAll)
 
   it("should be able to execute a script", async () => {
-    let res = await setup.runStep(
+    const res = await setup.runStep(setup.actions.EXECUTE_SCRIPT.stepId, {
-      setup.actions.EXECUTE_SCRIPT.stepId,
+      code: "return 1 + 1",
-      (inputs = {
+    })
-        code: "return 1 + 1",
-      })
-    )
     expect(res.value).toEqual(2)
     expect(res.success).toEqual(true)
   })
 
   it("should handle a null value", async () => {
-    let res = await setup.runStep(
+    const res = await setup.runStep(setup.actions.EXECUTE_SCRIPT.stepId, {
-      setup.actions.EXECUTE_SCRIPT.stepId,
+      code: null,
-      (inputs = {
+    })
-        code: null,
-      })
-    )
     expect(res.response.message).toEqual("Invalid inputs")
     expect(res.success).toEqual(false)
   })
 
-  it("should be able to handle an error gracefully", async () => {
+  it("should be able to get a value from context", async () => {
-    let res = await setup.runStep(
+    const res = await setup.runStep(
       setup.actions.EXECUTE_SCRIPT.stepId,
-      (inputs = {
+      {
-        code: "return something.map(x => x.name)",
+        code: "return steps.map(d => d.value)",
-      })
+      },
+      {
+        steps: [{ value: 0 }, { value: 1 }],
+      }
     )
+    expect(res.value).toEqual([0, 1])
+    expect(res.response).toBeUndefined()
+    expect(res.success).toEqual(true)
+  })
+
+  it("should be able to handle an error gracefully", async () => {
+    const res = await setup.runStep(setup.actions.EXECUTE_SCRIPT.stepId, {
+      code: "return something.map(x => x.name)",
+    })
     expect(res.response).toEqual("ReferenceError: something is not defined")
     expect(res.success).toEqual(false)
   })

packages/server/src/db/newid.ts

@@ -1,4 +1,4 @@
-const { v4 } = require("uuid")
+import { v4 } from "uuid"
 
 export default function (): string {
   return v4().replace(/-/g, "")

packages/server/src/environment.ts

@@ -71,10 +71,10 @@ const environment = {
   SELF_HOSTED: process.env.SELF_HOSTED,
   HTTP_MB_LIMIT: process.env.HTTP_MB_LIMIT,
   FORKED_PROCESS_NAME: process.env.FORKED_PROCESS_NAME || "main",
-  JS_PER_EXECUTION_TIME_LIMIT_MS:
+  JS_PER_INVOCATION_TIMEOUT_MS:
-    parseIntSafe(process.env.JS_PER_EXECUTION_TIME_LIMIT_MS) || 1000,
+    parseIntSafe(process.env.JS_PER_INVOCATION_TIMEOUT_MS) || 1000,
-  JS_PER_REQUEST_TIME_LIMIT_MS: parseIntSafe(
+  JS_PER_REQUEST_TIMEOUT_MS: parseIntSafe(
-    process.env.JS_PER_REQUEST_TIME_LIMIT_MS
+    process.env.JS_PER_REQUEST_TIMEOUT_MS
   ),
   // old
   CLIENT_ID: process.env.CLIENT_ID,
@@ -95,6 +95,8 @@ const environment = {
   TOP_LEVEL_PATH:
     process.env.TOP_LEVEL_PATH || process.env.SERVER_TOP_LEVEL_PATH,
   APP_MIGRATION_TIMEOUT: parseIntSafe(process.env.APP_MIGRATION_TIMEOUT),
+  JS_RUNNER_MEMORY_LIMIT:
+    parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) || 64,
 }
 
 // threading can cause memory issues with node-ts in development

packages/server/src/integrations/mongodb.ts

@@ -21,7 +21,6 @@ import environment from "../environment"
 interface MongoDBConfig {
   connectionString: string
   db: string
-  tlsCertificateFile: string
   tlsCertificateKeyFile: string
   tlsCAFile: string
 }
@@ -320,16 +319,11 @@ const getSchema = () => {
   if (environment.SELF_HOSTED) {
     schema.datasource = {
       ...schema.datasource,
-      //@ts-ignore
+      // @ts-ignore
       tls: {
         type: DatasourceFieldType.FIELD_GROUP,
         display: "Configure SSL",
         fields: {
-          tlsCertificateFile: {
-            type: DatasourceFieldType.STRING,
-            required: false,
-            display: "Certificate file path",
-          },
           tlsCertificateKeyFile: {
             type: DatasourceFieldType.STRING,
             required: false,
@@ -356,7 +350,6 @@ class MongoIntegration implements IntegrationBase {
   constructor(config: MongoDBConfig) {
     this.config = config
     const options: MongoClientOptions = {
-      tlsCertificateFile: config.tlsCertificateFile || undefined,
       tlsCertificateKeyFile: config.tlsCertificateKeyFile || undefined,
       tlsCAFile: config.tlsCAFile || undefined,
     }
@@ -525,7 +518,10 @@ class MongoIntegration implements IntegrationBase {
           return await collection.findOneAndUpdate(
             findAndUpdateJson.filter,
             findAndUpdateJson.update,
-            findAndUpdateJson.options
+            {
+              ...findAndUpdateJson.options,
+              includeResultMetadata: true,
+            }
           )
         }
         case "count": {

packages/server/src/integrations/tests/mongo.spec.ts

@@ -221,6 +221,7 @@ describe("MongoDB Integration", () => {
     })
     expect(args[2]).toEqual({
       upsert: false,
+      includeResultMetadata: true,
     })
   })
 

packages/server/src/jsRunner.ts

@@ -1,61 +0,0 @@
-import vm from "vm"
-import env from "./environment"
-import { setJSRunner } from "@budibase/string-templates"
-import { context, timers } from "@budibase/backend-core"
-import tracer from "dd-trace"
-
-type TrackerFn = <T>(f: () => T) => T
-
-export function init() {
-  setJSRunner((js: string, ctx: vm.Context) => {
-    return tracer.trace("runJS", {}, span => {
-      const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
-      let track: TrackerFn = f => f()
-      if (perRequestLimit) {
-        const bbCtx = tracer.trace("runJS.getCurrentContext", {}, span =>
-          context.getCurrentContext()
-        )
-        if (bbCtx) {
-          if (!bbCtx.jsExecutionTracker) {
-            span?.addTags({
-              createdExecutionTracker: true,
-            })
-            bbCtx.jsExecutionTracker = tracer.trace(
-              "runJS.createExecutionTimeTracker",
-              {},
-              span => timers.ExecutionTimeTracker.withLimit(perRequestLimit)
-            )
-          }
-          span?.addTags({
-            js: {
-              limitMS: bbCtx.jsExecutionTracker.limitMs,
-              elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
-            },
-          })
-          // We call checkLimit() here to prevent paying the cost of creating
-          // a new VM context below when we don't need to.
-          tracer.trace("runJS.checkLimitAndBind", {}, span => {
-            bbCtx.jsExecutionTracker!.checkLimit()
-            track = bbCtx.jsExecutionTracker!.track.bind(
-              bbCtx.jsExecutionTracker
-            )
-          })
-        }
-      }
-
-      ctx = {
-        ...ctx,
-        alert: undefined,
-        setInterval: undefined,
-        setTimeout: undefined,
-      }
-
-      vm.createContext(ctx)
-      return track(() =>
-        vm.runInNewContext(js, ctx, {
-          timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
-        })
-      )
-    })
-  })
-}

packages/server/src/jsRunner/bundles/README.md

@@ -0,0 +1,15 @@
+# Bundles for isolated-vm
+
+[Isolated-vm](https://github.com/laverdet/isolated-vm) requires for us to have some libraries, such as string-templates helpers, built in a single file without external dependencies. These libraries are pretty much static. To avoid building this in every dev command, in every test command and in every pipeline, these libraries are already compiled and commited into the repo.
+
+## How are they consumed?
+
+These libaries are compiled with a special extension: .ivm.bundle.js. This extension is configured in [esbuild](/scripts/build.js) in order to not be bundled as javascript, and to be treated as a `string` instead. This will allow us to read it's context on runtime and inject it to `isolated-vm`.
+
+## How to update it?
+
+These libraries are pretty much static, but they might require some updates from time to time when something changes on the source code. In order to do this, we just need to run the following command and commit the updated bundles:
+
+```
+yarn build:isolated-vm-libs
+```

packages/server/src/jsRunner/bundles/bsonPackage.ts

@@ -0,0 +1,4 @@
+import { EJSON } from "bson"
+
+export { deserialize } from "bson"
+export const toJson = EJSON.deserialize

packages/server/src/jsRunner/bundles/index-helpers.ts

@@ -0,0 +1,12 @@
+const {
+  getJsHelperList,
+} = require("../../../../string-templates/src/helpers/list.js")
+
+const helpers = getJsHelperList()
+export default {
+  ...helpers,
+  // pointing stripProtocol to a unexisting function to be able to declare it on isolated-vm
+  // @ts-ignore
+  // eslint-disable-next-line no-undef
+  stripProtocol: helpersStripProtocol,
+}

packages/server/src/jsRunner/bundles/index.ts

@@ -0,0 +1,28 @@
+import { utils } from "@budibase/shared-core"
+import environment from "../../environment"
+import fs from "fs"
+
+export const enum BundleType {
+  HELPERS = "helpers",
+  BSON = "bson",
+}
+
+const bundleSourceCode = {
+  [BundleType.HELPERS]: "../bundles/index-helpers.ivm.bundle.js",
+  [BundleType.BSON]: "../bundles/bson.ivm.bundle.js",
+}
+
+export function loadBundle(type: BundleType) {
+  if (environment.isJest()) {
+    return fs.readFileSync(require.resolve(bundleSourceCode[type]), "utf-8")
+  }
+
+  switch (type) {
+    case BundleType.HELPERS:
+      return require("../bundles/index-helpers.ivm.bundle.js")
+    case BundleType.BSON:
+      return require("../bundles/bson.ivm.bundle.js")
+    default:
+      utils.unreachable(type)
+  }
+}

packages/server/src/jsRunner/index.ts

@@ -0,0 +1,42 @@
+import env from "../environment"
+import { setJSRunner, JsErrorTimeout } from "@budibase/string-templates"
+import tracer from "dd-trace"
+
+import { IsolatedVM } from "./vm"
+import { context } from "@budibase/backend-core"
+
+export function init() {
+  setJSRunner((js: string, ctx: Record<string, any>) => {
+    return tracer.trace("runJS", {}, span => {
+      try {
+        const bbCtx = context.getCurrentContext()!
+
+        let { vm } = bbCtx
+        if (!vm) {
+          // Can't copy the native helpers into the isolate. We just ignore them as they are handled properly from the helpersSource
+          const { helpers, ...ctxToPass } = ctx
+
+          vm = new IsolatedVM({
+            memoryLimit: env.JS_RUNNER_MEMORY_LIMIT,
+            invocationTimeout: env.JS_PER_INVOCATION_TIMEOUT_MS,
+            isolateAccumulatedTimeout: env.JS_PER_REQUEST_TIMEOUT_MS,
+          })
+            .withContext(ctxToPass)
+            .withHelpers()
+
+          bbCtx.vm = vm
+        }
+
+        const result = vm.execute(js)
+
+        return result
+      } catch (error: any) {
+        if (error.message === "Script execution timed out.") {
+          throw new JsErrorTimeout()
+        }
+
+        throw error
+      }
+    })
+  })
+}

packages/server/src/jsRunner/tests/jsRunner.spec.ts

@@ -0,0 +1,75 @@
+import { validate as isValidUUID } from "uuid"
+
+jest.mock("@budibase/handlebars-helpers/lib/math", () => {
+  const actual = jest.requireActual("@budibase/handlebars-helpers/lib/math")
+
+  return {
+    ...actual,
+    random: () => 10,
+  }
+})
+jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
+  const actual = jest.requireActual("@budibase/handlebars-helpers/lib/uuid")
+
+  return {
+    ...actual,
+    uuid: () => "f34ebc66-93bd-4f7c-b79b-92b5569138bc",
+  }
+})
+
+import { processStringSync, encodeJSBinding } from "@budibase/string-templates"
+
+const { runJsHelpersTests } = require("@budibase/string-templates/test/utils")
+
+import tk from "timekeeper"
+import { init } from ".."
+import TestConfiguration from "../../tests/utilities/TestConfiguration"
+
+tk.freeze("2021-01-21T12:00:00")
+
+describe("jsRunner", () => {
+  const config = new TestConfiguration()
+
+  beforeAll(async () => {
+    // Register js runner
+    init()
+    await config.init()
+  })
+
+  const processJS = (js: string, context?: object) => {
+    return config.doInContext(config.getAppId(), async () =>
+      processStringSync(encodeJSBinding(js), context || {})
+    )
+  }
+
+  it("it can run a basic javascript", async () => {
+    const output = await processJS(`return 1 + 2`)
+    expect(output).toBe(3)
+  })
+
+  describe("helpers", () => {
+    runJsHelpersTests({
+      funcWrap: (func: any) => config.doInContext(config.getAppId(), func),
+      testsToSkip: ["random", "uuid"],
+    })
+
+    describe("uuid", () => {
+      it("uuid helper returns a valid uuid", async () => {
+        const result = await processJS("return helpers.uuid()")
+        expect(result).toBeDefined()
+        expect(isValidUUID(result)).toBe(true)
+      })
+    })
+
+    describe("random", () => {
+      it("random helper returns a valid number", async () => {
+        const min = 1
+        const max = 8
+        const result = await processJS(`return helpers.random(${min}, ${max})`)
+        expect(result).toBeDefined()
+        expect(result).toBeGreaterThanOrEqual(min)
+        expect(result).toBeLessThanOrEqual(max)
+      })
+    })
+  })
+})

packages/server/src/jsRunner/vm/index.ts

@@ -0,0 +1,270 @@
+import ivm from "isolated-vm"
+import bson from "bson"
+
+import url from "url"
+import crypto from "crypto"
+import querystring from "querystring"
+
+import { BundleType, loadBundle } from "../bundles"
+import { VM } from "@budibase/types"
+
+class ExecutionTimeoutError extends Error {
+  constructor(message: string) {
+    super(message)
+    this.name = "ExecutionTimeoutError"
+  }
+}
+
+class ModuleHandler {
+  private modules: {
+    import: string
+    moduleKey: string
+    module: ivm.Module
+  }[] = []
+
+  private generateRandomKey = () => `i${crypto.randomUUID().replace(/-/g, "")}`
+
+  registerModule(module: ivm.Module, imports: string) {
+    this.modules.push({
+      moduleKey: this.generateRandomKey(),
+      import: imports,
+      module: module,
+    })
+  }
+
+  generateImports() {
+    return this.modules
+      .map(m => `import ${m.import} from "${m.moduleKey}"`)
+      .join(";")
+  }
+
+  getModule(key: string) {
+    const module = this.modules.find(m => m.moduleKey === key)
+    return module?.module
+  }
+}
+
+export class IsolatedVM implements VM {
+  private isolate: ivm.Isolate
+  private vm: ivm.Context
+  private jail: ivm.Reference
+  private invocationTimeout: number
+  private isolateAccumulatedTimeout?: number
+
+  // By default the wrapper returns itself
+  private codeWrapper: (code: string) => string = code => code
+
+  private moduleHandler = new ModuleHandler()
+
+  private readonly resultKey = "results"
+
+  constructor({
+    memoryLimit,
+    invocationTimeout,
+    isolateAccumulatedTimeout,
+  }: {
+    memoryLimit: number
+    invocationTimeout: number
+    isolateAccumulatedTimeout?: number
+  }) {
+    this.isolate = new ivm.Isolate({ memoryLimit })
+    this.vm = this.isolate.createContextSync()
+    this.jail = this.vm.global
+    this.jail.setSync("global", this.jail.derefInto())
+
+    this.addToContext({
+      [this.resultKey]: { out: "" },
+    })
+
+    this.invocationTimeout = invocationTimeout
+    this.isolateAccumulatedTimeout = isolateAccumulatedTimeout
+  }
+
+  withHelpers() {
+    const urlModule = this.registerCallbacks({
+      resolve: url.resolve,
+      parse: url.parse,
+    })
+
+    const querystringModule = this.registerCallbacks({
+      escape: querystring.escape,
+    })
+
+    this.addToContext({
+      helpersStripProtocol: new ivm.Callback((str: string) => {
+        var parsed = url.parse(str) as any
+        parsed.protocol = ""
+        return parsed.format()
+      }),
+    })
+
+    const injectedRequire = `const require=function req(val) {
+        switch (val) {
+            case "url": return ${urlModule};
+            case "querystring": return ${querystringModule};
+        }
+      }`
+    const helpersSource = loadBundle(BundleType.HELPERS)
+    const helpersModule = this.isolate.compileModuleSync(
+      `${injectedRequire};${helpersSource}`
+    )
+
+    helpersModule.instantiateSync(this.vm, specifier => {
+      if (specifier === "crypto") {
+        const cryptoModule = this.registerCallbacks({
+          randomUUID: crypto.randomUUID,
+        })
+        const module = this.isolate.compileModuleSync(
+          `export default ${cryptoModule}`
+        )
+        module.instantiateSync(this.vm, specifier => {
+          throw new Error(`No imports allowed. Required: ${specifier}`)
+        })
+        return module
+      }
+      throw new Error(`No imports allowed. Required: ${specifier}`)
+    })
+
+    this.moduleHandler.registerModule(helpersModule, "helpers")
+    return this
+  }
+
+  withContext(context: Record<string, any>) {
+    this.addToContext(context)
+
+    return this
+  }
+
+  withParsingBson(data: any) {
+    this.addToContext({
+      bsonData: bson.BSON.serialize({ data }),
+    })
+
+    // If we need to parse bson, we follow the next steps:
+    // 1. Serialise the data from potential BSON to buffer before passing it to the isolate
+    // 2. Deserialise the data within the isolate, to get the original data
+    // 3. Process script
+    // 4. Stringify the result in order to convert the result from BSON to json
+    this.codeWrapper = code =>
+      `(function(){
+            const data = deserialize(bsonData, { validation: { utf8: false } }).data;
+            const result = ${code}
+            return toJson(result);
+        })();`
+
+    const bsonSource = loadBundle(BundleType.BSON)
+
+    this.addToContext({
+      textDecoderCb: new ivm.Callback(
+        (args: {
+          constructorArgs: any
+          functionArgs: Parameters<InstanceType<typeof TextDecoder>["decode"]>
+        }) => {
+          const result = new TextDecoder(...args.constructorArgs).decode(
+            ...args.functionArgs
+          )
+          return result
+        }
+      ),
+    })
+
+    // "Polyfilling" text decoder. `bson.deserialize` requires decoding. We are creating a bridge function so we don't need to inject the full library
+    const textDecoderPolyfill = class TextDecoder {
+      constructorArgs
+
+      constructor(...constructorArgs: any) {
+        this.constructorArgs = constructorArgs
+      }
+
+      decode(...input: any) {
+        // @ts-ignore
+        return textDecoderCb({
+          constructorArgs: this.constructorArgs,
+          functionArgs: input,
+        })
+      }
+    }.toString()
+    const bsonModule = this.isolate.compileModuleSync(
+      `${textDecoderPolyfill};${bsonSource}`
+    )
+    bsonModule.instantiateSync(this.vm, specifier => {
+      throw new Error(`No imports allowed. Required: ${specifier}`)
+    })
+
+    this.moduleHandler.registerModule(bsonModule, "{deserialize, toJson}")
+
+    return this
+  }
+
+  execute(code: string): any {
+    if (this.isolateAccumulatedTimeout) {
+      const cpuMs = Number(this.isolate.cpuTime) / 1e6
+      if (cpuMs > this.isolateAccumulatedTimeout) {
+        throw new ExecutionTimeoutError(
+          `CPU time limit exceeded (${cpuMs}ms > ${this.isolateAccumulatedTimeout}ms)`
+        )
+      }
+    }
+
+    code = `${this.moduleHandler.generateImports()};results.out=${this.codeWrapper(
+      code
+    )};`
+
+    const script = this.isolate.compileModuleSync(code)
+
+    script.instantiateSync(this.vm, specifier => {
+      const module = this.moduleHandler.getModule(specifier)
+      if (module) {
+        return module
+      }
+
+      throw new Error(`"${specifier}" import not allowed`)
+    })
+
+    script.evaluateSync({ timeout: this.invocationTimeout })
+
+    const result = this.getFromContext(this.resultKey)
+    return result.out
+  }
+
+  private registerCallbacks(functions: Record<string, any>) {
+    const libId = crypto.randomUUID().replace(/-/g, "")
+
+    const x: Record<string, string> = {}
+    for (const [funcName, func] of Object.entries(functions)) {
+      const key = `f${libId}${funcName}cb`
+      x[funcName] = key
+
+      this.addToContext({
+        [key]: new ivm.Callback((...params: any[]) => (func as any)(...params)),
+      })
+    }
+
+    const mod =
+      `{` +
+      Object.entries(x)
+        .map(([key, func]) => `${key}: ${func}`)
+        .join() +
+      "}"
+    return mod
+  }
+
+  private addToContext(context: Record<string, any>) {
+    for (let key in context) {
+      const value = context[key]
+      this.jail.setSync(
+        key,
+        typeof value === "function"
+          ? value
+          : new ivm.ExternalCopy(value).copyInto({ release: true })
+      )
+    }
+  }
+
+  private getFromContext(key: string) {
+    const ref = this.vm.global.getSync(key, { reference: true })
+    const result = ref.copySync()
+    ref.release()
+    return result
+  }
+}

packages/server/src/threads/definitions.ts

@@ -1,10 +1,10 @@
-import { QuerySchema, Row } from "@budibase/types"
+import { Datasource, QuerySchema, Row } from "@budibase/types"
 
 export type WorkerCallback = (error: any, response?: any) => void
 
 export interface QueryEvent {
   appId?: string
-  datasource: any
+  datasource: Datasource
   queryVerb: string
   fields: { [key: string]: any }
   parameters: { [key: string]: any }

packages/server/src/threads/query.ts

@@ -14,13 +14,13 @@ import { context, cache, auth } from "@budibase/backend-core"
 import { getGlobalIDFromUserMetadataID } from "../db/utils"
 import sdk from "../sdk"
 import { cloneDeep } from "lodash/fp"
-import { Query } from "@budibase/types"
+import { Datasource, Query, SourceName } from "@budibase/types"
 
 import { isSQL } from "../integrations/utils"
 import { interpolateSQL } from "../integrations/queries/sql"
 
 class QueryRunner {
-  datasource: any
+  datasource: Datasource
   queryVerb: string
   queryId: string
   fields: any
@@ -68,7 +68,7 @@ class QueryRunner {
       throw "Integration type does not exist."
     }
 
-    if (datasourceClone.config.authConfigs) {
+    if (datasourceClone.config?.authConfigs) {
       const updatedConfigs = []
       for (let config of datasourceClone.config.authConfigs) {
         updatedConfigs.push(await sdk.queries.enrichContext(config, this.ctx))
@@ -93,7 +93,7 @@ class QueryRunner {
     const enrichedContext = { ...enrichedParameters, ...this.ctx }
 
     // Parse global headers
-    if (datasourceClone.config.defaultHeaders) {
+    if (datasourceClone.config?.defaultHeaders) {
       datasourceClone.config.defaultHeaders = await sdk.queries.enrichContext(
         datasourceClone.config.defaultHeaders,
         enrichedContext
@@ -127,10 +127,16 @@ class QueryRunner {
 
     // transform as required
     if (transformer) {
-      const runner = new ScriptRunner(transformer, {
+      const runner = new ScriptRunner(
-        data: rows,
+        transformer,
-        params: enrichedParameters,
+        {
-      })
+          data: rows,
+          params: enrichedParameters,
+        },
+        {
+          parseBson: datasource.source === SourceName.MONGODB,
+        }
+      )
       rows = runner.execute()
     }
 

packages/server/src/utilities/fileSystem/filesystem.ts

@@ -4,7 +4,7 @@ import { resolve, join } from "path"
 import env from "../../environment"
 import tar from "tar"
 
-const uuid = require("uuid/v4")
+import { v4 as uuid } from "uuid"
 
 export const TOP_LEVEL_PATH =
   env.TOP_LEVEL_PATH || resolve(join(__dirname, "..", "..", ".."))

packages/server/src/utilities/scriptRunner.ts

@@ -1,28 +1,27 @@
-import fetch from "node-fetch"
+import env from "../environment"
-import { VM, VMScript } from "vm2"
+import { IsolatedVM } from "../jsRunner/vm"
 
 const JS_TIMEOUT_MS = 1000
 
 class ScriptRunner {
-  vm: VM
+  private code: string
-  results: { out: string }
+  private vm: IsolatedVM
-  script: VMScript
 
-  constructor(script: string, context: any) {
+  constructor(script: string, context: any, { parseBson = false } = {}) {
-    const code = `let fn = () => {\n${script}\n}; results.out = fn();`
+    this.code = `(() => {${script}})();`
-    this.vm = new VM({
+    this.vm = new IsolatedVM({
-      timeout: JS_TIMEOUT_MS,
+      memoryLimit: env.JS_RUNNER_MEMORY_LIMIT,
-    })
+      invocationTimeout: JS_TIMEOUT_MS,
-    this.results = { out: "" }
+    }).withContext(context)
-    this.vm.setGlobals(context)
+
-    this.vm.setGlobal("fetch", fetch)
+    if (parseBson && context.data) {
-    this.vm.setGlobal("results", this.results)
+      this.vm = this.vm.withParsingBson(context.data)
-    this.script = new VMScript(code)
+    }
   }
 
   execute() {
-    this.vm.run(this.script)
+    const result = this.vm.execute(this.code)
-    return this.results.out
+    return result
   }
 }
 

packages/string-templates/package.json

@@ -11,7 +11,8 @@
       "require": "./src/index.cjs",
       "import": "./dist/bundle.mjs"
     },
-    "./package.json": "./package.json"
+    "./package.json": "./package.json",
+    "./test/utils": "./test/utils.js"
   },
   "files": [
     "dist",
@@ -20,7 +21,7 @@
   ],
   "scripts": {
     "build": "tsc && rollup -c",
-    "dev": "tsc && rollup -cw",
+    "dev": "concurrently \"tsc --watch\" \"rollup -cw\"",
     "test": "jest",
     "manifest": "node ./scripts/gen-collection-info.js"
   },
@@ -34,6 +35,7 @@
   "devDependencies": {
     "@rollup/plugin-commonjs": "^17.1.0",
     "@rollup/plugin-json": "^4.1.0",
+    "concurrently": "^8.2.2",
     "doctrine": "^3.0.0",
     "jest": "29.7.0",
     "marked": "^4.0.10",

packages/string-templates/src/errors.js

@@ -0,0 +1,11 @@
+class JsErrorTimeout extends Error {
+  code = "ERR_SCRIPT_EXECUTION_TIMEOUT"
+
+  constructor() {
+    super()
+  }
+}
+
+module.exports = {
+  JsErrorTimeout,
+}

packages/string-templates/src/helpers/javascript.js

@@ -42,7 +42,7 @@ module.exports.processJS = (handlebars, context) => {
   try {
     // Wrap JS in a function and immediately invoke it.
     // This is required to allow the final `return` statement to be valid.
-    const js = `function run(){${atob(handlebars)}};run();`
+    const js = `(function(){${atob(handlebars)}})();`
 
     // Our $ context function gets a value from context.
     // We clone the context to avoid mutation in the binding affecting real

packages/string-templates/src/helpers/list.js

@@ -1,29 +1,42 @@
-const externalHandlebars = require("./external")
+const { date, duration } = require("./date")
-const helperList = require("@budibase/handlebars-helpers")
 
-let helpers = undefined
+// https://github.com/evanw/esbuild/issues/56
+const externalCollections = {
+  math: require("@budibase/handlebars-helpers/lib/math"),
+  array: require("@budibase/handlebars-helpers/lib/array"),
+  number: require("@budibase/handlebars-helpers/lib/number"),
+  url: require("@budibase/handlebars-helpers/lib/url"),
+  string: require("@budibase/handlebars-helpers/lib/string"),
+  comparison: require("@budibase/handlebars-helpers/lib/comparison"),
+  object: require("@budibase/handlebars-helpers/lib/object"),
+  regex: require("@budibase/handlebars-helpers/lib/regex"),
+  uuid: require("@budibase/handlebars-helpers/lib/uuid"),
+}
 
 const helpersToRemoveForJs = ["sortBy"]
 module.exports.helpersToRemoveForJs = helpersToRemoveForJs
 
+const addedHelpers = {
+  date: date,
+  duration: duration,
+}
+
+let helpers = undefined
+
 module.exports.getJsHelperList = () => {
   if (helpers) {
     return helpers
   }
 
   helpers = {}
-  let constructed = []
+  for (let collection of Object.values(externalCollections)) {
-  for (let collection of externalHandlebars.externalCollections) {
-    constructed.push(helperList[collection]())
-  }
-  for (let collection of constructed) {
     for (let [key, func] of Object.entries(collection)) {
       // Handlebars injects the hbs options to the helpers by default. We are adding an empty {} as a last parameter to simulate it
       helpers[key] = (...props) => func(...props, {})
     }
   }
-  for (let key of Object.keys(externalHandlebars.addedHelpers)) {
+  for (let key of Object.keys(addedHelpers)) {
-    helpers[key] = externalHandlebars.addedHelpers[key]
+    helpers[key] = addedHelpers[key]
   }
 
   for (const toRemove of helpersToRemoveForJs) {

packages/string-templates/src/index.cjs

@@ -36,3 +36,8 @@ if (!process.env.NO_JS) {
     return vm.run(js)
   })
 }
+
+const errors = require("./errors")
+for (const error in errors) {
+  module.exports[error] = errors[error]
+}

packages/string-templates/src/index.js

@@ -395,4 +395,9 @@ module.exports.convertToJS = hbs => {
 }
 
 module.exports.FIND_ANY_HBS_REGEX = FIND_ANY_HBS_REGEX
+
+const errors = require("./errors")
+// We cannot use dynamic exports, otherwise the typescript file will not be generating it
+module.exports.JsErrorTimeout = errors.JsErrorTimeout
+
 module.exports.helpersToRemoveForJs = helpersToRemoveForJs

packages/string-templates/src/index.mjs

@@ -38,3 +38,5 @@ if (process && !process.env.NO_JS) {
     return vm.runInNewContext(js, context, { timeout: 1000 })
   })
 }
+
+export * from "./errors.js"

packages/string-templates/test/manifest.spec.js

@@ -15,81 +15,23 @@ jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
   }
 })
 
-const fs = require("fs")
+const { processString } = require("../src/index.cjs")
-const {
-  processString,
-  convertToJS,
-  processStringSync,
-  encodeJSBinding,
-} = require("../src/index.cjs")
 
 const tk = require("timekeeper")
-const { getJsHelperList } = require("../src/helpers")
+const { getParsedManifest, runJsHelpersTests } = require("./utils")
 
 tk.freeze("2021-01-21T12:00:00")
 
-const processJS = (js, context) => {
-  return processStringSync(encodeJSBinding(js), context)
-}
-
-const manifest = JSON.parse(
-  fs.readFileSync(require.resolve("../manifest.json"), "utf8")
-)
-
-const collections = Object.keys(manifest)
-const examples = collections.reduce((acc, collection) => {
-  const functions = Object.entries(manifest[collection])
-    .filter(([_, details]) => details.example)
-    .map(([name, details]) => {
-      const example = details.example
-      let [hbs, js] = example.split("->").map(x => x.trim())
-      if (!js) {
-        // The function has no return value
-        return
-      }
-
-      // Trim 's
-      js = js.replace(/^\'|\'$/g, "")
-      if ((parsedExpected = tryParseJson(js))) {
-        if (Array.isArray(parsedExpected)) {
-          if (typeof parsedExpected[0] === "object") {
-            js = JSON.stringify(parsedExpected)
-          } else {
-            js = parsedExpected.join(",")
-          }
-        }
-      }
-      const requiresHbsBody = details.requiresBlock
-      return [name, { hbs, js, requiresHbsBody }]
-    })
-    .filter(x => !!x)
-
-  if (Object.keys(functions).length) {
-    acc[collection] = functions
-  }
-  return acc
-}, {})
-
 function escapeRegExp(string) {
   return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") // $& means the whole matched string
 }
 
-function tryParseJson(str) {
-  if (typeof str !== "string") {
-    return
-  }
-
-  try {
-    return JSON.parse(str.replace(/\'/g, '"'))
-  } catch (e) {
-    return
-  }
-}
-
 describe("manifest", () => {
+  const manifest = getParsedManifest()
+
   describe("examples are valid", () => {
-    describe.each(Object.keys(examples))("%s", collection => {
+    describe.each(Object.keys(manifest))("%s", collection => {
-      it.each(examples[collection])("%s", async (_, { hbs, js }) => {
+      it.each(manifest[collection])("%s", async (_, { hbs, js }) => {
         const context = {
           double: i => i * 2,
           isString: x => typeof x === "string",
@@ -108,36 +50,5 @@ describe("manifest", () => {
     })
   })
 
-  describe("can be parsed and run as js", () => {
+  runJsHelpersTests()
-    const jsHelpers = getJsHelperList()
-    const jsExamples = Object.keys(examples).reduce((acc, v) => {
-      acc[v] = examples[v].filter(([key]) => jsHelpers[key])
-      return acc
-    }, {})
-
-    describe.each(Object.keys(jsExamples))("%s", collection => {
-      it.each(
-        jsExamples[collection].filter(
-          ([_, { requiresHbsBody }]) => !requiresHbsBody
-        )
-      )("%s", async (_, { hbs, js }) => {
-        const context = {
-          double: i => i * 2,
-          isString: x => typeof x === "string",
-        }
-
-        const arrays = hbs.match(/\[[^/\]]+\]/)
-        arrays?.forEach((arrayString, i) => {
-          hbs = hbs.replace(new RegExp(escapeRegExp(arrayString)), `array${i}`)
-          context[`array${i}`] = JSON.parse(arrayString.replace(/\'/g, '"'))
-        })
-
-        let convertedJs = convertToJS(hbs)
-
-        let result = processJS(convertedJs, context)
-        result = result.replace(/ /g, " ")
-        expect(result).toEqual(js)
-      })
-    })
-  })
 })

packages/string-templates/test/utils.js

@@ -0,0 +1,111 @@
+const { getManifest } = require("../src")
+const { getJsHelperList } = require("../src/helpers")
+
+const {
+  convertToJS,
+  processStringSync,
+  encodeJSBinding,
+} = require("../src/index.cjs")
+
+function tryParseJson(str) {
+  if (typeof str !== "string") {
+    return
+  }
+
+  try {
+    return JSON.parse(str.replace(/'/g, '"'))
+  } catch (e) {
+    return
+  }
+}
+
+const getParsedManifest = () => {
+  const manifest = getManifest()
+  const collections = Object.keys(manifest)
+  const examples = collections.reduce((acc, collection) => {
+    const functions = Object.entries(manifest[collection])
+      .filter(([_, details]) => details.example)
+      .map(([name, details]) => {
+        const example = details.example
+        let [hbs, js] = example.split("->").map(x => x.trim())
+        if (!js) {
+          // The function has no return value
+          return
+        }
+
+        // Trim 's
+        js = js.replace(/^'|'$/g, "")
+        let parsedExpected
+        if ((parsedExpected = tryParseJson(js))) {
+          if (Array.isArray(parsedExpected)) {
+            if (typeof parsedExpected[0] === "object") {
+              js = JSON.stringify(parsedExpected)
+            } else {
+              js = parsedExpected.join(",")
+            }
+          }
+        }
+        const requiresHbsBody = details.requiresBlock
+        return [name, { hbs, js, requiresHbsBody }]
+      })
+      .filter(x => !!x)
+
+    if (Object.keys(functions).length) {
+      acc[collection] = functions
+    }
+    return acc
+  }, {})
+
+  return examples
+}
+module.exports.getParsedManifest = getParsedManifest
+
+module.exports.runJsHelpersTests = ({ funcWrap, testsToSkip } = {}) => {
+  funcWrap = funcWrap || (delegate => delegate())
+  const manifest = getParsedManifest()
+
+  const processJS = (js, context) => {
+    return funcWrap(() => processStringSync(encodeJSBinding(js), context))
+  }
+
+  function escapeRegExp(string) {
+    return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") // $& means the whole matched string
+  }
+
+  describe("can be parsed and run as js", () => {
+    const jsHelpers = getJsHelperList()
+    const jsExamples = Object.keys(manifest).reduce((acc, v) => {
+      acc[v] = manifest[v].filter(([key]) => jsHelpers[key])
+      return acc
+    }, {})
+
+    describe.each(Object.keys(jsExamples))("%s", collection => {
+      const examplesToRun = jsExamples[collection]
+        .filter(([_, { requiresHbsBody }]) => !requiresHbsBody)
+        .filter(([key]) => !testsToSkip?.includes(key))
+
+      examplesToRun.length &&
+        it.each(examplesToRun)("%s", async (_, { hbs, js }) => {
+          const context = {
+            double: i => i * 2,
+            isString: x => typeof x === "string",
+          }
+
+          const arrays = hbs.match(/\[[^/\]]+\]/)
+          arrays?.forEach((arrayString, i) => {
+            hbs = hbs.replace(
+              new RegExp(escapeRegExp(arrayString)),
+              `array${i}`
+            )
+            context[`array${i}`] = JSON.parse(arrayString.replace(/'/g, '"'))
+          })
+
+          let convertedJs = convertToJS(hbs)
+
+          let result = await processJS(convertedJs, context)
+          result = result.replace(/ /g, " ")
+          expect(result).toEqual(js)
+        })
+    })
+  })
+}

packages/types/src/sdk/index.ts

@@ -20,3 +20,4 @@ export * from "./cli"
 export * from "./websocket"
 export * from "./permissions"
 export * from "./row"
+export * from "./vm"

packages/types/src/sdk/vm.ts

@@ -0,0 +1,3 @@
+export interface VM {
+  execute(code: string): any
+}

packages/worker/Dockerfile

@@ -44,6 +44,8 @@ EXPOSE 4001
 # due to this causing yarn to stop installing dev dependencies
 # which are actually needed to get this environment up and running
 ENV NODE_ENV=production
+# this is required for isolated-vm to work on Node 20+
+ENV NODE_OPTIONS="--no-node-snapshot"
 ENV CLUSTER_MODE=${CLUSTER_MODE}
 ENV SERVICE=worker-service
 ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU

packages/worker/nodemon.json

@@ -8,6 +8,6 @@
     "../string-templates"
   ],
   "ext": "js,ts,json",
-  "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../*/dist/**/*"],
+  "ignore": ["**/*.spec.ts", "**/*.spec.js", "../*/dist/**/*"],
-  "exec": "yarn build && node dist/index.js"
+  "exec": "yarn build && node --no-node-snapshot dist/index.js"
 }

packages/worker/package.json

@@ -68,7 +68,9 @@
     "passport-local": "1.0.0",
     "pouchdb": "7.3.0",
     "pouchdb-all-dbs": "1.1.1",
-    "server-destroy": "1.0.1"
+    "server-destroy": "1.0.1",
+    "undici": "^6.0.1",
+    "undici-types": "^6.0.1"
   },
   "devDependencies": {
     "@swc/core": "1.3.71",

scripts/build.js

@@ -49,6 +49,7 @@ function runBuild(entry, outfile) {
     preserveSymlinks: true,
     loader: {
       ".svelte": "copy",
+      ".ivm.bundle.js": "text",
     },
     metafile: true,
     external: [
@@ -60,6 +61,7 @@ function runBuild(entry, outfile) {
       "bcrypt",
       "bcryptjs",
       "graphql/*",
+      "bson",
     ],
   }