Merge branch 'master' into fix-flaky-backup-test

hosting/single/Dockerfile

@@ -38,7 +38,7 @@ COPY packages/worker/pm2.config.js packages/worker/pm2.config.js
 COPY packages/string-templates packages/string-templates
 
 
-FROM budibase/couchdb as runner
+FROM budibase/couchdb:v3.3.3 as runner
 ARG TARGETARCH
 ENV TARGETARCH $TARGETARCH
 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service)

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.20.1",
+  "version": "2.20.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/pro

@@ -1 +1 @@
-Subproject commit 6f3a0b7f72d16f9604179af98ff7602ca0df2737
+Subproject commit 60e47a8249fd6291a6bc20fe3fe6776b11938fa1

packages/server/package.json

@@ -114,7 +114,6 @@
     "undici-types": "^6.0.1",
     "uuid": "^8.3.2",
     "validate.js": "0.13.1",
-    "vm2": "^3.9.19",
     "worker-farm": "1.7.0",
     "xml2js": "0.5.0"
   },

packages/server/src/api/routes/tests/table.spec.ts

@@ -368,10 +368,12 @@ describe("/tables", () => {
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
-      const fetchedTable = res.body[0]
+
-      expect(fetchedTable.name).toEqual(testTable.name)
+      const table = res.body.find((t: Table) => t._id === testTable._id)
-      expect(fetchedTable.type).toEqual("table")
+      expect(table).toBeDefined()
-      expect(fetchedTable.sourceType).toEqual("internal")
+      expect(table.name).toEqual(testTable.name)
+      expect(table.type).toEqual("table")
+      expect(table.sourceType).toEqual("internal")
     })
 
     it("should apply authorization to endpoint", async () => {

packages/server/src/jsRunner/index.ts

@@ -8,18 +8,11 @@ import {
 import { context, logging } from "@budibase/backend-core"
 import tracer from "dd-trace"
 
-import { BuiltInVM, IsolatedVM } from "./vm"
+import { IsolatedVM } from "./vm"
-
-const USE_ISOLATED_VM = true
 
 export function init() {
   setJSRunner((js: string, ctx: Record<string, any>) => {
     return tracer.trace("runJS", {}, span => {
-      if (!USE_ISOLATED_VM) {
-        const vm = new BuiltInVM(ctx, span)
-        return vm.execute(js)
-      }
-
       try {
         const bbCtx = context.getCurrentContext()
 

packages/server/src/jsRunner/vm/builtin-vm.ts

@@ -1,76 +0,0 @@
-import vm from "vm"
-import env from "../../environment"
-import { context, timers } from "@budibase/backend-core"
-import tracer, { Span } from "dd-trace"
-import { VM } from "@budibase/types"
-
-type TrackerFn = <T>(f: () => T) => T
-
-export class BuiltInVM implements VM {
-  private ctx: vm.Context
-  private span?: Span
-
-  constructor(ctx: vm.Context, span?: Span) {
-    this.ctx = ctx
-    this.span = span
-  }
-
-  withContext<T>(context: Record<string, any>, executeWithContext: () => T): T {
-    this.ctx = vm.createContext(context)
-    try {
-      return executeWithContext()
-    } finally {
-      for (const key in context) {
-        delete this.ctx[key]
-      }
-    }
-  }
-
-  execute(code: string) {
-    const perRequestLimit = env.JS_PER_REQUEST_TIMEOUT_MS
-    let track: TrackerFn = f => f()
-    if (perRequestLimit) {
-      const bbCtx = tracer.trace("runJS.getCurrentContext", {}, span =>
-        context.getCurrentContext()
-      )
-      if (bbCtx) {
-        if (!bbCtx.jsExecutionTracker) {
-          this.span?.addTags({
-            createdExecutionTracker: true,
-          })
-          bbCtx.jsExecutionTracker = tracer.trace(
-            "runJS.createExecutionTimeTracker",
-            {},
-            span => timers.ExecutionTimeTracker.withLimit(perRequestLimit)
-          )
-        }
-        this.span?.addTags({
-          js: {
-            limitMS: bbCtx.jsExecutionTracker.limitMs,
-            elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
-          },
-        })
-        // We call checkLimit() here to prevent paying the cost of creating
-        // a new VM context below when we don't need to.
-        tracer.trace("runJS.checkLimitAndBind", {}, span => {
-          bbCtx.jsExecutionTracker!.checkLimit()
-          track = bbCtx.jsExecutionTracker!.track.bind(bbCtx.jsExecutionTracker)
-        })
-      }
-    }
-
-    this.ctx = {
-      ...this.ctx,
-      alert: undefined,
-      setInterval: undefined,
-      setTimeout: undefined,
-    }
-
-    vm.createContext(this.ctx)
-    return track(() =>
-      vm.runInNewContext(code, this.ctx, {
-        timeout: env.JS_PER_INVOCATION_TIMEOUT_MS,
-      })
-    )
-  }
-}

packages/server/src/jsRunner/vm/index.ts

@@ -1,3 +1 @@
 export * from "./isolated-vm"
-export * from "./builtin-vm"
-export * from "./vm2"

packages/server/src/jsRunner/vm/vm2.ts

@@ -1,36 +0,0 @@
-import vm2 from "vm2"
-import { VM } from "@budibase/types"
-
-const JS_TIMEOUT_MS = 1000
-
-export class VM2 implements VM {
-  vm: vm2.VM
-  results: { out: string }
-
-  constructor() {
-    this.vm = new vm2.VM({
-      timeout: JS_TIMEOUT_MS,
-    })
-    this.results = { out: "" }
-    this.vm.setGlobal("fetch", fetch)
-    this.vm.setGlobal("results", this.results)
-  }
-
-  withContext<T>(context: Record<string, any>, executeWithContext: () => T): T {
-    this.vm.setGlobals(context)
-    try {
-      return executeWithContext()
-    } finally {
-      for (const key in context) {
-        this.vm.setGlobal(key, undefined)
-      }
-    }
-  }
-
-  execute(script: string) {
-    const code = `let fn = () => {\n${script}\n}; results.out = fn();`
-    const vmScript = new vm2.VMScript(code)
-    this.vm.run(vmScript)
-    return this.results.out
-  }
-}

packages/server/src/threads/query.ts

@@ -7,20 +7,18 @@ import {
   QueryVariable,
   QueryResponse,
 } from "./definitions"
-import { IsolatedVM, VM2 } from "../jsRunner/vm"
+import { IsolatedVM } from "../jsRunner/vm"
 import { getIntegration } from "../integrations"
 import { processStringSync } from "@budibase/string-templates"
 import { context, cache, auth } from "@budibase/backend-core"
 import { getGlobalIDFromUserMetadataID } from "../db/utils"
 import sdk from "../sdk"
 import { cloneDeep } from "lodash/fp"
-import { Datasource, Query, SourceName, VM } from "@budibase/types"
+import { Datasource, Query, SourceName } from "@budibase/types"
 
 import { isSQL } from "../integrations/utils"
 import { interpolateSQL } from "../integrations/queries/sql"
 
-const USE_ISOLATED_VM = true
-
 class QueryRunner {
   datasource: Datasource
   queryVerb: string
@@ -129,23 +127,17 @@ class QueryRunner {
 
     // transform as required
     if (transformer) {
-      let runner: VM
+      transformer = `(function(){\n${transformer}\n})();`
-      if (!USE_ISOLATED_VM) {
+      let vm = new IsolatedVM()
-        runner = new VM2()
+      if (datasource.source === SourceName.MONGODB) {
-      } else {
+        vm = vm.withParsingBson(rows)
-        transformer = `(function(){\n${transformer}\n})();`
-        let vm = new IsolatedVM()
-        if (datasource.source === SourceName.MONGODB) {
-          vm = vm.withParsingBson(rows)
-        }
-        runner = vm
       }
 
       const ctx = {
         data: rows,
         params: enrichedParameters,
       }
-      rows = runner.withContext(ctx, () => runner.execute(transformer))
+      rows = vm.withContext(ctx, () => vm.execute(transformer))
     }
 
     // if the request fails we retry once, invalidating the cached value

yarn.lock

@@ -6463,7 +6463,7 @@ acorn@^7.1.1:
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa"
   integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==
 
-acorn@^8.1.0, acorn@^8.10.0, acorn@^8.2.4, acorn@^8.4.1, acorn@^8.5.0, acorn@^8.7.0, acorn@^8.7.1, acorn@^8.8.1, acorn@^8.8.2, acorn@^8.9.0:
+acorn@^8.1.0, acorn@^8.10.0, acorn@^8.2.4, acorn@^8.4.1, acorn@^8.5.0, acorn@^8.7.1, acorn@^8.8.1, acorn@^8.8.2, acorn@^8.9.0:
   version "8.11.3"
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.11.3.tgz#71e0b14e13a4ec160724b38fb7b0f233b1b81d7a"
   integrity sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==
@@ -21633,14 +21633,6 @@ vlq@^0.2.2:
   resolved "https://registry.yarnpkg.com/vlq/-/vlq-0.2.3.tgz#8f3e4328cf63b1540c0d67e1b2778386f8975b26"
   integrity sha512-DRibZL6DsNhIgYQ+wNdWDL2SL3bKPlVrRiBqV5yuMm++op8W4kGFtaQfCs4KEJn0wBZcHVHJ3eoywX8983k1ow==
 
-vm2@^3.9.19:
-  version "3.9.19"
-  resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.19.tgz#be1e1d7a106122c6c492b4d51c2e8b93d3ed6a4a"
-  integrity sha512-J637XF0DHDMV57R6JyVsTak7nIL8gy5KH4r1HiwWLf/4GBbb5MKL5y7LpmF4A8E2nR6XmzpmMFQ7V7ppPTmUQg==
-  dependencies:
-    acorn "^8.7.0"
-    acorn-walk "^8.2.0"
-
 vuvuzela@1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/vuvuzela/-/vuvuzela-1.0.3.tgz#3be145e58271c73ca55279dd851f12a682114b0b"