Merge pull request #12922 from Budibase/test-isolated-vm

Test isolated vm

packages/backend-core/package.json

@@ -54,7 +54,7 @@
     "sanitize-s3-objectkey": "0.0.1",
     "semver": "7.3.7",
     "tar-fs": "2.1.1",
-    "uuid": "8.3.2"
+    "uuid": "^8.3.2"
   },
   "devDependencies": {
     "@shopify/jest-koa-mocks": "5.1.1",

packages/server/package.json

@@ -106,7 +106,7 @@
     "svelte": "^3.49.0",
     "tar": "6.1.15",
     "to-json-schema": "0.2.5",
-    "uuid": "3.3.2",
+    "uuid": "^8.3.2",
     "validate.js": "0.13.1",
     "worker-farm": "1.7.0",
     "xml2js": "0.5.0"
@@ -130,6 +130,7 @@
     "@types/server-destroy": "1.0.1",
     "@types/supertest": "2.0.14",
     "@types/tar": "6.1.5",
+    "@types/uuid": "8.3.4",
     "apidoc": "0.50.4",
     "copyfiles": "2.4.1",
     "docker-compose": "0.23.17",

packages/server/src/api/controllers/static/index.ts

@@ -3,7 +3,7 @@ import { InvalidFileExtensions } from "@budibase/shared-core"
 require("svelte/register")
 
 import { join } from "../../../utilities/centralPath"
-import uuid from "uuid"
+import * as uuid from "uuid"
 import { ObjectStoreBuckets } from "../../../constants"
 import { processString } from "@budibase/string-templates"
 import {

packages/server/src/api/routes/tests/table.spec.ts

@@ -16,7 +16,7 @@ import {
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 import sdk from "../../../sdk"
-import uuid from "uuid"
+import * as uuid from "uuid"
 
 const { basicTable } = setup.structures
 

packages/server/src/db/newid.ts

@@ -1,4 +1,4 @@
-const { v4 } = require("uuid")
+import { v4 } from "uuid"
 
 export default function (): string {
   return v4().replace(/-/g, "")

packages/server/src/jsRunner/index.ts

@@ -1,11 +1,12 @@
 import ivm from "isolated-vm"
-import env from "./environment"
+import env from "../environment"
 import { setJSRunner, JsErrorTimeout } from "@budibase/string-templates"
 import { context } from "@budibase/backend-core"
 import tracer from "dd-trace"
 import fs from "fs"
 import url from "url"
 import crypto from "crypto"
+import querystring from "querystring"
 
 const helpersSource = fs.readFileSync(
   `${require.resolve("@budibase/string-templates/index-helpers")}`,
@@ -39,6 +40,10 @@ export function init() {
               resolve: (...params) => urlResolveCb(...params),
               parse: (...params) => urlParseCb(...params),
             }
+          case "querystring":
+            return {
+              escape: (...params) => querystringEscapeCb(...params),
+            }
         }
       };`
 
@@ -57,6 +62,23 @@ export function init() {
             )
           )
 
+          global.setSync(
+            "querystringEscapeCb",
+            new ivm.Callback(
+              (...params: Parameters<typeof querystring.escape>) =>
+                querystring.escape(...params)
+            )
+          )
+
+          global.setSync(
+            "helpersStripProtocol",
+            new ivm.Callback((str: string) => {
+              var parsed = url.parse(str) as any
+              parsed.protocol = ""
+              return parsed.format()
+            })
+          )
+
           const helpersModule = jsIsolate.compileModuleSync(
             `${injectedRequire};${helpersSource}`
           )

packages/server/src/jsRunner/tests/jsRunner.spec.ts

@@ -0,0 +1,75 @@
+import { validate as isValidUUID } from "uuid"
+
+jest.mock("@budibase/handlebars-helpers/lib/math", () => {
+  const actual = jest.requireActual("@budibase/handlebars-helpers/lib/math")
+
+  return {
+    ...actual,
+    random: () => 10,
+  }
+})
+jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
+  const actual = jest.requireActual("@budibase/handlebars-helpers/lib/uuid")
+
+  return {
+    ...actual,
+    uuid: () => "f34ebc66-93bd-4f7c-b79b-92b5569138bc",
+  }
+})
+
+import { processStringSync, encodeJSBinding } from "@budibase/string-templates"
+
+const { runJsHelpersTests } = require("@budibase/string-templates/test/utils")
+
+import tk from "timekeeper"
+import { init } from ".."
+import TestConfiguration from "../../tests/utilities/TestConfiguration"
+
+tk.freeze("2021-01-21T12:00:00")
+
+describe("jsRunner", () => {
+  const config = new TestConfiguration()
+
+  beforeAll(async () => {
+    // Register js runner
+    init()
+    await config.init()
+  })
+
+  const processJS = (js: string, context?: object) => {
+    return config.doInContext(config.getAppId(), async () =>
+      processStringSync(encodeJSBinding(js), context || {})
+    )
+  }
+
+  it("it can run a basic javascript", async () => {
+    const output = await processJS(`return 1 + 2`)
+    expect(output).toBe(3)
+  })
+
+  describe("helpers", () => {
+    runJsHelpersTests({
+      funcWrap: (func: any) => config.doInContext(config.getAppId(), func),
+      testsToSkip: ["random", "uuid"],
+    })
+
+    describe("uuid", () => {
+      it("uuid helper returns a valid uuid", async () => {
+        const result = await processJS("return helpers.uuid()")
+        expect(result).toBeDefined()
+        expect(isValidUUID(result)).toBe(true)
+      })
+    })
+
+    describe("random", () => {
+      it("random helper returns a valid number", async () => {
+        const min = 1
+        const max = 8
+        const result = await processJS(`return helpers.random(${min}, ${max})`)
+        expect(result).toBeDefined()
+        expect(result).toBeGreaterThanOrEqual(min)
+        expect(result).toBeLessThanOrEqual(max)
+      })
+    })
+  })
+})

packages/server/src/utilities/fileSystem/filesystem.ts

@@ -4,7 +4,7 @@ import { resolve, join } from "path"
 import env from "../../environment"
 import tar from "tar"
 
-const uuid = require("uuid/v4")
+import { v4 as uuid } from "uuid"
 
 export const TOP_LEVEL_PATH =
   env.TOP_LEVEL_PATH || resolve(join(__dirname, "..", "..", ".."))

packages/string-templates/package.json

@@ -12,7 +12,8 @@
       "import": "./dist/bundle.mjs"
     },
     "./package.json": "./package.json",
-    "./index-helpers": "./dist/index-helpers.bundled.js"
+    "./index-helpers": "./dist/index-helpers.bundled.js",
+    "./test/utils": "./test/utils.js"
   },
   "files": [
     "dist",

packages/string-templates/src/index-helpers.js

@@ -1,3 +1,9 @@
 const { getJsHelperList } = require("./helpers/list")
 
-module.exports = getJsHelperList()
+const helpers = getJsHelperList()
+module.exports = {
+  ...helpers,
+  // pointing stripProtocol to a unexisting function to be able to declare it on isolated-vm
+  // eslint-disable-next-line no-undef
+  stripProtocol: helpersStripProtocol,
+}

packages/string-templates/test/manifest.spec.js

@@ -15,81 +15,23 @@ jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
   }
 })
 
-const fs = require("fs")
+const { processString } = require("../src/index.cjs")
-const {
-  processString,
-  convertToJS,
-  processStringSync,
-  encodeJSBinding,
-} = require("../src/index.cjs")
 
 const tk = require("timekeeper")
-const { getJsHelperList } = require("../src/helpers")
+const { getParsedManifest, runJsHelpersTests } = require("./utils")
 
 tk.freeze("2021-01-21T12:00:00")
 
-const processJS = (js, context) => {
-  return processStringSync(encodeJSBinding(js), context)
-}
-
-const manifest = JSON.parse(
-  fs.readFileSync(require.resolve("../manifest.json"), "utf8")
-)
-
-const collections = Object.keys(manifest)
-const examples = collections.reduce((acc, collection) => {
-  const functions = Object.entries(manifest[collection])
-    .filter(([_, details]) => details.example)
-    .map(([name, details]) => {
-      const example = details.example
-      let [hbs, js] = example.split("->").map(x => x.trim())
-      if (!js) {
-        // The function has no return value
-        return
-      }
-
-      // Trim 's
-      js = js.replace(/^\'|\'$/g, "")
-      if ((parsedExpected = tryParseJson(js))) {
-        if (Array.isArray(parsedExpected)) {
-          if (typeof parsedExpected[0] === "object") {
-            js = JSON.stringify(parsedExpected)
-          } else {
-            js = parsedExpected.join(",")
-          }
-        }
-      }
-      const requiresHbsBody = details.requiresBlock
-      return [name, { hbs, js, requiresHbsBody }]
-    })
-    .filter(x => !!x)
-
-  if (Object.keys(functions).length) {
-    acc[collection] = functions
-  }
-  return acc
-}, {})
-
 function escapeRegExp(string) {
   return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") // $& means the whole matched string
 }
 
-function tryParseJson(str) {
-  if (typeof str !== "string") {
-    return
-  }
-
-  try {
-    return JSON.parse(str.replace(/\'/g, '"'))
-  } catch (e) {
-    return
-  }
-}
-
 describe("manifest", () => {
+  const manifest = getParsedManifest()
+
   describe("examples are valid", () => {
-    describe.each(Object.keys(examples))("%s", collection => {
+    describe.each(Object.keys(manifest))("%s", collection => {
-      it.each(examples[collection])("%s", async (_, { hbs, js }) => {
+      it.each(manifest[collection])("%s", async (_, { hbs, js }) => {
         const context = {
           double: i => i * 2,
           isString: x => typeof x === "string",
@@ -108,36 +50,5 @@ describe("manifest", () => {
     })
   })
 
-  describe("can be parsed and run as js", () => {
+  runJsHelpersTests()
-    const jsHelpers = getJsHelperList()
-    const jsExamples = Object.keys(examples).reduce((acc, v) => {
-      acc[v] = examples[v].filter(([key]) => jsHelpers[key])
-      return acc
-    }, {})
-
-    describe.each(Object.keys(jsExamples))("%s", collection => {
-      it.each(
-        jsExamples[collection].filter(
-          ([_, { requiresHbsBody }]) => !requiresHbsBody
-        )
-      )("%s", async (_, { hbs, js }) => {
-        const context = {
-          double: i => i * 2,
-          isString: x => typeof x === "string",
-        }
-
-        const arrays = hbs.match(/\[[^/\]]+\]/)
-        arrays?.forEach((arrayString, i) => {
-          hbs = hbs.replace(new RegExp(escapeRegExp(arrayString)), `array${i}`)
-          context[`array${i}`] = JSON.parse(arrayString.replace(/\'/g, '"'))
-        })
-
-        let convertedJs = convertToJS(hbs)
-
-        let result = processJS(convertedJs, context)
-        result = result.replace(/ /g, " ")
-        expect(result).toEqual(js)
-      })
-    })
-  })
 })

packages/string-templates/test/utils.js

@@ -0,0 +1,111 @@
+const { getManifest } = require("../src")
+const { getJsHelperList } = require("../src/helpers")
+
+const {
+  convertToJS,
+  processStringSync,
+  encodeJSBinding,
+} = require("../src/index.cjs")
+
+function tryParseJson(str) {
+  if (typeof str !== "string") {
+    return
+  }
+
+  try {
+    return JSON.parse(str.replace(/'/g, '"'))
+  } catch (e) {
+    return
+  }
+}
+
+const getParsedManifest = () => {
+  const manifest = getManifest()
+  const collections = Object.keys(manifest)
+  const examples = collections.reduce((acc, collection) => {
+    const functions = Object.entries(manifest[collection])
+      .filter(([_, details]) => details.example)
+      .map(([name, details]) => {
+        const example = details.example
+        let [hbs, js] = example.split("->").map(x => x.trim())
+        if (!js) {
+          // The function has no return value
+          return
+        }
+
+        // Trim 's
+        js = js.replace(/^'|'$/g, "")
+        let parsedExpected
+        if ((parsedExpected = tryParseJson(js))) {
+          if (Array.isArray(parsedExpected)) {
+            if (typeof parsedExpected[0] === "object") {
+              js = JSON.stringify(parsedExpected)
+            } else {
+              js = parsedExpected.join(",")
+            }
+          }
+        }
+        const requiresHbsBody = details.requiresBlock
+        return [name, { hbs, js, requiresHbsBody }]
+      })
+      .filter(x => !!x)
+
+    if (Object.keys(functions).length) {
+      acc[collection] = functions
+    }
+    return acc
+  }, {})
+
+  return examples
+}
+module.exports.getParsedManifest = getParsedManifest
+
+module.exports.runJsHelpersTests = ({ funcWrap, testsToSkip } = {}) => {
+  funcWrap = funcWrap || (delegate => delegate())
+  const manifest = getParsedManifest()
+
+  const processJS = (js, context) => {
+    return funcWrap(() => processStringSync(encodeJSBinding(js), context))
+  }
+
+  function escapeRegExp(string) {
+    return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") // $& means the whole matched string
+  }
+
+  describe("can be parsed and run as js", () => {
+    const jsHelpers = getJsHelperList()
+    const jsExamples = Object.keys(manifest).reduce((acc, v) => {
+      acc[v] = manifest[v].filter(([key]) => jsHelpers[key])
+      return acc
+    }, {})
+
+    describe.each(Object.keys(jsExamples))("%s", collection => {
+      const examplesToRun = jsExamples[collection]
+        .filter(([_, { requiresHbsBody }]) => !requiresHbsBody)
+        .filter(([key]) => !testsToSkip?.includes(key))
+
+      examplesToRun.length &&
+        it.each(examplesToRun)("%s", async (_, { hbs, js }) => {
+          const context = {
+            double: i => i * 2,
+            isString: x => typeof x === "string",
+          }
+
+          const arrays = hbs.match(/\[[^/\]]+\]/)
+          arrays?.forEach((arrayString, i) => {
+            hbs = hbs.replace(
+              new RegExp(escapeRegExp(arrayString)),
+              `array${i}`
+            )
+            context[`array${i}`] = JSON.parse(arrayString.replace(/'/g, '"'))
+          })
+
+          let convertedJs = convertToJS(hbs)
+
+          let result = await processJS(convertedJs, context)
+          result = result.replace(/ /g, " ")
+          expect(result).toEqual(js)
+        })
+    })
+  })
+}