Merge remote-tracking branch 'origin/master' into feature/app-list-actions

packages/server/src/api/controllers/script.ts

@@ -1,13 +1,11 @@
 import { Ctx } from "@budibase/types"
 import { IsolatedVM } from "../../jsRunner/vm"
+import { iifeWrapper } from "../../jsRunner/utilities"
 
 export async function execute(ctx: Ctx) {
   const { script, context } = ctx.request.body
   const vm = new IsolatedVM()
-  const result = vm.withContext(context, () =>
+  ctx.body = vm.withContext(context, () => vm.execute(iifeWrapper(script)))
-    vm.execute(`(function(){\n${script}\n})();`)
-  )
-  ctx.body = result
 }
 
 export async function save(ctx: Ctx) {

packages/server/src/jsRunner/index.ts

@@ -7,7 +7,6 @@ import {
 } from "@budibase/string-templates"
 import { context, logging } from "@budibase/backend-core"
 import tracer from "dd-trace"
-
 import { IsolatedVM } from "./vm"
 
 export function init() {

packages/server/src/jsRunner/tests/isolatedVM.spec.ts

@@ -0,0 +1,110 @@
+import fs from "fs"
+import path from "path"
+import { IsolatedVM } from "../vm"
+import { iifeWrapper } from "../utilities"
+
+function runJSWithIsolatedVM(script: string, context: Record<string, any>) {
+  const runner = new IsolatedVM()
+  return runner.withContext(context, () => {
+    return runner.execute(iifeWrapper(script))
+  })
+}
+
+describe("Test isolated vm directly", () => {
+  it("should handle a very large file", () => {
+    const marked = fs.readFileSync(
+      path.join(__dirname, "largeJSExample.txt"),
+      "utf-8"
+    )
+    const result = runJSWithIsolatedVM(marked, {
+      trigger: { row: { Message: "dddd" } },
+    })
+    expect(result).toBe("<p>dddd</p>\n")
+  })
+
+  it("handle a mapping case", async () => {
+    const context = {
+      data: {
+        data: {
+          searchProducts: {
+            results: [{ imageLinks: ["_S/"] }],
+          },
+        },
+      },
+    }
+    const result = await runJSWithIsolatedVM(
+      `
+      const dataUnnested = data.data.searchProducts.results
+      const emptyLink = "https://budibase.com"
+      let pImage = emptyLink
+      let sImage = emptyLink
+      let uImage = emptyLink
+      let lImage = emptyLink
+      let b1Image = emptyLink
+      let b2Image = emptyLink
+
+      const dataTransformed = dataUnnested.map(x=> {
+        let imageLinks = x.imageLinks
+        for (let i = 0; i < imageLinks.length; i++){ 
+          if(imageLinks[i].includes("_P/") || imageLinks[i].includes("_p/")){
+            pImage = imageLinks[i]
+          } else if (imageLinks[i].includes("_S/") || imageLinks[i].includes("_s/")){
+            sImage = imageLinks[i]
+          } else if (imageLinks[i].includes("_U/") || imageLinks[i].includes("_u/")){
+            uImage = imageLinks[i]
+          } else if (imageLinks[i].includes("_L/") || imageLinks[i].includes("_l/")){
+            lImage = imageLinks[i]
+          } else if (imageLinks[i].includes("_B/") || imageLinks[i].includes("_b/")){
+            b1Image = imageLinks[i]
+          } else if (imageLinks[i].includes("_B2/") || imageLinks[i].includes("_b2/")){
+            b2Image = imageLinks[i]
+          }
+        }
+
+        const arrangedLinks = [pImage, sImage, uImage, lImage, b1Image, b2Image]
+        x.imageLinks = arrangedLinks
+
+        return x
+      })
+
+      return dataTransformed
+      `,
+      context
+    )
+    expect(result).toBeDefined()
+    expect(result.length).toBe(1)
+    expect(result[0].imageLinks).toEqual([
+      "https://budibase.com",
+      "_S/",
+      "https://budibase.com",
+      "https://budibase.com",
+      "https://budibase.com",
+      "https://budibase.com",
+    ])
+  })
+
+  it("should handle automation script example", () => {
+    const context = {
+      steps: [{}, { response: "hello" }, { items: [{ rows: [{ a: 1 }] }] }],
+    }
+    const result = runJSWithIsolatedVM(
+      `const queryResults = steps[2].items;
+      
+      const intervals = steps[1].response;
+      const whereNoItemsReturned = [];
+      let index = 0;
+      
+      for (let queryResult of queryResults) {
+        if (queryResult.rows.length === 0) {
+          whereNoItemsReturned.push(intervals[index]);
+        }
+        index++;
+      }
+      
+      return whereNoItemsReturned;
+      `,
+      context
+    )
+    expect(result).toEqual([])
+  })
+})

packages/server/src/jsRunner/tests/jsRunner.spec.ts

@@ -7,7 +7,9 @@ import tk from "timekeeper"
 import { init } from ".."
 import TestConfiguration from "../../tests/utilities/TestConfiguration"
 
-tk.freeze("2021-01-21T12:00:00")
+const DATE = "2021-01-21T12:00:00"
+
+tk.freeze(DATE)
 
 describe("jsRunner (using isolated-vm)", () => {
   const config = new TestConfiguration()
@@ -70,4 +72,278 @@ describe("jsRunner (using isolated-vm)", () => {
       })
     })
   })
+
+  // the test cases here were extracted from templates/real world examples of JS in Budibase
+  describe("real test cases from Budicloud", () => {
+    const context = {
+      "Unit Value": 2,
+      Quantity: 1,
+    }
+    it("handle test case 1", async () => {
+      const result = await processJS(
+        `
+        var Gross = $("[Unit Value]") * $("[Quantity]")
+        return Gross.toFixed(2)`,
+        context
+      )
+      expect(result).toBeDefined()
+      expect(result).toBe("2.00")
+    })
+
+    it("handle test case 2", async () => {
+      const context = {
+        "Purchase Date": DATE,
+      }
+      const result = await processJS(
+        `
+        var purchase = new Date($("[Purchase Date]"));
+        let purchaseyear = purchase.getFullYear();
+        let purchasemonth = purchase.getMonth();
+
+        var today = new Date ();
+        let todayyear = today.getFullYear();
+        let todaymonth = today.getMonth();
+
+        var age = todayyear - purchaseyear
+
+        if (((todaymonth - purchasemonth) < 6) == true){
+          return age
+        }
+        `,
+        context
+      )
+      expect(result).toBeDefined()
+      expect(result).toBe(3)
+    })
+
+    it("should handle test case 3", async () => {
+      const context = {
+        Escalate: true,
+        "Budget ($)": 1100,
+      }
+      const result = await processJS(
+        `
+        if ($("[Escalate]") == true) {
+          if ($("Budget ($)") <= 1000)
+            {return 2;}
+          if ($("Budget ($)") > 1000) 
+            {return 3;}
+          }
+          else {
+            if ($("Budget ($)") <= 1000)
+              {return 1;}
+            if ($("Budget ($)") > 1000)
+              if ($("Budget ($)") < 10000) 
+                {return 2;}
+              else 
+                {return 3}
+          }
+        `,
+        context
+      )
+      expect(result).toBeDefined()
+      expect(result).toBe(3)
+    })
+
+    it("should handle test case 4", async () => {
+      const context = {
+        "Time Sheets": ["a", "b"],
+      }
+      const result = await processJS(
+        `
+      let hours = 0
+      if (($("[Time Sheets]") != null) == true){
+        for (i = 0; i < $("[Time Sheets]").length; i++){
+          let hoursLogged = "Time Sheets." + i + ".Hours"
+          hours += $(hoursLogged)
+        }
+        return hours
+      }
+      if (($("[Time Sheets]") != null) == false){
+        return hours
+      }
+      `,
+        context
+      )
+      expect(result).toBeDefined()
+      expect(result).toBe("0ab")
+    })
+
+    it("should handle test case 5", async () => {
+      const context = {
+        change: JSON.stringify({ a: 1, primaryDisplay: "a" }),
+        previous: JSON.stringify({ a: 2, primaryDisplay: "b" }),
+      }
+      const result = await processJS(
+        `
+      let change = $("[change]") ? JSON.parse($("[change]")) : {}
+      let previous = $("[previous]") ? JSON.parse($("[previous]")) : {}
+
+      function simplifyLink(originalKey, value, parent) {
+        if (Array.isArray(value)) {
+          if (value.filter(item => Object.keys(item || {}).includes("primaryDisplay")).length > 0) {
+            parent[originalKey] = value.map(link => link.primaryDisplay)
+          }
+        }
+      }
+
+      for (let entry of Object.entries(change)) {
+        simplifyLink(entry[0], entry[1], change)
+      }
+      for (let entry of Object.entries(previous)) {
+        simplifyLink(entry[0], entry[1], previous)
+      }
+      
+      let diff = Object.fromEntries(Object.entries(change).filter(([k, v]) => previous[k]?.toString() !== v?.toString()))
+      
+      delete diff.audit_change
+      delete diff.audit_previous
+      delete diff._id
+      delete diff._rev
+      delete diff.tableId
+      delete diff.audit
+      
+      for (let entry of Object.entries(diff)) {
+        simplifyLink(entry[0], entry[1], diff)
+      }
+      
+      return JSON.stringify(change)?.replaceAll(",\\"", ",\\n\\t\\"").replaceAll("{\\"", "{\\n\\t\\"").replaceAll("}", "\\n}")
+      `,
+        context
+      )
+      expect(result).toBe(`{\n\t"a":1,\n\t"primaryDisplay":"a"\n}`)
+    })
+
+    it("should handle test case 6", async () => {
+      const context = {
+        "Join Date": DATE,
+      }
+      const result = await processJS(
+        `
+        var rate = 5;
+        var today = new Date();
+        
+        // comment
+        function monthDiff(dateFrom, dateTo) {
+         return dateTo.getMonth() - dateFrom.getMonth() + 
+           (12 * (dateTo.getFullYear() - dateFrom.getFullYear()))
+        }
+        var serviceMonths = monthDiff( new Date($("[Join Date]")), today);
+        var serviceYears = serviceMonths / 12;
+        
+        if (serviceYears >= 1 && serviceYears < 5){
+          rate = 10;
+        }
+        if (serviceYears >= 5 && serviceYears < 10){
+          rate = 15;
+        }
+        if (serviceYears >= 10){
+          rate = 15;
+          rate += 0.5 * (Number(serviceYears.toFixed(0)) - 10);
+        }
+        return rate;
+        `,
+        context
+      )
+      expect(result).toBe(10)
+    })
+
+    it("should handle test case 7", async () => {
+      const context = {
+        "P I": "Pass",
+        "PA I": "Pass",
+        "F I": "Fail",
+        "V I": "Pass",
+      }
+      const result = await processJS(
+        `if (($("[P I]") == "Pass") == true)
+              if (($("[  P I]") == "Pass") == true)
+                if (($("[F I]") == "Pass") == true)
+                  if (($("[V I]") == "Pass") == true)
+                    {return "Pass"}
+            
+            if (($("[PA I]") == "Fail") == true)
+              {return "Fail"}
+            if (($("[  P I]") == "Fail") == true)
+              {return "Fail"}
+            if (($("[F I]") == "Fail") == true)
+              {return "Fail"}
+            if (($("[V I]") == "Fail") == true)
+              {return "Fail"}
+            
+            else
+              {return ""}`,
+        context
+      )
+      expect(result).toBe("Fail")
+    })
+
+    it("should handle test case 8", async () => {
+      const context = {
+        "T L": [{ Hours: 10 }],
+        "B H": 50,
+      }
+      const result = await processJS(
+        `var totalHours = 0;
+            if (($("[T L]") != null) == true){
+            for (let i = 0; i < ($("[T L]").length); i++){
+              var individualHours = "T L." + i + ".Hours";
+              var hoursNum = Number($(individualHours));
+              totalHours += hoursNum;
+            }
+            return totalHours.toFixed(2);
+            }
+            if (($("[T L]") != null) == false) {
+              return totalHours.toFixed(2);
+            }
+        `,
+        context
+      )
+      expect(result).toBe("10.00")
+    })
+
+    it("should handle test case 9", async () => {
+      const context = {
+        "T L": [{ Hours: 10 }],
+        "B H": 50,
+      }
+      const result = await processJS(
+        `var totalHours = 0;
+            if (($("[T L]") != null) == true){
+            for (let i = 0; i < ($("[T L]").length); i++){
+              var individualHours = "T L." + i + ".Hours";
+              var hoursNum = Number($(individualHours));
+              totalHours += hoursNum;
+            }
+            return ($("[B H]") - totalHours).toFixed(2);
+            }
+            if (($("[T L]") != null) == false) {
+              return ($("[B H]") - totalHours).toFixed(2);
+            }`,
+        context
+      )
+      expect(result).toBe("40.00")
+    })
+
+    it("should handle test case 10", async () => {
+      const context = {
+        "F F": [{ "F S": 10 }],
+      }
+      const result = await processJS(
+        `var rating = 0;
+          
+          if ($("[F F]") != null){
+          for (i = 0; i < $("[F F]").length; i++){
+            var individualRating = $("F F." + i + ".F S");
+            rating += individualRating;
+            }
+            rating = (rating / $("[F F]").length);
+          }
+          return rating;
+          `,
+        context
+      )
+      expect(result).toBe(10)
+    })
+  })
 })

packages/server/src/jsRunner/utilities.ts

@@ -0,0 +1,3 @@
+export function iifeWrapper(script: string) {
+  return `(function(){\n${script}\n})();`
+}

packages/server/src/jsRunner/vm/isolated-vm.ts

@@ -7,6 +7,7 @@ import querystring from "querystring"
 
 import { BundleType, loadBundle } from "../bundles"
 import { VM } from "@budibase/types"
+import { iifeWrapper } from "../utilities"
 import environment from "../../environment"
 
 class ExecutionTimeoutError extends Error {
@@ -118,11 +119,11 @@ export class IsolatedVM implements VM {
     // 3. Process script
     // 4. Stringify the result in order to convert the result from BSON to json
     this.codeWrapper = code =>
-      `(function(){
+      iifeWrapper(`
         const data = bson.deserialize(bsonData, { validation: { utf8: false } }).data;
         const result = ${code}
         return bson.toJson(result);
-        })();`
+      `)
 
     const bsonSource = loadBundle(BundleType.BSON)
 

packages/server/src/threads/query.ts

@@ -8,6 +8,7 @@ import {
   QueryResponse,
 } from "./definitions"
 import { IsolatedVM } from "../jsRunner/vm"
+import { iifeWrapper } from "../jsRunner/utilities"
 import { getIntegration } from "../integrations"
 import { processStringSync } from "@budibase/string-templates"
 import { context, cache, auth } from "@budibase/backend-core"
@@ -127,7 +128,7 @@ class QueryRunner {
 
     // transform as required
     if (transformer) {
-      transformer = `(function(){\n${transformer}\n})();`
+      transformer = iifeWrapper(transformer)
       let vm = new IsolatedVM()
       if (datasource.source === SourceName.MONGODB) {
         vm = vm.withParsingBson(rows)