Merge branches 'master' and 'budi-7664-sqs-self-host-ui-for-detecting-lack-of-sqs-support-2' of github.com:budibase/budibase into budi-7664-sqs-self-host-ui-for-detecting-lack-of-sqs-support-2

.eslintrc.json

@@ -45,6 +45,16 @@
         "no-prototype-builtins": "off",
         "local-rules/no-budibase-imports": "error"
       }
+    },
+    {
+      "files": [
+        "packages/builder/**/*",
+        "packages/client/**/*",
+        "packages/frontend-core/**/*"
+      ],
+      "rules": {
+        "no-console": ["error", { "allow": ["warn", "error", "debug"] } ]
+      }
     }
   ],
   "rules": {

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.16.0",
+  "version": "2.16.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/package.json

@@ -21,7 +21,7 @@
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
-    "@budibase/nano": "10.1.4",
+    "@budibase/nano": "10.1.5",
     "@budibase/pouchdb-replication-stream": "1.2.10",
     "@budibase/shared-core": "0.0.0",
     "@budibase/types": "0.0.0",

packages/backend-core/src/objectStore/cloudfront.ts

@@ -23,7 +23,7 @@ const getCloudfrontSignParams = () => {
   return {
     keypairId: env.CLOUDFRONT_PUBLIC_KEY_ID!,
     privateKeyString: getPrivateKey(),
-    expireTime: new Date().getTime() + 1000 * 60 * 60, // 1 hour
+    expireTime: new Date().getTime() + 1000 * 60 * 60 * 24, // 1 day
   }
 }
 

packages/builder/src/builderStore/dataBinding.js

@@ -364,7 +364,6 @@ const getContextBindings = (asset, componentId) => {
  * Generates a set of bindings for a given component context
  */
 const generateComponentContextBindings = (asset, componentContext) => {
-  console.log("Hello ")
   const { component, definition, contexts } = componentContext
   if (!component || !definition || !contexts?.length) {
     return []

packages/builder/src/builderStore/websocket.js

@@ -21,7 +21,7 @@ export const createBuilderWebsocket = appId => {
     })
   })
   socket.on("connect_error", err => {
-    console.log("Failed to connect to builder websocket:", err.message)
+    console.error("Failed to connect to builder websocket:", err.message)
   })
   socket.on("disconnect", () => {
     userStore.actions.reset()

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -373,6 +373,7 @@
   confirmText="Save"
   onConfirm={saveRelationship}
   disabled={!valid}
+  size="L"
 >
   <div class="headings">
     <Detail>Tables</Detail>

packages/builder/src/components/common/CodeEditor/index.js

@@ -312,7 +312,7 @@ export const insertBinding = (view, from, to, text, mode) => {
   } else if (mode.name == "handlebars") {
     parsedInsert = hbInsert(view.state.doc?.toString(), from, to, text)
   } else {
-    console.log("Unsupported")
+    console.warn("Unsupported")
     return
   }
 

packages/builder/src/components/common/RelationshipSelector.svelte

@@ -17,7 +17,7 @@
 </script>
 
 <div class="relationship-container">
-  <div class="relationship-part">
+  <div class="relationship-type">
     <Select
       disabled={linkEditDisabled}
       bind:value={relationshipPart1}
@@ -39,7 +39,7 @@
   </div>
 </div>
 <div class="relationship-container">
-  <div class="relationship-part">
+  <div class="relationship-type">
     <Select
       disabled={linkEditDisabled}
       bind:value={relationshipPart2}
@@ -79,6 +79,10 @@
   }
 
   .relationship-part {
-    flex-basis: 60%;
+    flex-basis: 70%;
+  }
+
+  .relationship-type {
+    flex-basis: 30%;
   }
 </style>

packages/builder/src/components/common/bindings/BindingPicker.svelte

@@ -47,9 +47,10 @@
     })
   $: filteredHelpers = helpers?.filter(helper => {
     return (
-      !search ||
+      (!search ||
         helper.label.match(searchRgx) ||
-      helper.description.match(searchRgx)
+        helper.description.match(searchRgx)) &&
+      (mode.name !== "javascript" || helper.allowsJs)
     )
   })
 

packages/builder/src/components/portal/onboarding/TourPopover.svelte

@@ -67,7 +67,7 @@
         }))
         navigateStep(target)
       } else {
-        console.log("Could not retrieve step")
+        console.warn("Could not retrieve step")
       }
     } else {
       if (typeof tourStep.onComplete === "function") {

packages/builder/src/components/portal/onboarding/tourHandler.js

@@ -3,11 +3,11 @@ import { get } from "svelte/store"
 
 const registerNode = async (node, tourStepKey) => {
   if (!node) {
-    console.log("Tour Handler - an anchor node is required")
+    console.warn("Tour Handler - an anchor node is required")
   }
 
   if (!get(store).tourKey) {
-    console.log("Tour Handler - No active tour ", tourStepKey, node)
+    console.error("Tour Handler - No active tour ", tourStepKey, node)
     return
   }
 

packages/builder/src/components/portal/onboarding/tours.js

@@ -45,7 +45,7 @@ const endUserOnboarding = async ({ skipped = false } = {}) => {
         onboarding: false,
       }))
     } catch (e) {
-      console.log("Onboarding failed", e)
+      console.error("Onboarding failed", e)
       return false
     }
     return true

packages/builder/src/constants/completions.js

@@ -1,4 +1,4 @@
-import { getManifest } from "@budibase/string-templates"
+import { getManifest, helpersToRemoveForJs } from "@budibase/string-templates"
 
 export function handlebarsCompletions() {
   const manifest = getManifest()
@@ -11,6 +11,9 @@ export function handlebarsCompletions() {
       label: helperName,
       displayText: helperName,
       description: helperConfig.description,
+      allowsJs:
+        !helperConfig.requiresBlock &&
+        !helpersToRemoveForJs.includes(helperName),
     }))
   )
 }

packages/builder/src/helpers/urlStateSync.js

@@ -52,7 +52,7 @@ export const syncURLToState = options => {
   let cachedPage = get(routify.page)
   let previousParamsHash = null
   let debug = false
-  const log = (...params) => debug && console.log(`[${urlParam}]`, ...params)
+  const log = (...params) => debug && console.debug(`[${urlParam}]`, ...params)
 
   // Navigate to a certain URL
   const gotoUrl = (url, params) => {

packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte

@@ -107,7 +107,7 @@
       return
     }
     if (!prodAppId) {
-      console.log("Application id required")
+      console.error("Application id required")
       return
     }
     await usersFetch.update({

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Screen/GeneralPanel.svelte

@@ -66,7 +66,7 @@
     try {
       await store.actions.screens.updateSetting(get(selectedScreen), key, value)
     } catch (error) {
-      console.log(error)
+      console.error(error)
       notifications.error("Error saving screen settings")
     }
   }

packages/builder/src/pages/builder/app/[application]/design/_components/NewScreen/CreateScreenModal.svelte

@@ -71,7 +71,7 @@
       $goto(`./${screenId}`)
       store.actions.screens.select(screenId)
     } catch (error) {
-      console.log(error)
+      console.error(error)
       notifications.error("Error creating screens")
     }
   }

packages/builder/src/pages/builder/auth/login.svelte

@@ -31,7 +31,7 @@
   async function login() {
     form.validate()
     if (Object.keys(errors).length > 0) {
-      console.log("errors", errors)
+      console.error("errors", errors)
       return
     }
     try {

packages/client/src/components/app/embedded-map/EmbeddedMap.svelte

@@ -307,7 +307,7 @@
       // Reset view
       resetView()
     } catch (e) {
-      console.log("There was a problem with the map", e)
+      console.error("There was a problem with the map", e)
     }
   }
 

packages/client/src/components/app/forms/CodeScanner.svelte

@@ -61,7 +61,7 @@
           resolve({ initialised: true })
         })
         .catch(err => {
-          console.log("There was a problem scanning the image", err)
+          console.error("There was a problem scanning the image", err)
           resolve({ initialised: false })
         })
     })

packages/client/src/stores/org.js

@@ -14,7 +14,7 @@ const createOrgStore = () => {
       const settingsConfigDoc = await API.getTenantConfig(tenantId)
       set({ logoUrl: settingsConfigDoc.config.logoUrl })
     } catch (e) {
-      console.log("Could not init org ", e)
+      console.error("Could not init org ", e)
     }
   }
 

packages/frontend-core/src/components/grid/lib/websocket.js

@@ -29,7 +29,7 @@ export const createGridWebsocket = context => {
     connectToDatasource(get(datasource))
   })
   socket.on("connect_error", err => {
-    console.log("Failed to connect to grid websocket:", err.message)
+    console.error("Failed to connect to grid websocket:", err.message)
   })
 
   // User events

packages/server/scripts/test.sh

@@ -5,10 +5,10 @@ if [[ -n $CI ]]
 then
   # Running in ci, where resources are limited
   export NODE_OPTIONS="--max-old-space-size=4096"
-  echo "jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail"
+  echo "jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail $@"
-  jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail
+  jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail $@
 else
   # --maxWorkers performs better in development
-  echo "jest --coverage --maxWorkers=2 --forceExit"
+  echo "jest --coverage --maxWorkers=2 --forceExit $@"
-  jest --coverage --maxWorkers=2 --forceExit
+  jest --coverage --maxWorkers=2 --forceExit $@
 fi

packages/server/src/automations/automationUtils.ts

@@ -5,7 +5,7 @@ import {
 } from "@budibase/string-templates"
 import sdk from "../sdk"
 import { Row } from "@budibase/types"
-import { LoopStep, LoopStepType, LoopInput } from "../definitions/automations"
+import { LoopInput, LoopStep, LoopStepType } from "../definitions/automations"
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -128,23 +128,28 @@ export function substituteLoopStep(hbsString: string, substitute: string) {
 }
 
 export function stringSplit(value: string | string[]) {
-  if (value == null || Array.isArray(value)) {
+  if (value == null) {
-    return value || []
+    return []
-  }
-  if (value.split("\n").length > 1) {
-    value = value.split("\n")
-  } else {
-    value = value.split(",")
   }
+  if (Array.isArray(value)) {
     return value
+  }
+  if (typeof value !== "string") {
+    throw new Error(`Unable to split value of type ${typeof value}: ${value}`)
+  }
+  const splitOnNewLine = value.split("\n")
+  if (splitOnNewLine.length > 1) {
+    return splitOnNewLine
+  }
+  return value.split(",")
 }
 
-export function typecastForLooping(loopStep: LoopStep, input: LoopInput) {
+export function typecastForLooping(input: LoopInput) {
   if (!input || !input.binding) {
     return null
   }
   try {
-    switch (loopStep.inputs.option) {
+    switch (input.option) {
       case LoopStepType.ARRAY:
         if (typeof input.binding === "string") {
           return JSON.parse(input.binding)

packages/server/src/automations/tests/loop.spec.ts

@@ -3,11 +3,13 @@ import * as triggers from "../triggers"
 import { loopAutomation } from "../../tests/utilities/structures"
 import { context } from "@budibase/backend-core"
 import * as setup from "./utilities"
+import { Row, Table } from "@budibase/types"
+import { LoopInput, LoopStepType } from "../../definitions/automations"
 
 describe("Attempt to run a basic loop automation", () => {
   let config = setup.getConfig(),
-    table: any,
+    table: Table,
-    row: any
+    row: Row
 
   beforeEach(async () => {
     await automation.init()
@@ -18,12 +20,12 @@ describe("Attempt to run a basic loop automation", () => {
 
   afterAll(setup.afterAll)
 
-  async function runLoop(loopOpts?: any) {
+  async function runLoop(loopOpts?: LoopInput) {
     const appId = config.getAppId()
     return await context.doInAppContext(appId, async () => {
       const params = { fields: { appId } }
       return await triggers.externalTrigger(
-        loopAutomation(table._id, loopOpts),
+        loopAutomation(table._id!, loopOpts),
         params,
         { getResponses: true }
       )
@@ -37,9 +39,17 @@ describe("Attempt to run a basic loop automation", () => {
 
   it("test a loop with a string", async () => {
     const resp = await runLoop({
-      type: "String",
+      option: LoopStepType.STRING,
       binding: "a,b,c",
     })
     expect(resp.steps[2].outputs.iterations).toBe(3)
   })
+
+  it("test a loop with a binding that returns an integer", async () => {
+    const resp = await runLoop({
+      option: LoopStepType.ARRAY,
+      binding: "{{ 1 }}",
+    })
+    expect(resp.steps[2].outputs.iterations).toBe(1)
+  })
 })

packages/server/src/automations/triggers.ts

@@ -9,7 +9,7 @@ import * as utils from "./utils"
 import env from "../environment"
 import { context, db as dbCore } from "@budibase/backend-core"
 import { Automation, Row, AutomationData, AutomationJob } from "@budibase/types"
-import { executeSynchronously } from "../threads/automation"
+import { executeInThread } from "../threads/automation"
 
 export const TRIGGER_DEFINITIONS = definitions
 const JOB_OPTS = {
@@ -117,8 +117,7 @@ export async function externalTrigger(
       appId: context.getAppId(),
       automation,
     }
-    const job = { data } as AutomationJob
+    return executeInThread({ data } as AutomationJob)
-    return executeSynchronously(job)
   } else {
     return automationQueue.add(data, JOB_OPTS)
   }

packages/server/src/automations/unitTests/automationUtils.spec.ts

@@ -1,10 +1,15 @@
-const automationUtils = require("../automationUtils")
+import { LoopStep, LoopStepType } from "../../definitions/automations"
+import {
+  typecastForLooping,
+  cleanInputValues,
+  substituteLoopStep,
+} from "../automationUtils"
 
 describe("automationUtils", () => {
   describe("substituteLoopStep", () => {
     it("should allow multiple loop binding substitutes", () => {
       expect(
-        automationUtils.substituteLoopStep(
+        substituteLoopStep(
           `{{ loop.currentItem._id }} {{ loop.currentItem._id }} {{ loop.currentItem._id }}`,
           "step.2"
         )
@@ -15,7 +20,7 @@ describe("automationUtils", () => {
 
     it("should handle not subsituting outside of curly braces", () => {
       expect(
-        automationUtils.substituteLoopStep(
+        substituteLoopStep(
           `loop {{ loop.currentItem._id }}loop loop{{ loop.currentItem._id }}loop`,
           "step.2"
         )
@@ -28,37 +33,20 @@ describe("automationUtils", () => {
   describe("typeCastForLooping", () => {
     it("should parse to correct type", () => {
       expect(
-        automationUtils.typecastForLooping(
+        typecastForLooping({ option: LoopStepType.ARRAY, binding: [1, 2, 3] })
-          { inputs: { option: "Array" } },
-          { binding: [1, 2, 3] }
-        )
       ).toEqual([1, 2, 3])
       expect(
-        automationUtils.typecastForLooping(
+        typecastForLooping({ option: LoopStepType.ARRAY, binding: "[1,2,3]" })
-          { inputs: { option: "Array" } },
-          { binding: "[1, 2, 3]" }
-        )
       ).toEqual([1, 2, 3])
       expect(
-        automationUtils.typecastForLooping(
+        typecastForLooping({ option: LoopStepType.STRING, binding: [1, 2, 3] })
-          { inputs: { option: "String" } },
-          { binding: [1, 2, 3] }
-        )
       ).toEqual("1,2,3")
     })
     it("should handle null values", () => {
       // expect it to handle where the binding is null
-      expect(
+      expect(typecastForLooping({ option: LoopStepType.ARRAY })).toEqual(null)
-        automationUtils.typecastForLooping(
-          { inputs: { option: "Array" } },
-          { binding: null }
-        )
-      ).toEqual(null)
       expect(() =>
-        automationUtils.typecastForLooping(
+        typecastForLooping({ option: LoopStepType.ARRAY, binding: "test" })
-          { inputs: { option: "Array" } },
-          { binding: "test" }
-        )
       ).toThrow()
     })
   })
@@ -80,7 +68,7 @@ describe("automationUtils", () => {
         },
       }
       expect(
-        automationUtils.cleanInputValues(
+        cleanInputValues(
           {
             row: {
               relationship: `[{"_id": "ro_ta_users_us_3"}]`,
@@ -113,7 +101,7 @@ describe("automationUtils", () => {
         },
       }
       expect(
-        automationUtils.cleanInputValues(
+        cleanInputValues(
           {
             row: {
               relationship: `ro_ta_users_us_3`,

packages/server/src/definitions/automations.ts

@@ -6,14 +6,14 @@ export enum LoopStepType {
 }
 
 export interface LoopStep extends AutomationStep {
-  inputs: {
+  inputs: LoopInput
-    option: LoopStepType
-    [key: string]: any
-  }
 }
 
 export interface LoopInput {
-  binding: string[] | string
+  option: LoopStepType
+  binding?: string[] | string | number[]
+  iterations?: string
+  failure?: any
 }
 
 export interface TriggerOutput {

packages/server/src/tests/utilities/structures.ts

@@ -23,6 +23,7 @@ import {
   TableSourceType,
   Query,
 } from "@budibase/types"
+import { LoopInput, LoopStepType } from "../../definitions/automations"
 
 const { BUILTIN_ROLE_IDS } = roles
 
@@ -204,10 +205,13 @@ export function serverLogAutomation(appId?: string): Automation {
   }
 }
 
-export function loopAutomation(tableId: string, loopOpts?: any): Automation {
+export function loopAutomation(
+  tableId: string,
+  loopOpts?: LoopInput
+): Automation {
   if (!loopOpts) {
     loopOpts = {
-      option: "Array",
+      option: LoopStepType.ARRAY,
       binding: "{{ steps.1.rows }}",
     }
   }

packages/server/src/threads/automation.ts

@@ -43,22 +43,19 @@ const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
 
 function getLoopIterations(loopStep: LoopStep) {
-  let binding = loopStep.inputs.binding
+  const binding = loopStep.inputs.binding
   if (!binding) {
     return 0
   }
-  const isString = typeof binding === "string"
   try {
-    if (isString) {
+    const json = typeof binding === "string" ? JSON.parse(binding) : binding
-      binding = JSON.parse(binding)
+    if (Array.isArray(json)) {
+      return json.length
     }
   } catch (err) {
     // ignore error - wasn't able to parse
   }
-  if (Array.isArray(binding)) {
+  if (typeof binding === "string") {
-    return binding.length
-  }
-  if (isString) {
     return automationUtils.stringSplit(binding).length
   }
   return 0
@@ -256,7 +253,7 @@ class Orchestrator {
         this._context.env = await sdkUtils.getEnvironmentVariables()
         let automation = this._automation
         let stopped = false
-        let loopStep: AutomationStep | undefined = undefined
+        let loopStep: LoopStep | undefined = undefined
 
         let stepCount = 0
         let loopStepNumber: any = undefined
@@ -311,7 +308,7 @@ class Orchestrator {
 
             stepCount++
             if (step.stepId === LOOP_STEP_ID) {
-              loopStep = step
+              loopStep = step as LoopStep
               loopStepNumber = stepCount
               continue
             }
@@ -331,7 +328,6 @@ class Orchestrator {
                 }
                 try {
                   loopStep.inputs.binding = automationUtils.typecastForLooping(
-                    loopStep as LoopStep,
                     loopStep.inputs as LoopInput
                   )
                 } catch (err) {
@@ -348,7 +344,7 @@ class Orchestrator {
                   loopStep = undefined
                   break
                 }
-                let item = []
+                let item: any[] = []
                 if (
                   typeof loopStep.inputs.binding === "string" &&
                   loopStep.inputs.option === "String"
@@ -399,7 +395,8 @@ class Orchestrator {
 
                 if (
                   index === env.AUTOMATION_MAX_ITERATIONS ||
-                  index === parseInt(loopStep.inputs.iterations)
+                  (loopStep.inputs.iterations &&
+                    index === parseInt(loopStep.inputs.iterations))
                 ) {
                   this.updateContextAndOutput(
                     loopStepNumber,
@@ -615,7 +612,7 @@ export function execute(job: Job<AutomationData>, callback: WorkerCallback) {
   })
 }
 
-export function executeSynchronously(job: Job) {
+export async function executeInThread(job: Job<AutomationData>) {
   const appId = job.data.event.appId
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")
@@ -627,10 +624,10 @@ export function executeSynchronously(job: Job) {
     }, job.data.event.timeout || 12000)
   })
 
-  return context.doInAppContext(appId, async () => {
+  return await context.doInAppContext(appId, async () => {
     const envVars = await sdkUtils.getEnvironmentVariables()
     // put into automation thread for whole context
-    return context.doInEnvironmentContext(envVars, async () => {
+    return await context.doInEnvironmentContext(envVars, async () => {
       const automationOrchestrator = new Orchestrator(job)
       return await Promise.race([
         automationOrchestrator.execute(),

packages/string-templates/package.json

@@ -25,7 +25,7 @@
     "manifest": "node ./scripts/gen-collection-info.js"
   },
   "dependencies": {
-    "@budibase/handlebars-helpers": "^0.13.0",
+    "@budibase/handlebars-helpers": "^0.13.1",
     "dayjs": "^1.10.8",
     "handlebars": "^4.7.6",
     "lodash.clonedeep": "^4.5.0",

packages/string-templates/scripts/gen-collection-info.js

@@ -10,8 +10,8 @@ const marked = require("marked")
  * https://github.com/budibase/handlebars-helpers
  */
 const { join } = require("path")
+const path = require("path")
 
-const DIRECTORY = join(__dirname, "..", "..", "..")
 const COLLECTIONS = [
   "math",
   "array",
@@ -115,6 +115,8 @@ function getCommentInfo(file, func) {
     docs.example = docs.example.replace("product", "multiply")
   }
   docs.description = blocks[0].trim()
+  docs.acceptsBlock = docs.tags.some(el => el.title === "block")
+  docs.acceptsInline = docs.tags.some(el => el.title === "inline")
   return docs
 }
 
@@ -127,7 +129,7 @@ function run() {
   const foundNames = []
   for (let collection of COLLECTIONS) {
     const collectionFile = fs.readFileSync(
-      `${DIRECTORY}/node_modules/${HELPER_LIBRARY}/lib/${collection}.js`,
+      `${path.dirname(require.resolve(HELPER_LIBRARY))}/lib/${collection}.js`,
       "utf8"
     )
     const collectionInfo = {}
@@ -159,6 +161,7 @@ function run() {
         numArgs: args.length,
         example: jsDocInfo.example || undefined,
         description: jsDocInfo.description,
+        requiresBlock: jsDocInfo.acceptsBlock && !jsDocInfo.acceptsInline,
       })
     }
     outputJSON[collection] = collectionInfo

packages/string-templates/src/conversion/index.js

@@ -1,4 +1,4 @@
-const { getHelperList } = require("../helpers")
+const { getJsHelperList } = require("../helpers")
 
 function getLayers(fullBlock) {
   let layers = []
@@ -109,7 +109,7 @@ module.exports.convertHBSBlock = (block, blockNumber) => {
   const layers = getLayers(block)
 
   let value = null
-  const list = getHelperList()
+  const list = getJsHelperList()
   for (let layer of layers) {
     const parts = splitBySpace(layer)
     if (value || parts.length > 1 || list[parts[0]]) {

packages/string-templates/src/helpers/date.js

@@ -115,7 +115,7 @@ module.exports.duration = (str, pattern, format) => {
   setLocale(config.str, config.pattern)
 
   const duration = dayjs.duration(config.str, config.pattern)
-  if (!isOptions(format)) {
+  if (format && !isOptions(format)) {
     return duration.format(format)
   } else {
     return duration.humanize()

packages/string-templates/src/helpers/index.js

@@ -7,7 +7,7 @@ const {
   HelperFunctionBuiltin,
   LITERAL_MARKER,
 } = require("./constants")
-const { getHelperList } = require("./list")
+const { getJsHelperList } = require("./list")
 
 const HTML_SWAPS = {
   "<": "&lt;",
@@ -97,4 +97,4 @@ module.exports.unregisterAll = handlebars => {
   externalHandlebars.unregisterAll(handlebars)
 }
 
-module.exports.getHelperList = getHelperList
+module.exports.getJsHelperList = getJsHelperList

packages/string-templates/src/helpers/javascript.js

@@ -1,7 +1,7 @@
 const { atob } = require("../utilities")
 const cloneDeep = require("lodash.clonedeep")
 const { LITERAL_MARKER } = require("../helpers/constants")
-const { getHelperList } = require("./list")
+const { getJsHelperList } = require("./list")
 
 // The method of executing JS scripts depends on the bundle being built.
 // This setter is used in the entrypoint (either index.cjs or index.mjs).
@@ -49,7 +49,7 @@ module.exports.processJS = (handlebars, context) => {
     // app context.
     const sandboxContext = {
       $: path => getContextValue(path, cloneDeep(context)),
-      helpers: getHelperList(),
+      helpers: getJsHelperList(),
     }
 
     // Create a sandbox with our context and run the JS

packages/string-templates/src/helpers/list.js

@@ -3,7 +3,10 @@ const helperList = require("@budibase/handlebars-helpers")
 
 let helpers = undefined
 
-module.exports.getHelperList = () => {
+const helpersToRemoveForJs = ["sortBy"]
+module.exports.helpersToRemoveForJs = helpersToRemoveForJs
+
+module.exports.getJsHelperList = () => {
   if (helpers) {
     return helpers
   }
@@ -15,12 +18,17 @@ module.exports.getHelperList = () => {
   }
   for (let collection of constructed) {
     for (let [key, func] of Object.entries(collection)) {
-      helpers[key] = func
+      // Handlebars injects the hbs options to the helpers by default. We are adding an empty {} as a last parameter to simulate it
+      helpers[key] = (...props) => func(...props, {})
     }
   }
   for (let key of Object.keys(externalHandlebars.addedHelpers)) {
     helpers[key] = externalHandlebars.addedHelpers[key]
   }
+
+  for (const toRemove of helpersToRemoveForJs) {
+    delete helpers[toRemove]
+  }
   Object.freeze(helpers)
   return helpers
 }

packages/string-templates/src/index.cjs

@@ -20,6 +20,7 @@ module.exports.findHBSBlocks = templates.findHBSBlocks
 module.exports.convertToJS = templates.convertToJS
 module.exports.setJSRunner = templates.setJSRunner
 module.exports.FIND_ANY_HBS_REGEX = templates.FIND_ANY_HBS_REGEX
+module.exports.helpersToRemoveForJs = templates.helpersToRemoveForJs
 
 if (!process.env.NO_JS) {
   const { VM } = require("vm2")

packages/string-templates/src/index.js

@@ -10,6 +10,7 @@ const {
 } = require("./utilities")
 const { convertHBSBlock } = require("./conversion")
 const javascript = require("./helpers/javascript")
+const { helpersToRemoveForJs } = require("./helpers/list")
 
 const hbsInstance = handlebars.create()
 registerAll(hbsInstance)
@@ -394,3 +395,4 @@ module.exports.convertToJS = hbs => {
 }
 
 module.exports.FIND_ANY_HBS_REGEX = FIND_ANY_HBS_REGEX
+module.exports.helpersToRemoveForJs = helpersToRemoveForJs

packages/string-templates/src/index.mjs

@@ -21,6 +21,7 @@ export const findHBSBlocks = templates.findHBSBlocks
 export const convertToJS = templates.convertToJS
 export const setJSRunner = templates.setJSRunner
 export const FIND_ANY_HBS_REGEX = templates.FIND_ANY_HBS_REGEX
+export const helpersToRemoveForJs = templates.helpersToRemoveForJs
 
 if (process && !process.env.NO_JS) {
   /**

packages/string-templates/test/manifest.spec.js

@@ -16,21 +16,55 @@ jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
 })
 
 const fs = require("fs")
-const { processString } = require("../src/index.cjs")
+const {
+  processString,
+  convertToJS,
+  processStringSync,
+  encodeJSBinding,
+} = require("../src/index.cjs")
 
 const tk = require("timekeeper")
+const { getJsHelperList } = require("../src/helpers")
+
 tk.freeze("2021-01-21T12:00:00")
 
+const processJS = (js, context) => {
+  return processStringSync(encodeJSBinding(js), context)
+}
+
 const manifest = JSON.parse(
   fs.readFileSync(require.resolve("../manifest.json"), "utf8")
 )
 
 const collections = Object.keys(manifest)
 const examples = collections.reduce((acc, collection) => {
-  const functions = Object.keys(manifest[collection]).filter(
+  const functions = Object.entries(manifest[collection])
-    fnc => manifest[collection][fnc].example
+    .filter(([_, details]) => details.example)
-  )
+    .map(([name, details]) => {
-  if (functions.length) {
+      const example = details.example
+      let [hbs, js] = example.split("->").map(x => x.trim())
+      if (!js) {
+        // The function has no return value
+        return
+      }
+
+      // Trim 's
+      js = js.replace(/^\'|\'$/g, "")
+      if ((parsedExpected = tryParseJson(js))) {
+        if (Array.isArray(parsedExpected)) {
+          if (typeof parsedExpected[0] === "object") {
+            js = JSON.stringify(parsedExpected)
+          } else {
+            js = parsedExpected.join(",")
+          }
+        }
+      }
+      const requiresHbsBody = details.requiresBlock
+      return [name, { hbs, js, requiresHbsBody }]
+    })
+    .filter(x => !!x)
+
+  if (Object.keys(functions).length) {
     acc[collection] = functions
   }
   return acc
@@ -55,11 +89,7 @@ function tryParseJson(str) {
 describe("manifest", () => {
   describe("examples are valid", () => {
     describe.each(Object.keys(examples))("%s", collection => {
-      it.each(examples[collection])("%s", async func => {
+      it.each(examples[collection])("%s", async (_, { hbs, js }) => {
-        const example = manifest[collection][func].example
-
-        let [hbs, js] = example.split("->").map(x => x.trim())
-
         const context = {
           double: i => i * 2,
           isString: x => typeof x === "string",
@@ -71,23 +101,40 @@ describe("manifest", () => {
           context[`array${i}`] = JSON.parse(arrayString.replace(/\'/g, '"'))
         })
 
-        if (js === undefined) {
+        let result = await processString(hbs, context)
-          // The function has no return value
+        result = result.replace(/ /g, " ")
-          return
+        expect(result).toEqual(js)
+      })
+    })
+  })
+
+  describe("can be parsed and run as js", () => {
+    const jsHelpers = getJsHelperList()
+    const jsExamples = Object.keys(examples).reduce((acc, v) => {
+      acc[v] = examples[v].filter(([key]) => jsHelpers[key])
+      return acc
+    }, {})
+
+    describe.each(Object.keys(jsExamples))("%s", collection => {
+      it.each(
+        jsExamples[collection].filter(
+          ([_, { requiresHbsBody }]) => !requiresHbsBody
+        )
+      )("%s", async (_, { hbs, js }) => {
+        const context = {
+          double: i => i * 2,
+          isString: x => typeof x === "string",
         }
 
-        let result = await processString(hbs, context)
+        const arrays = hbs.match(/\[[^/\]]+\]/)
-        // Trim 's
+        arrays?.forEach((arrayString, i) => {
-        js = js.replace(/^\'|\'$/g, "")
+          hbs = hbs.replace(new RegExp(escapeRegExp(arrayString)), `array${i}`)
-        if ((parsedExpected = tryParseJson(js))) {
+          context[`array${i}`] = JSON.parse(arrayString.replace(/\'/g, '"'))
-          if (Array.isArray(parsedExpected)) {
+        })
-            if (typeof parsedExpected[0] === "object") {
+
-              js = JSON.stringify(parsedExpected)
+        let convertedJs = convertToJS(hbs)
-            } else {
+
-              js = parsedExpected.join(",")
+        let result = processJS(convertedJs, context)
-            }
-          }
-        }
         result = result.replace(/ /g, " ")
         expect(result).toEqual(js)
       })

packages/types/package.json

@@ -15,7 +15,7 @@
   },
   "jest": {},
   "devDependencies": {
-    "@budibase/nano": "10.1.4",
+    "@budibase/nano": "10.1.5",
     "@types/koa": "2.13.4",
     "@types/pouchdb": "6.4.0",
     "@types/redlock": "4.0.3",

yarn.lock

@@ -2031,10 +2031,10 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/handlebars-helpers@^0.13.0":
+"@budibase/handlebars-helpers@^0.13.1":
-  version "0.13.0"
+  version "0.13.1"
-  resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.0.tgz#224333d14e3900b7dacf48286af1e624a9fd62ea"
+  resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.1.tgz#d02e73c0df8305cd675e70dc37f8427eb0842080"
-  integrity sha512-g8+sFrMNxsIDnK+MmdUICTVGr6ReUFtnPp9hJX0VZwz1pN3Ynolpk/Qbu6rEWAvoU1sEqY1mXr9uo/+kEfeGbQ==
+  integrity sha512-v4RbXhr3igvK3i2pj5cNltu/4NMxdPIzcUt/o0RoInhesNH1VSLRdweSFr6/Y34fsCR5jHZ6vltdcz2RgrTKgw==
   dependencies:
     get-object "^0.2.0"
     get-value "^3.0.1"
@@ -2050,10 +2050,10 @@
     to-gfm-code-block "^0.1.1"
     uuid "^9.0.1"
 
-"@budibase/nano@10.1.4":
+"@budibase/nano@10.1.5":
-  version "10.1.4"
+  version "10.1.5"
-  resolved "https://registry.yarnpkg.com/@budibase/nano/-/nano-10.1.4.tgz#5c2670d0b4c12d736ddd6581c57d47c0aa45efad"
+  resolved "https://registry.yarnpkg.com/@budibase/nano/-/nano-10.1.5.tgz#eeaded7bfc707ecabf8fde604425b865a90c06ec"
-  integrity sha512-J+IVaAljGideDvJss/AUxXA1599HEIUJo5c0LLlmc1KMA3GZWZjyX+w2fxAw3qF7hqFvX+qAStQgdcD3+/GPMA==
+  integrity sha512-q1eKIsYKo+iK17zsJYd3VBl+5ufQMPpHYLec0wVsid8wnJVrTQk7RNpBlBUn/EDgXM7t8XNNHlERqHu+CxJu8Q==
   dependencies:
     "@types/tough-cookie" "^4.0.2"
     axios "^1.1.3"