Update how snippets are fetched and enriched into context, because HBS helpers can't be async

packages/backend-core/src/context/mainContext.ts

@@ -10,7 +10,7 @@ import {
   StaticDatabases,
   DEFAULT_TENANT_ID,
 } from "../constants"
-import { Database, IdentityContext } from "@budibase/types"
+import { Database, IdentityContext, Snippet, App } from "@budibase/types"
 import { ContextMap } from "./types"
 
 let TEST_APP_ID: string | null = null
@@ -281,6 +281,27 @@ export function doInScimContext(task: any) {
   return newContext(updates, task)
 }
 
+export async function ensureSnippetContext() {
+  const ctx = getCurrentContext()
+
+  // If we've already added snippets to context, continue
+  if (!ctx || ctx.snippets) {
+    return
+  }
+
+  // Otherwise get snippets for this app and update context
+  let snippets: Snippet[] | undefined
+  const db = getAppDB()
+  if (db) {
+    const app = await db.get<App>(DocumentType.APP_METADATA)
+    snippets = app.snippets
+  }
+
+  // Always set snippets to a non-null value so that we can tell we've attempted
+  // to load snippets
+  ctx.snippets = snippets || []
+}
+
 export function getEnvironmentVariables() {
   const context = Context.get()
   if (!context.environmentVariables) {

packages/backend-core/src/context/types.ts

@@ -1,4 +1,4 @@
-import { IdentityContext, VM } from "@budibase/types"
+import { IdentityContext, Snippet, VM } from "@budibase/types"
 import { ExecutionTimeTracker } from "../timers"
 
 // keep this out of Budibase types, don't want to expose context info
@@ -12,4 +12,5 @@ export type ContextMap = {
   isMigrating?: boolean
   jsExecutionTracker?: ExecutionTimeTracker
   vm?: VM
+  snippets?: Snippet[]
 }

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -437,11 +437,11 @@ export class ExternalRequest<T extends Operation> {
     return { row: newRow, manyRelationships }
   }
 
-  processRelationshipFields(
+  async processRelationshipFields(
     table: Table,
     row: Row,
     relationships: RelationshipsJson[]
-  ): Row {
+  ): Promise<Row> {
     for (let relationship of relationships) {
       const linkedTable = this.tables[relationship.tableName]
       if (!linkedTable || !row[relationship.column]) {
@@ -457,7 +457,7 @@ export class ExternalRequest<T extends Operation> {
         }
         // process additional types
         relatedRow = processDates(table, relatedRow)
-        relatedRow = processFormulas(linkedTable, relatedRow)
+        relatedRow = await processFormulas(linkedTable, relatedRow)
         row[relationship.column][key] = relatedRow
       }
     }
@@ -521,7 +521,7 @@ export class ExternalRequest<T extends Operation> {
     return rows
   }
 
-  outputProcessing(
+  async outputProcessing(
     rows: Row[] = [],
     table: Table,
     relationships: RelationshipsJson[]
@@ -561,9 +561,12 @@ export class ExternalRequest<T extends Operation> {
     }
 
     // make sure all related rows are correct
-    let finalRowArray = Object.values(finalRows).map(row =>
-      this.processRelationshipFields(table, row, relationships)
+    let finalRowArray = []
+    for (let row of Object.values(finalRows)) {
+      finalRowArray.push(
+        await this.processRelationshipFields(table, row, relationships)
       )
+    }
 
     // process some additional types
     finalRowArray = processDates(table, finalRowArray)
@@ -934,7 +937,11 @@ export class ExternalRequest<T extends Operation> {
         processed.manyRelationships
       )
     }
-    const output = this.outputProcessing(responseRows, table, relationships)
+    const output = await this.outputProcessing(
+      responseRows,
+      table,
+      relationships
+    )
     // if reading it'll just be an array of rows, return whole thing
     if (operation === Operation.READ) {
       return (

packages/server/src/api/controllers/row/staticFormula.ts

@@ -110,7 +110,7 @@ export async function updateAllFormulasInTable(table: Table) {
       (enriched: Row) => enriched._id === row._id
     )
     if (enrichedRow) {
-      const processed = processFormulas(table, cloneDeep(row), {
+      const processed = await processFormulas(table, cloneDeep(row), {
         dynamic: false,
         contextRows: [enrichedRow],
       })
@@ -143,7 +143,7 @@ export async function finaliseRow(
     squash: false,
   })) as Row
   // use enriched row to generate formulas for saving, specifically only use as context
-  row = processFormulas(table, row, {
+  row = await processFormulas(table, row, {
     dynamic: false,
     contextRows: [enrichedRow],
   })
@@ -179,7 +179,7 @@ export async function finaliseRow(
   const response = await db.put(row)
   // for response, calculate the formulas for the enriched row
   enrichedRow._rev = response.rev
-  enrichedRow = processFormulas(table, enrichedRow, {
+  enrichedRow = await processFormulas(table, enrichedRow, {
     dynamic: false,
   })
   // this updates the related formulas in other rows based on the relations to this row

packages/server/src/db/linkedRows/index.ts

@@ -202,7 +202,8 @@ export async function attachFullLinkedDocs(
           table => table._id === linkedTableId
         )
         if (linkedTable) {
-          row[link.fieldName].push(processFormulas(linkedTable, linkedRow))
+          const processed = await processFormulas(linkedTable, linkedRow)
+          row[link.fieldName].push(processed)
         }
       }
     }

packages/server/src/jsRunner/index.ts

@@ -8,40 +8,24 @@ import {
 import { context, logging } from "@budibase/backend-core"
 import tracer from "dd-trace"
 import { IsolatedVM } from "./vm"
-import { App, DocumentType, Snippet, VM } from "@budibase/types"
 
-async function getIsolate(ctx: any): Promise<VM> {
-  // Reuse the existing isolate if one exists
-  if (ctx?.vm) {
-    return ctx.vm
-  }
-
-  // Get snippets to build into new isolate, if inside app context
-  let snippets: Snippet[] | undefined
-  const db = context.getAppDB()
-  if (db) {
-    console.log("READ APP METADATA")
-    const app = await db.get<App>(DocumentType.APP_METADATA)
-    snippets = app.snippets
-  }
-
-  // Build a new isolate
-  return new IsolatedVM({
+export function init() {
+  setJSRunner((js: string, ctx: Record<string, any>) => {
+    return tracer.trace("runJS", {}, span => {
+      try {
+        // Reuse an existing isolate from context, or make a new one
+        const bbCtx = context.getCurrentContext()
+        const vm =
+          bbCtx?.vm ||
+          new IsolatedVM({
             memoryLimit: env.JS_RUNNER_MEMORY_LIMIT,
             invocationTimeout: env.JS_PER_INVOCATION_TIMEOUT_MS,
             isolateAccumulatedTimeout: env.JS_PER_REQUEST_TIMEOUT_MS,
           })
             .withHelpers()
-    .withSnippets(snippets)
-}
+            .withSnippets(bbCtx?.snippets)
 
-export function init() {
-  setJSRunner((js: string, ctx: Record<string, any>) => {
-    return tracer.trace("runJS", {}, async span => {
-      try {
-        // Reuse an existing isolate from context, or make a new one
-        const bbCtx = context.getCurrentContext()
-        const vm = await getIsolate(bbCtx)
+        // Persist isolate in context so we can reuse it
         if (bbCtx) {
           bbCtx.vm = vm
         }
@@ -49,7 +33,7 @@ export function init() {
         // Strip helpers (an array) and snippets (a proxy isntance) as these
         // will not survive the isolated-vm barrier
         const { helpers, snippets, ...rest } = ctx
-        return vm.withContext(rest, () => vm!.execute(js))
+        return vm.withContext(rest, () => vm.execute(js))
       } catch (error: any) {
         if (error.message === "Script execution timed out.") {
           throw new JsErrorTimeout()

packages/server/src/utilities/rowProcessor/index.ts

@@ -245,7 +245,7 @@ export async function outputProcessing<T extends Row[] | Row>(
   }
 
   // process formulas after the complex types had been processed
-  enriched = processFormulas(table, enriched, { dynamic: true })
+  enriched = await processFormulas(table, enriched, { dynamic: true })
 
   if (opts.squash) {
     enriched = (await linkRows.squashLinksToPrimaryDisplay(

packages/server/src/utilities/rowProcessor/utils.ts

@@ -10,6 +10,8 @@ import {
   FieldType,
 } from "@budibase/types"
 import tracer from "dd-trace"
+import { context } from "@budibase/backend-core"
+import { getCurrentContext } from "@budibase/backend-core/src/context"
 
 interface FormulaOpts {
   dynamic?: boolean
@@ -44,16 +46,19 @@ export function fixAutoColumnSubType(
 /**
  * Looks through the rows provided and finds formulas - which it then processes.
  */
-export function processFormulas<T extends Row | Row[]>(
+export async function processFormulas<T extends Row | Row[]>(
   table: Table,
   inputRows: T,
   { dynamic, contextRows }: FormulaOpts = { dynamic: true }
-): T {
-  return tracer.trace("processFormulas", {}, span => {
+): Promise<T> {
+  return tracer.trace("processFormulas", {}, async span => {
     const numRows = Array.isArray(inputRows) ? inputRows.length : 1
     span?.addTags({ table_id: table._id, dynamic, numRows })
     const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
     if (rows) {
+      // Ensure we have snippet context
+      await context.ensureSnippetContext()
+
       for (let [column, schema] of Object.entries(table.schema)) {
         if (schema.type !== FieldType.FORMULA) {
           continue