Merge branch 'master' into chore/fix-oss

packages/server/src/db/linkedRows/index.ts

@@ -10,7 +10,10 @@ import flatten from "lodash/flatten"
 import { USER_METDATA_PREFIX } from "../utils"
 import partition from "lodash/partition"
 import { getGlobalUsersFromMetadata } from "../../utilities/global"
-import { outputProcessing, processFormulas } from "../../utilities/rowProcessor"
+import {
+  coreOutputProcessing,
+  processFormulas,
+} from "../../utilities/rowProcessor"
 import { context, features } from "@budibase/backend-core"
 import {
   ContextUser,
@@ -156,9 +159,6 @@ export async function updateLinks(args: {
 /**
  * Given a table and a list of rows this will retrieve all of the attached docs and enrich them into the row.
  * This is required for formula fields, this may only be utilised internally (for now).
- * @param table The table from which the rows originated.
- * @param rows The rows which are to be enriched.
- * @param opts optional - options like passing in a base row to use for enrichment.
  * @return returns the rows with all of the enriched relationships on it.
  */
 export async function attachFullLinkedDocs(
@@ -248,9 +248,6 @@ export type SquashTableFields = Record<string, { visibleFieldNames: string[] }>
 
 /**
  * This function will take the given enriched rows and squash the links to only contain the primary display field.
- * @param table The table from which the rows originated.
- * @param enriched The pre-enriched rows (full docs) which are to be squashed.
- * @param squashFields Per link column (key) define which columns are allowed while squashing.
  * @returns The rows after having their links squashed to only contain the ID and primary display.
  */
 export async function squashLinks<T = Row[] | Row>(
@@ -283,21 +280,18 @@ export async function squashLinks<T = Row[] | Row>(
       if (schema.type !== FieldType.LINK || !Array.isArray(row[column])) {
         continue
       }
-      const newLinks = []
+      const relatedTable = await getLinkedTable(schema.tableId, linkedTables)
-      for (const link of row[column]) {
+      if (viewSchema[column]?.columns) {
-        const linkTblId =
+        row[column] = await coreOutputProcessing(relatedTable, row[column])
-          link.tableId || getRelatedTableForField(table.schema, column)
+      }
-        const linkedTable = await getLinkedTable(linkTblId!, linkedTables)
+      row[column] = row[column].map((link: Row) => {
         const obj: any = { _id: link._id }
-        obj.primaryDisplay = getPrimaryDisplayValue(link, linkedTable)
+        obj.primaryDisplay = getPrimaryDisplayValue(link, relatedTable)
 
         if (viewSchema[column]?.columns) {
-          const enrichedLink = await outputProcessing(linkedTable, link, {
+          const squashFields = Object.entries(viewSchema[column].columns || {})
-            squash: false,
-          })
-          const squashFields = Object.entries(viewSchema[column].columns)
             .filter(([columnName, viewColumnConfig]) => {
-              const tableColumn = linkedTable.schema[columnName]
+              const tableColumn = relatedTable.schema[columnName]
               if (!tableColumn) {
                 return false
               }
@@ -315,13 +309,14 @@ export async function squashLinks<T = Row[] | Row>(
             .map(([columnName]) => columnName)
 
           for (const relField of squashFields) {
-            obj[relField] = enrichedLink[relField]
+            if (link[relField] != null) {
+              obj[relField] = link[relField]
+            }
           }
         }
 
-        newLinks.push(obj)
+        return obj
-      }
+      })
-      row[column] = newLinks
     }
   }
   return (isArray ? enrichedArray : enrichedArray[0]) as T

packages/server/src/utilities/rowProcessor/index.ts

@@ -264,6 +264,7 @@ export async function outputProcessing<T extends Row[] | Row>(
   } else {
     safeRows = rows
   }
+  // SQS returns the rows with full relationship contents
   // attach any linked row information
   let enriched = !opts.preserveLinks
     ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, {
@@ -271,11 +272,39 @@ export async function outputProcessing<T extends Row[] | Row>(
       })
     : safeRows
 
-  // make sure squash is enabled if needed
   if (!opts.squash && utils.hasCircularStructure(rows)) {
     opts.squash = true
   }
 
+  enriched = await coreOutputProcessing(table, enriched, opts)
+
+  if (opts.squash) {
+    enriched = await linkRows.squashLinks(table, enriched, {
+      fromViewId: opts?.fromViewId,
+    })
+  }
+
+  return (wasArray ? enriched : enriched[0]) as T
+}
+
+/**
+ * This function is similar to the outputProcessing function above, it makes sure that all the provided
+ * rows are ready for output, but does not have enrichment for squash capabilities which can cause performance issues.
+ * outputProcessing should be used when responding from the API, while this should be used when internally processing
+ * rows for any reason (like part of view operations).
+ */
+export async function coreOutputProcessing(
+  table: Table,
+  rows: Row[],
+  opts: {
+    preserveLinks?: boolean
+    skipBBReferences?: boolean
+    fromViewId?: string
+  } = {
+    preserveLinks: false,
+    skipBBReferences: false,
+  }
+): Promise<Row[]> {
   // process complex types: attachments, bb references...
   for (const [property, column] of Object.entries(table.schema)) {
     if (
@@ -283,7 +312,7 @@ export async function outputProcessing<T extends Row[] | Row>(
       column.type === FieldType.ATTACHMENT_SINGLE ||
       column.type === FieldType.SIGNATURE_SINGLE
     ) {
-      for (const row of enriched) {
+      for (const row of rows) {
         if (row[property] == null) {
           continue
         }
@@ -308,7 +337,7 @@ export async function outputProcessing<T extends Row[] | Row>(
       !opts.skipBBReferences &&
       column.type == FieldType.BB_REFERENCE
     ) {
-      for (const row of enriched) {
+      for (const row of rows) {
         row[property] = await processOutputBBReferences(
           row[property],
           column.subtype
@@ -318,14 +347,14 @@ export async function outputProcessing<T extends Row[] | Row>(
       !opts.skipBBReferences &&
       column.type == FieldType.BB_REFERENCE_SINGLE
     ) {
-      for (const row of enriched) {
+      for (const row of rows) {
         row[property] = await processOutputBBReference(
           row[property],
           column.subtype
         )
       }
     } else if (column.type === FieldType.DATETIME && column.timeOnly) {
-      for (const row of enriched) {
+      for (const row of rows) {
         if (row[property] instanceof Date) {
           const hours = row[property].getUTCHours().toString().padStart(2, "0")
           const minutes = row[property]
@@ -340,7 +369,7 @@ export async function outputProcessing<T extends Row[] | Row>(
         }
       }
     } else if (column.type === FieldType.LINK) {
-      for (let row of enriched) {
+      for (let row of rows) {
         // if relationship is empty - remove the array, this has been part of the API for some time
         if (Array.isArray(row[property]) && row[property].length === 0) {
           delete row[property]
@@ -350,17 +379,12 @@ export async function outputProcessing<T extends Row[] | Row>(
   }
 
   // process formulas after the complex types had been processed
-  enriched = await processFormulas(table, enriched, { dynamic: true })
+  rows = await processFormulas(table, rows, { dynamic: true })
 
-  if (opts.squash) {
-    enriched = await linkRows.squashLinks(table, enriched, {
-      fromViewId: opts?.fromViewId,
-    })
-  }
   // remove null properties to match internal API
   const isExternal = isExternalTableID(table._id!)
   if (isExternal || (await features.flags.isEnabled("SQS"))) {
-    for (const row of enriched) {
+    for (const row of rows) {
       for (const key of Object.keys(row)) {
         if (row[key] === null) {
           delete row[key]
@@ -388,7 +412,7 @@ export async function outputProcessing<T extends Row[] | Row>(
     const fields = [...tableFields, ...protectedColumns].map(f =>
       f.toLowerCase()
     )
-    for (const row of enriched) {
+    for (const row of rows) {
       for (const key of Object.keys(row)) {
         if (!fields.includes(key.toLowerCase())) {
           delete row[key]
@@ -397,5 +421,5 @@ export async function outputProcessing<T extends Row[] | Row>(
     }
   }
 
-  return (wasArray ? enriched : enriched[0]) as T
+  return rows
 }