From c41c25a6037f69797ca313eca602f211ff8a9a7f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 27 Sep 2024 12:19:24 +0100 Subject: [PATCH] This improves the performance of views when the enriched relationships feature has been enabled. There was an issue that caused the squashLinks and outputProcessing to loop onto each other until things broke down - this fixes the issue. --- packages/server/src/db/linkedRows/index.ts | 40 +++++------- .../src/utilities/rowProcessor/index.ts | 65 +++++++++++++------ 2 files changed, 62 insertions(+), 43 deletions(-) diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index c2b043785f..7faaa1c252 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -10,7 +10,11 @@ import flatten from "lodash/flatten" import { USER_METDATA_PREFIX } from "../utils" import partition from "lodash/partition" import { getGlobalUsersFromMetadata } from "../../utilities/global" -import { outputProcessing, processFormulas } from "../../utilities/rowProcessor" +import { + coreOutputProcessing, + outputProcessing, + processFormulas, +} from "../../utilities/rowProcessor" import { context, features } from "@budibase/backend-core" import { ContextUser, @@ -156,9 +160,6 @@ export async function updateLinks(args: { /** * Given a table and a list of rows this will retrieve all of the attached docs and enrich them into the row. * This is required for formula fields, this may only be utilised internally (for now). - * @param table The table from which the rows originated. - * @param rows The rows which are to be enriched. - * @param opts optional - options like passing in a base row to use for enrichment. * @return returns the rows with all of the enriched relationships on it. */ export async function attachFullLinkedDocs( @@ -248,9 +249,6 @@ export type SquashTableFields = Record /** * This function will take the given enriched rows and squash the links to only contain the primary display field. - * @param table The table from which the rows originated. - * @param enriched The pre-enriched rows (full docs) which are to be squashed. - * @param squashFields Per link column (key) define which columns are allowed while squashing. * @returns The rows after having their links squashed to only contain the ID and primary display. */ export async function squashLinks( @@ -283,21 +281,18 @@ export async function squashLinks( if (schema.type !== FieldType.LINK || !Array.isArray(row[column])) { continue } - const newLinks = [] - for (const link of row[column]) { - const linkTblId = - link.tableId || getRelatedTableForField(table.schema, column) - const linkedTable = await getLinkedTable(linkTblId!, linkedTables) + const relatedTable = await getLinkedTable(schema.tableId, linkedTables) + if (viewSchema[column]?.columns) { + row[column] = await coreOutputProcessing(relatedTable, row[column]) + } + row[column] = row[column].map((link: Row) => { const obj: any = { _id: link._id } - obj.primaryDisplay = getPrimaryDisplayValue(link, linkedTable) + obj.primaryDisplay = getPrimaryDisplayValue(link, relatedTable) if (viewSchema[column]?.columns) { - const enrichedLink = await outputProcessing(linkedTable, link, { - squash: false, - }) - const squashFields = Object.entries(viewSchema[column].columns) + const squashFields = Object.entries(viewSchema[column].columns || {}) .filter(([columnName, viewColumnConfig]) => { - const tableColumn = linkedTable.schema[columnName] + const tableColumn = relatedTable.schema[columnName] if (!tableColumn) { return false } @@ -315,13 +310,14 @@ export async function squashLinks( .map(([columnName]) => columnName) for (const relField of squashFields) { - obj[relField] = enrichedLink[relField] + if (link[relField] != null) { + obj[relField] = link[relField] + } } } - newLinks.push(obj) - } - row[column] = newLinks + return obj + }) } } return (isArray ? enrichedArray : enrichedArray[0]) as T diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index f6cf44d6d6..f20a95c2fa 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -264,18 +264,46 @@ export async function outputProcessing( } else { safeRows = rows } - // attach any linked row information - let enriched = !opts.preserveLinks - ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, { - fromRow: opts?.fromRow, - }) - : safeRows + let enriched: Row[] + // SQS returns the rows with full relationship contents + if (!(await features.flags.isEnabled("SQS"))) { + // attach any linked row information + enriched = !opts.preserveLinks + ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, { + fromRow: opts?.fromRow, + }) + : safeRows + } else { + enriched = cloneDeep(safeRows) + } - // make sure squash is enabled if needed if (!opts.squash && utils.hasCircularStructure(rows)) { opts.squash = true } + enriched = await coreOutputProcessing(table, enriched, opts) + + if (opts.squash) { + enriched = await linkRows.squashLinks(table, enriched, { + fromViewId: opts?.fromViewId, + }) + } + + return (wasArray ? enriched : enriched[0]) as T +} + +export async function coreOutputProcessing( + table: Table, + rows: Row[], + opts: { + preserveLinks?: boolean + skipBBReferences?: boolean + fromViewId?: string + } = { + preserveLinks: false, + skipBBReferences: false, + } +): Promise { // process complex types: attachments, bb references... for (const [property, column] of Object.entries(table.schema)) { if ( @@ -283,7 +311,7 @@ export async function outputProcessing( column.type === FieldType.ATTACHMENT_SINGLE || column.type === FieldType.SIGNATURE_SINGLE ) { - for (const row of enriched) { + for (const row of rows) { if (row[property] == null) { continue } @@ -308,7 +336,7 @@ export async function outputProcessing( !opts.skipBBReferences && column.type == FieldType.BB_REFERENCE ) { - for (const row of enriched) { + for (const row of rows) { row[property] = await processOutputBBReferences( row[property], column.subtype @@ -318,14 +346,14 @@ export async function outputProcessing( !opts.skipBBReferences && column.type == FieldType.BB_REFERENCE_SINGLE ) { - for (const row of enriched) { + for (const row of rows) { row[property] = await processOutputBBReference( row[property], column.subtype ) } } else if (column.type === FieldType.DATETIME && column.timeOnly) { - for (const row of enriched) { + for (const row of rows) { if (row[property] instanceof Date) { const hours = row[property].getUTCHours().toString().padStart(2, "0") const minutes = row[property] @@ -340,7 +368,7 @@ export async function outputProcessing( } } } else if (column.type === FieldType.LINK) { - for (let row of enriched) { + for (let row of rows) { // if relationship is empty - remove the array, this has been part of the API for some time if (Array.isArray(row[property]) && row[property].length === 0) { delete row[property] @@ -350,17 +378,12 @@ export async function outputProcessing( } // process formulas after the complex types had been processed - enriched = await processFormulas(table, enriched, { dynamic: true }) + rows = await processFormulas(table, rows, { dynamic: true }) - if (opts.squash) { - enriched = await linkRows.squashLinks(table, enriched, { - fromViewId: opts?.fromViewId, - }) - } // remove null properties to match internal API const isExternal = isExternalTableID(table._id!) if (isExternal || (await features.flags.isEnabled("SQS"))) { - for (const row of enriched) { + for (const row of rows) { for (const key of Object.keys(row)) { if (row[key] === null) { delete row[key] @@ -388,7 +411,7 @@ export async function outputProcessing( const fields = [...tableFields, ...protectedColumns].map(f => f.toLowerCase() ) - for (const row of enriched) { + for (const row of rows) { for (const key of Object.keys(row)) { if (!fields.includes(key.toLowerCase())) { delete row[key] @@ -397,5 +420,5 @@ export async function outputProcessing( } } - return (wasArray ? enriched : enriched[0]) as T + return rows }