From c41c25a6037f69797ca313eca602f211ff8a9a7f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 27 Sep 2024 12:19:24 +0100 Subject: [PATCH 1/5] This improves the performance of views when the enriched relationships feature has been enabled. There was an issue that caused the squashLinks and outputProcessing to loop onto each other until things broke down - this fixes the issue. --- packages/server/src/db/linkedRows/index.ts | 40 +++++------- .../src/utilities/rowProcessor/index.ts | 65 +++++++++++++------ 2 files changed, 62 insertions(+), 43 deletions(-) diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index c2b043785f..7faaa1c252 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -10,7 +10,11 @@ import flatten from "lodash/flatten" import { USER_METDATA_PREFIX } from "../utils" import partition from "lodash/partition" import { getGlobalUsersFromMetadata } from "../../utilities/global" -import { outputProcessing, processFormulas } from "../../utilities/rowProcessor" +import { + coreOutputProcessing, + outputProcessing, + processFormulas, +} from "../../utilities/rowProcessor" import { context, features } from "@budibase/backend-core" import { ContextUser, @@ -156,9 +160,6 @@ export async function updateLinks(args: { /** * Given a table and a list of rows this will retrieve all of the attached docs and enrich them into the row. * This is required for formula fields, this may only be utilised internally (for now). - * @param table The table from which the rows originated. - * @param rows The rows which are to be enriched. - * @param opts optional - options like passing in a base row to use for enrichment. * @return returns the rows with all of the enriched relationships on it. */ export async function attachFullLinkedDocs( @@ -248,9 +249,6 @@ export type SquashTableFields = Record /** * This function will take the given enriched rows and squash the links to only contain the primary display field. - * @param table The table from which the rows originated. - * @param enriched The pre-enriched rows (full docs) which are to be squashed. - * @param squashFields Per link column (key) define which columns are allowed while squashing. * @returns The rows after having their links squashed to only contain the ID and primary display. */ export async function squashLinks( @@ -283,21 +281,18 @@ export async function squashLinks( if (schema.type !== FieldType.LINK || !Array.isArray(row[column])) { continue } - const newLinks = [] - for (const link of row[column]) { - const linkTblId = - link.tableId || getRelatedTableForField(table.schema, column) - const linkedTable = await getLinkedTable(linkTblId!, linkedTables) + const relatedTable = await getLinkedTable(schema.tableId, linkedTables) + if (viewSchema[column]?.columns) { + row[column] = await coreOutputProcessing(relatedTable, row[column]) + } + row[column] = row[column].map((link: Row) => { const obj: any = { _id: link._id } - obj.primaryDisplay = getPrimaryDisplayValue(link, linkedTable) + obj.primaryDisplay = getPrimaryDisplayValue(link, relatedTable) if (viewSchema[column]?.columns) { - const enrichedLink = await outputProcessing(linkedTable, link, { - squash: false, - }) - const squashFields = Object.entries(viewSchema[column].columns) + const squashFields = Object.entries(viewSchema[column].columns || {}) .filter(([columnName, viewColumnConfig]) => { - const tableColumn = linkedTable.schema[columnName] + const tableColumn = relatedTable.schema[columnName] if (!tableColumn) { return false } @@ -315,13 +310,14 @@ export async function squashLinks( .map(([columnName]) => columnName) for (const relField of squashFields) { - obj[relField] = enrichedLink[relField] + if (link[relField] != null) { + obj[relField] = link[relField] + } } } - newLinks.push(obj) - } - row[column] = newLinks + return obj + }) } } return (isArray ? enrichedArray : enrichedArray[0]) as T diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index f6cf44d6d6..f20a95c2fa 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -264,18 +264,46 @@ export async function outputProcessing( } else { safeRows = rows } - // attach any linked row information - let enriched = !opts.preserveLinks - ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, { - fromRow: opts?.fromRow, - }) - : safeRows + let enriched: Row[] + // SQS returns the rows with full relationship contents + if (!(await features.flags.isEnabled("SQS"))) { + // attach any linked row information + enriched = !opts.preserveLinks + ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, { + fromRow: opts?.fromRow, + }) + : safeRows + } else { + enriched = cloneDeep(safeRows) + } - // make sure squash is enabled if needed if (!opts.squash && utils.hasCircularStructure(rows)) { opts.squash = true } + enriched = await coreOutputProcessing(table, enriched, opts) + + if (opts.squash) { + enriched = await linkRows.squashLinks(table, enriched, { + fromViewId: opts?.fromViewId, + }) + } + + return (wasArray ? enriched : enriched[0]) as T +} + +export async function coreOutputProcessing( + table: Table, + rows: Row[], + opts: { + preserveLinks?: boolean + skipBBReferences?: boolean + fromViewId?: string + } = { + preserveLinks: false, + skipBBReferences: false, + } +): Promise { // process complex types: attachments, bb references... for (const [property, column] of Object.entries(table.schema)) { if ( @@ -283,7 +311,7 @@ export async function outputProcessing( column.type === FieldType.ATTACHMENT_SINGLE || column.type === FieldType.SIGNATURE_SINGLE ) { - for (const row of enriched) { + for (const row of rows) { if (row[property] == null) { continue } @@ -308,7 +336,7 @@ export async function outputProcessing( !opts.skipBBReferences && column.type == FieldType.BB_REFERENCE ) { - for (const row of enriched) { + for (const row of rows) { row[property] = await processOutputBBReferences( row[property], column.subtype @@ -318,14 +346,14 @@ export async function outputProcessing( !opts.skipBBReferences && column.type == FieldType.BB_REFERENCE_SINGLE ) { - for (const row of enriched) { + for (const row of rows) { row[property] = await processOutputBBReference( row[property], column.subtype ) } } else if (column.type === FieldType.DATETIME && column.timeOnly) { - for (const row of enriched) { + for (const row of rows) { if (row[property] instanceof Date) { const hours = row[property].getUTCHours().toString().padStart(2, "0") const minutes = row[property] @@ -340,7 +368,7 @@ export async function outputProcessing( } } } else if (column.type === FieldType.LINK) { - for (let row of enriched) { + for (let row of rows) { // if relationship is empty - remove the array, this has been part of the API for some time if (Array.isArray(row[property]) && row[property].length === 0) { delete row[property] @@ -350,17 +378,12 @@ export async function outputProcessing( } // process formulas after the complex types had been processed - enriched = await processFormulas(table, enriched, { dynamic: true }) + rows = await processFormulas(table, rows, { dynamic: true }) - if (opts.squash) { - enriched = await linkRows.squashLinks(table, enriched, { - fromViewId: opts?.fromViewId, - }) - } // remove null properties to match internal API const isExternal = isExternalTableID(table._id!) if (isExternal || (await features.flags.isEnabled("SQS"))) { - for (const row of enriched) { + for (const row of rows) { for (const key of Object.keys(row)) { if (row[key] === null) { delete row[key] @@ -388,7 +411,7 @@ export async function outputProcessing( const fields = [...tableFields, ...protectedColumns].map(f => f.toLowerCase() ) - for (const row of enriched) { + for (const row of rows) { for (const key of Object.keys(row)) { if (!fields.includes(key.toLowerCase())) { delete row[key] @@ -397,5 +420,5 @@ export async function outputProcessing( } } - return (wasArray ? enriched : enriched[0]) as T + return rows } From aa4cc2079f2e36ead729b9f08f716b3dee5323f4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 27 Sep 2024 12:24:40 +0100 Subject: [PATCH 2/5] Adding comment to explain new function. --- packages/server/src/utilities/rowProcessor/index.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index f20a95c2fa..2d0098ede6 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -292,6 +292,12 @@ export async function outputProcessing( return (wasArray ? enriched : enriched[0]) as T } +/** + * This function is similar to the outputProcessing function above, it makes sure that all the provided + * rows are ready for output, but does not have enrichment for squash capabilities which can cause performance issues. + * outputProcessing should be used when responding from the API, while this should be used when internally processing + * rows for any reason (like part of view operations). + */ export async function coreOutputProcessing( table: Table, rows: Row[], From d71ba7d37f43cd6edb35845689bacf8c88c11138 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 27 Sep 2024 12:27:41 +0100 Subject: [PATCH 3/5] Removing un-needed import. --- packages/server/src/db/linkedRows/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 7faaa1c252..2f7f7fd44c 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -12,7 +12,6 @@ import partition from "lodash/partition" import { getGlobalUsersFromMetadata } from "../../utilities/global" import { coreOutputProcessing, - outputProcessing, processFormulas, } from "../../utilities/rowProcessor" import { context, features } from "@budibase/backend-core" From 2a78409e8f81d6abc937769b8e5e699e509bf3f3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 27 Sep 2024 13:07:17 +0100 Subject: [PATCH 4/5] Removing feature check. --- .../server/src/utilities/rowProcessor/index.ts | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 2d0098ede6..14f2690231 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -266,16 +266,12 @@ export async function outputProcessing( } let enriched: Row[] // SQS returns the rows with full relationship contents - if (!(await features.flags.isEnabled("SQS"))) { - // attach any linked row information - enriched = !opts.preserveLinks - ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, { - fromRow: opts?.fromRow, - }) - : safeRows - } else { - enriched = cloneDeep(safeRows) - } + // attach any linked row information + enriched = !opts.preserveLinks + ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, { + fromRow: opts?.fromRow, + }) + : safeRows if (!opts.squash && utils.hasCircularStructure(rows)) { opts.squash = true From c30fe28fe8e487112e7dc14d33dcdc8729d0638e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 27 Sep 2024 13:07:44 +0100 Subject: [PATCH 5/5] Lint. --- packages/server/src/utilities/rowProcessor/index.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 14f2690231..2f227dd646 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -264,10 +264,9 @@ export async function outputProcessing( } else { safeRows = rows } - let enriched: Row[] // SQS returns the rows with full relationship contents // attach any linked row information - enriched = !opts.preserveLinks + let enriched = !opts.preserveLinks ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, { fromRow: opts?.fromRow, })