MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

This improves the performance of views when the enriched relationships feature has been enabled. There was an issue that caused the squashLinks and outputProcessing to loop onto each other until things broke down - this fixes the issue.

This commit is contained in:
mike12345567 2024-09-27 12:19:24 +01:00
parent 0d43730a65
commit c41c25a603
2 changed files with 62 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
packages/server/src/db/linkedRows/index.ts
View File

@ -10,7 +10,11 @@ import flatten from "lodash/flatten"
import { USER_METDATA_PREFIX } from "../utils"
import partition from "lodash/partition"
import { getGlobalUsersFromMetadata } from "../../utilities/global"
import { outputProcessing, processFormulas } from "../../utilities/rowProcessor"
import {
coreOutputProcessing,
outputProcessing,
processFormulas,
} from "../../utilities/rowProcessor"
import { context, features } from "@budibase/backend-core"
import {
ContextUser,
@ -156,9 +160,6 @@ export async function updateLinks(args: {
/**
* Given a table and a list of rows this will retrieve all of the attached docs and enrich them into the row.
* This is required for formula fields, this may only be utilised internally (for now).
* @param table The table from which the rows originated.
* @param rows The rows which are to be enriched.
* @param opts optional - options like passing in a base row to use for enrichment.
* @return returns the rows with all of the enriched relationships on it.
*/
export async function attachFullLinkedDocs(
@ -248,9 +249,6 @@ export type SquashTableFields = Record<string, { visibleFieldNames: string[] }>
/**
* This function will take the given enriched rows and squash the links to only contain the primary display field.
* @param table The table from which the rows originated.
* @param enriched The pre-enriched rows (full docs) which are to be squashed.
* @param squashFields Per link column (key) define which columns are allowed while squashing.
* @returns The rows after having their links squashed to only contain the ID and primary display.
*/
export async function squashLinks<T = Row[] | Row>(
@ -283,21 +281,18 @@ export async function squashLinks<T = Row[] | Row>(
if (schema.type !== FieldType.LINK || !Array.isArray(row[column])) {
continue
}
const newLinks = []
for (const link of row[column]) {
const linkTblId =
link.tableId || getRelatedTableForField(table.schema, column)
const linkedTable = await getLinkedTable(linkTblId!, linkedTables)
const relatedTable = await getLinkedTable(schema.tableId, linkedTables)
if (viewSchema[column]?.columns) {
row[column] = await coreOutputProcessing(relatedTable, row[column])
}
row[column] = row[column].map((link: Row) => {
const obj: any = { _id: link._id }
obj.primaryDisplay = getPrimaryDisplayValue(link, linkedTable)
obj.primaryDisplay = getPrimaryDisplayValue(link, relatedTable)
if (viewSchema[column]?.columns) {
const enrichedLink = await outputProcessing(linkedTable, link, {
squash: false,
})
const squashFields = Object.entries(viewSchema[column].columns)
const squashFields = Object.entries(viewSchema[column].columns || {})
.filter(([columnName, viewColumnConfig]) => {
const tableColumn = linkedTable.schema[columnName]
const tableColumn = relatedTable.schema[columnName]
if (!tableColumn) {
return false
}
@ -315,13 +310,14 @@ export async function squashLinks<T = Row[] | Row>(
.map(([columnName]) => columnName)
for (const relField of squashFields) {
obj[relField] = enrichedLink[relField]
if (link[relField] != null) {
obj[relField] = link[relField]
}
}
}
newLinks.push(obj)
}
row[column] = newLinks
return obj
})
}
}
return (isArray ? enrichedArray : enrichedArray[0]) as T

65
packages/server/src/utilities/rowProcessor/index.ts
View File

@ -264,18 +264,46 @@ export async function outputProcessing<T extends Row[] | Row>(
} else {
safeRows = rows
}
// attach any linked row information
let enriched = !opts.preserveLinks
? await linkRows.attachFullLinkedDocs(table.schema, safeRows, {
fromRow: opts?.fromRow,
})
: safeRows
let enriched: Row[]
// SQS returns the rows with full relationship contents
if (!(await features.flags.isEnabled("SQS"))) {
// attach any linked row information
enriched = !opts.preserveLinks
? await linkRows.attachFullLinkedDocs(table.schema, safeRows, {
fromRow: opts?.fromRow,
})
: safeRows
} else {
enriched = cloneDeep(safeRows)
}
// make sure squash is enabled if needed
if (!opts.squash && utils.hasCircularStructure(rows)) {
opts.squash = true
}
enriched = await coreOutputProcessing(table, enriched, opts)
if (opts.squash) {
enriched = await linkRows.squashLinks(table, enriched, {
fromViewId: opts?.fromViewId,
})
}
return (wasArray ? enriched : enriched[0]) as T
}
export async function coreOutputProcessing(
table: Table,
rows: Row[],
opts: {
preserveLinks?: boolean
skipBBReferences?: boolean
fromViewId?: string
} = {
preserveLinks: false,
skipBBReferences: false,
}
): Promise<Row[]> {
// process complex types: attachments, bb references...
for (const [property, column] of Object.entries(table.schema)) {
if (
@ -283,7 +311,7 @@ export async function outputProcessing<T extends Row[] | Row>(
column.type === FieldType.ATTACHMENT_SINGLE ||
column.type === FieldType.SIGNATURE_SINGLE
) {
for (const row of enriched) {
for (const row of rows) {
if (row[property] == null) {
continue
}
@ -308,7 +336,7 @@ export async function outputProcessing<T extends Row[] | Row>(
!opts.skipBBReferences &&
column.type == FieldType.BB_REFERENCE
) {
for (const row of enriched) {
for (const row of rows) {
row[property] = await processOutputBBReferences(
row[property],
column.subtype
@ -318,14 +346,14 @@ export async function outputProcessing<T extends Row[] | Row>(
!opts.skipBBReferences &&
column.type == FieldType.BB_REFERENCE_SINGLE
) {
for (const row of enriched) {
for (const row of rows) {
row[property] = await processOutputBBReference(
row[property],
column.subtype
)
}
} else if (column.type === FieldType.DATETIME && column.timeOnly) {
for (const row of enriched) {
for (const row of rows) {
if (row[property] instanceof Date) {
const hours = row[property].getUTCHours().toString().padStart(2, "0")
const minutes = row[property]
@ -340,7 +368,7 @@ export async function outputProcessing<T extends Row[] | Row>(
}
}
} else if (column.type === FieldType.LINK) {
for (let row of enriched) {
for (let row of rows) {
// if relationship is empty - remove the array, this has been part of the API for some time
if (Array.isArray(row[property]) && row[property].length === 0) {
delete row[property]
@ -350,17 +378,12 @@ export async function outputProcessing<T extends Row[] | Row>(
}
// process formulas after the complex types had been processed
enriched = await processFormulas(table, enriched, { dynamic: true })
rows = await processFormulas(table, rows, { dynamic: true })
if (opts.squash) {
enriched = await linkRows.squashLinks(table, enriched, {
fromViewId: opts?.fromViewId,
})
}
// remove null properties to match internal API
const isExternal = isExternalTableID(table._id!)
if (isExternal || (await features.flags.isEnabled("SQS"))) {
for (const row of enriched) {
for (const row of rows) {
for (const key of Object.keys(row)) {
if (row[key] === null) {
delete row[key]
@ -388,7 +411,7 @@ export async function outputProcessing<T extends Row[] | Row>(
const fields = [...tableFields, ...protectedColumns].map(f =>
f.toLowerCase()
)
for (const row of enriched) {
for (const row of rows) {
for (const key of Object.keys(row)) {
if (!fields.includes(key.toLowerCase())) {
delete row[key]
@ -397,5 +420,5 @@ export async function outputProcessing<T extends Row[] | Row>(
}
}
return (wasArray ? enriched : enriched[0]) as T
return rows
}