MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14650 from Budibase/fix/view-performance-improvements 

Enriched relationship view performance improvement
This commit is contained in:
Michael Drury 2024-09-27 13:24:20 +01:00 committed by GitHub
parent 029344dde5 c5ba891793
commit 78733da79b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 56 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
packages/server/src/db/linkedRows/index.ts
View File

@ -10,7 +10,10 @@ import flatten from "lodash/flatten"
import { USER_METDATA_PREFIX } from "../utils"
import partition from "lodash/partition"
import { getGlobalUsersFromMetadata } from "../../utilities/global"
import { outputProcessing, processFormulas } from "../../utilities/rowProcessor"
import {
coreOutputProcessing,
processFormulas,
} from "../../utilities/rowProcessor"
import { context, features } from "@budibase/backend-core"
import {
ContextUser,
@ -156,9 +159,6 @@ export async function updateLinks(args: {
/**
* Given a table and a list of rows this will retrieve all of the attached docs and enrich them into the row.
* This is required for formula fields, this may only be utilised internally (for now).
* @param table The table from which the rows originated.
* @param rows The rows which are to be enriched.
* @param opts optional - options like passing in a base row to use for enrichment.
* @return returns the rows with all of the enriched relationships on it.
*/
export async function attachFullLinkedDocs(
@ -248,9 +248,6 @@ export type SquashTableFields = Record<string, { visibleFieldNames: string[] }>
/**
* This function will take the given enriched rows and squash the links to only contain the primary display field.
* @param table The table from which the rows originated.
* @param enriched The pre-enriched rows (full docs) which are to be squashed.
* @param squashFields Per link column (key) define which columns are allowed while squashing.
* @returns The rows after having their links squashed to only contain the ID and primary display.
*/
export async function squashLinks<T = Row[] | Row>(
@ -283,21 +280,18 @@ export async function squashLinks<T = Row[] | Row>(
if (schema.type !== FieldType.LINK || !Array.isArray(row[column])) {
continue
}
const newLinks = []
for (const link of row[column]) {
const linkTblId =
link.tableId || getRelatedTableForField(table.schema, column)
const linkedTable = await getLinkedTable(linkTblId!, linkedTables)
const relatedTable = await getLinkedTable(schema.tableId, linkedTables)
if (viewSchema[column]?.columns) {
row[column] = await coreOutputProcessing(relatedTable, row[column])
}
row[column] = row[column].map((link: Row) => {
const obj: any = { _id: link._id }
obj.primaryDisplay = getPrimaryDisplayValue(link, linkedTable)
obj.primaryDisplay = getPrimaryDisplayValue(link, relatedTable)
if (viewSchema[column]?.columns) {
const enrichedLink = await outputProcessing(linkedTable, link, {
squash: false,
})
const squashFields = Object.entries(viewSchema[column].columns)
const squashFields = Object.entries(viewSchema[column].columns || {})
.filter(([columnName, viewColumnConfig]) => {
const tableColumn = linkedTable.schema[columnName]
const tableColumn = relatedTable.schema[columnName]
if (!tableColumn) {
return false
}
@ -315,13 +309,14 @@ export async function squashLinks<T = Row[] | Row>(
.map(([columnName]) => columnName)
for (const relField of squashFields) {
obj[relField] = enrichedLink[relField]
if (link[relField] != null) {
obj[relField] = link[relField]
}
}
}
newLinks.push(obj)
}
row[column] = newLinks
return obj
})
}
}
return (isArray ? enrichedArray : enrichedArray[0]) as T

54
packages/server/src/utilities/rowProcessor/index.ts
View File

@ -264,6 +264,7 @@ export async function outputProcessing<T extends Row[] | Row>(
} else {
safeRows = rows
}
// SQS returns the rows with full relationship contents
// attach any linked row information
let enriched = !opts.preserveLinks
? await linkRows.attachFullLinkedDocs(table.schema, safeRows, {
@ -271,11 +272,39 @@ export async function outputProcessing<T extends Row[] | Row>(
})
: safeRows
// make sure squash is enabled if needed
if (!opts.squash && utils.hasCircularStructure(rows)) {
opts.squash = true
}
enriched = await coreOutputProcessing(table, enriched, opts)
if (opts.squash) {
enriched = await linkRows.squashLinks(table, enriched, {
fromViewId: opts?.fromViewId,
})
}
return (wasArray ? enriched : enriched[0]) as T
}
/**
* This function is similar to the outputProcessing function above, it makes sure that all the provided
* rows are ready for output, but does not have enrichment for squash capabilities which can cause performance issues.
* outputProcessing should be used when responding from the API, while this should be used when internally processing
* rows for any reason (like part of view operations).
*/
export async function coreOutputProcessing(
table: Table,
rows: Row[],
opts: {
preserveLinks?: boolean
skipBBReferences?: boolean
fromViewId?: string
} = {
preserveLinks: false,
skipBBReferences: false,
}
): Promise<Row[]> {
// process complex types: attachments, bb references...
for (const [property, column] of Object.entries(table.schema)) {
if (
@ -283,7 +312,7 @@ export async function outputProcessing<T extends Row[] | Row>(
column.type === FieldType.ATTACHMENT_SINGLE ||
column.type === FieldType.SIGNATURE_SINGLE
) {
for (const row of enriched) {
for (const row of rows) {
if (row[property] == null) {
continue
}
@ -308,7 +337,7 @@ export async function outputProcessing<T extends Row[] | Row>(
!opts.skipBBReferences &&
column.type == FieldType.BB_REFERENCE
) {
for (const row of enriched) {
for (const row of rows) {
row[property] = await processOutputBBReferences(
row[property],
column.subtype
@ -318,14 +347,14 @@ export async function outputProcessing<T extends Row[] | Row>(
!opts.skipBBReferences &&
column.type == FieldType.BB_REFERENCE_SINGLE
) {
for (const row of enriched) {
for (const row of rows) {
row[property] = await processOutputBBReference(
row[property],
column.subtype
)
}
} else if (column.type === FieldType.DATETIME && column.timeOnly) {
for (const row of enriched) {
for (const row of rows) {
if (row[property] instanceof Date) {
const hours = row[property].getUTCHours().toString().padStart(2, "0")
const minutes = row[property]
@ -340,7 +369,7 @@ export async function outputProcessing<T extends Row[] | Row>(
}
}
} else if (column.type === FieldType.LINK) {
for (let row of enriched) {
for (let row of rows) {
// if relationship is empty - remove the array, this has been part of the API for some time
if (Array.isArray(row[property]) && row[property].length === 0) {
delete row[property]
@ -350,17 +379,12 @@ export async function outputProcessing<T extends Row[] | Row>(
}
// process formulas after the complex types had been processed
enriched = await processFormulas(table, enriched, { dynamic: true })
rows = await processFormulas(table, rows, { dynamic: true })
if (opts.squash) {
enriched = await linkRows.squashLinks(table, enriched, {
fromViewId: opts?.fromViewId,
})
}
// remove null properties to match internal API
const isExternal = isExternalTableID(table._id!)
if (isExternal || (await features.flags.isEnabled("SQS"))) {
for (const row of enriched) {
for (const row of rows) {
for (const key of Object.keys(row)) {
if (row[key] === null) {
delete row[key]
@ -388,7 +412,7 @@ export async function outputProcessing<T extends Row[] | Row>(
const fields = [...tableFields, ...protectedColumns].map(f =>
f.toLowerCase()
)
for (const row of enriched) {
for (const row of rows) {
for (const key of Object.keys(row)) {
if (!fields.includes(key.toLowerCase())) {
delete row[key]
@ -397,5 +421,5 @@ export async function outputProcessing<T extends Row[] | Row>(
}
}
return (wasArray ? enriched : enriched[0]) as T
return rows
}