MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Some work towards improving performance of internal DB enrichment, there is a problem with cyclic enrichment due to the outputProcessing, need to decide how to handle formulas on enrichment.

This commit is contained in:
mike12345567 2023-10-04 19:18:21 +01:00
parent 96b077f8a5
commit 069fd33964
6 changed files with 134 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
packages/server/src/api/controllers/row/internal.ts
View File

@ -1,6 +1,7 @@
import * as linkRows from "../../../db/linkedRows"
import {
generateRowID,
getMultiIDParams,
getTableIDFromRowID,
InternalTables,
} from "../../../db/utils"
@ -24,6 +25,8 @@ import {
UserCtx,
} from "@budibase/types"
import sdk from "../../../sdk"
import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils"
import { flatten } from "lodash"
export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
const tableId = utils.getTableId(ctx)
@ -224,57 +227,43 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
const db = context.getAppDB()
const tableId = utils.getTableId(ctx)
const rowId = ctx.params.rowId
// need table to work out where links go in row
let [table, row] = await Promise.all([
// need table to work out where links go in row, as well as the link docs
let response = await Promise.all([
sdk.tables.getTable(tableId),
utils.findRow(ctx, tableId, rowId),
linkRows.getLinkDocuments({ tableId, rowId }),
])
// get the link docs
const linkVals = (await linkRows.getLinkDocuments({
tableId,
rowId,
})) as LinkDocumentValue[]
const table = response[0] as Table
const row = response[1] as Row
const linkVals = response[2] as LinkDocumentValue[]
// look up the actual rows based on the ids
let response = (
await db.allDocs({
include_docs: true,
keys: linkVals.map(linkVal => linkVal.id),
})
).rows.map(row => row.doc)
// group responses by table
let groups: any = {},
tables: Record<string, Table> = {}
for (let row of response) {
if (!row.tableId) {
row.tableId = getTableIDFromRowID(row._id)
}
const linkedTableId = row.tableId
if (groups[linkedTableId] == null) {
groups[linkedTableId] = [row]
tables[linkedTableId] = await db.get(linkedTableId)
} else {
groups[linkedTableId].push(row)
}
}
let linkedRows: Row[] = []
for (let [tableId, rows] of Object.entries(groups)) {
// need to include the IDs in these rows for any links they may have
linkedRows = linkedRows.concat(
await outputProcessing(tables[tableId], rows as Row[])
const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id))
let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc)
// get the linked tables
const linkTableIds = getLinkedTableIDs(table as Table)
const linkTables = await sdk.tables.getTables(linkTableIds)
// perform output processing
let final = []
for (let linkTable of linkTables) {
const relatedRows = linkedRows.filter(row => row.tableId === linkTable._id)
final = final.concat(
outputProcessing(linkTable, relatedRows, { preserveLinks: true })
)
}
// finalise the promises
linkedRows = flatten(await Promise.all(final))
// insert the link rows in the correct place throughout the main row
for (let fieldName of Object.keys(table.schema)) {
let field = table.schema[fieldName]
if (field.type === FieldTypes.LINK) {
// find the links that pertain to this field, get their indexes
const linkIndexes = linkVals
.filter(link => link.fieldName === fieldName)
.map(link => linkVals.indexOf(link))
// find the links that pertain to this field
const links = linkVals.filter(link => link.fieldName === fieldName)
// find the rows that the links state are linked to this field
row[fieldName] = linkedRows.filter((linkRow, index) =>
linkIndexes.includes(index)
row[fieldName] = linkedRows.filter(linkRow =>
links.find(link => link.id === linkRow._id)
)
}
}

6
packages/server/src/api/controllers/user.ts
View File

@ -1,13 +1,11 @@
import { generateUserFlagID } from "../../db/utils"
import { InternalTables } from "../../db/utils"
import { generateUserFlagID, InternalTables } from "../../db/utils"
import { getFullUser } from "../../utilities/users"
import { context } from "@budibase/backend-core"
import { Ctx, UserCtx } from "@budibase/types"
import sdk from "../../sdk"
export async function fetchMetadata(ctx: Ctx) {
const users = await sdk.users.fetchMetadata()
ctx.body = users
ctx.body = await sdk.users.fetchMetadata()
}
export async function updateSelfMetadata(ctx: UserCtx) {

26
packages/server/src/db/linkedRows/index.ts
View File

@ -9,13 +9,13 @@ import {
getLinkedTable,
} from "./linkUtils"
import flatten from "lodash/flatten"
import { FieldTypes } from "../../constants"
import { getMultiIDParams, USER_METDATA_PREFIX } from "../utils"
import partition from "lodash/partition"
import { getGlobalUsersFromMetadata } from "../../utilities/global"
import { processFormulas } from "../../utilities/rowProcessor"
import { context } from "@budibase/backend-core"
import { Table, Row, LinkDocumentValue } from "@budibase/types"
import { Table, Row, LinkDocumentValue, FieldType } from "@budibase/types"
import sdk from "../../sdk"
export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils"
@ -35,7 +35,7 @@ export const EventType = {
function clearRelationshipFields(table: Table, rows: Row[]) {
for (let [key, field] of Object.entries(table.schema)) {
if (field.type === FieldTypes.LINK) {
if (field.type === FieldType.LINK) {
rows = rows.map(row => {
delete row[key]
return row
@ -45,7 +45,7 @@ function clearRelationshipFields(table: Table, rows: Row[]) {
return rows
}
async function getLinksForRows(rows: Row[]) {
async function getLinksForRows(rows: Row[]): Promise<LinkDocumentValue[]> {
const tableIds = [...new Set(rows.map(el => el.tableId))]
// start by getting all the link values for performance reasons
const promises = tableIds.map(tableId =>
@ -153,15 +153,19 @@ export async function attachFullLinkedDocs(table: Table, rows: Row[]) {
if (linkedTableIds.length === 0) {
return rows
}
// get all the links
const links = (await getLinksForRows(rows)).filter(link =>
// get tables and links
let response = await Promise.all([
getLinksForRows(rows),
sdk.tables.getTables(linkedTableIds),
])
const links = (response[0] as LinkDocumentValue[]).filter(link =>
rows.some(row => row._id === link.thisId)
)
const linkedTables = response[1] as Table[]
// clear any existing links that could be dupe'd
rows = clearRelationshipFields(table, rows)
// now get the docs and combine into the rows
let linked = await getFullLinkedDocs(links)
const linkedTables: Table[] = []
for (let row of rows) {
for (let link of links.filter(link => link.thisId === row._id)) {
if (row[link.fieldName] == null) {
@ -171,7 +175,9 @@ export async function attachFullLinkedDocs(table: Table, rows: Row[]) {
if (linkedRow) {
const linkedTableId =
linkedRow.tableId || getRelatedTableForField(table, link.fieldName)
const linkedTable = await getLinkedTable(linkedTableId, linkedTables)
const linkedTable = linkedTables.find(
table => table._id === linkedTableId
)
if (linkedTable) {
row[link.fieldName].push(processFormulas(linkedTable, linkedRow))
}
@ -199,13 +205,13 @@ export async function squashLinksToPrimaryDisplay(
// this only fetches the table if its not already in array
const rowTable = await getLinkedTable(row.tableId!, linkedTables)
for (let [column, schema] of Object.entries(rowTable?.schema || {})) {
if (schema.type !== FieldTypes.LINK || !Array.isArray(row[column])) {
if (schema.type !== FieldType.LINK || !Array.isArray(row[column])) {
continue
}
const newLinks = []
for (let link of row[column]) {
const linkTblId = link.tableId || getRelatedTableForField(table, column)
const linkedTable = await getLinkedTable(linkTblId, linkedTables)
const linkedTable = await getLinkedTable(linkTblId!, linkedTables)
const obj: any = { _id: link._id }
if (linkedTable?.primaryDisplay && link[linkedTable.primaryDisplay]) {
obj.primaryDisplay = link[linkedTable.primaryDisplay]

4
packages/server/src/db/linkedRows/linkUtils.ts
View File

@ -91,10 +91,10 @@ export function getUniqueByProp(array: any[], prop: string) {
})
}
export function getLinkedTableIDs(table: Table) {
export function getLinkedTableIDs(table: Table): string[] {
return Object.values(table.schema)
.filter((column: FieldSchema) => column.type === FieldTypes.LINK)
.map(column => column.tableId)
.map(column => column.tableId!)
}
export async function getLinkedTable(id: string, tables: Table[]) {

31
packages/server/src/sdk/app/datasources/datasources.ts
View File

@ -23,10 +23,13 @@ import {
getTableParams,
} from "../../../db/utils"
import sdk from "../../index"
import datasource from "../../../api/routes/datasource"
const ENV_VAR_PREFIX = "env."
export async function fetch() {
export async function fetch(opts?: {
enriched: boolean
}): Promise<Datasource[]> {
// Get internal tables
const db = context.getAppDB()
const internalTables = await db.allDocs(
@ -44,7 +47,7 @@ export async function fetch() {
const bbInternalDb = {
...BudibaseInternalDB,
}
} as Datasource
// Get external datasources
const datasources = (
@ -66,7 +69,18 @@ export async function fetch() {
}
}
return [bbInternalDb, ...datasources]
if (opts?.enriched) {
const envVars = await getEnvironmentVariables()
const promises = datasources.map(datasource =>
enrichDatasourceWithValues(datasource, envVars)
)
const enriched = (await Promise.all(promises)).map(
result => result.datasource
)
return [bbInternalDb, ...enriched]
} else {
return [bbInternalDb, ...datasources]
}
}
export function areRESTVariablesValid(datasource: Datasource) {
@ -107,9 +121,12 @@ export function checkDatasourceTypes(schema: Integration, config: any) {
return config
}
async function enrichDatasourceWithValues(datasource: Datasource) {
async function enrichDatasourceWithValues(
datasource: Datasource,
variables?: Record<string, string>
) {
const cloned = cloneDeep(datasource)
const env = await getEnvironmentVariables()
const env = variables ? variables : await getEnvironmentVariables()
//Do not process entities, as we do not want to process formulas
const { entities, ...clonedWithoutEntities } = cloned
const processed = processObjectSync(
@ -235,9 +252,9 @@ export function mergeConfigs(update: Datasource, old: Datasource) {
if (value !== PASSWORD_REPLACEMENT) {
continue
}
if (old.config?.[key]) {
if (update.config && old.config && old.config?.[key]) {
update.config[key] = old.config?.[key]
} else {
} else if (update.config) {
delete update.config[key]
}
}

75
packages/server/src/sdk/app/tables/index.ts
View File

@ -1,11 +1,16 @@
import { context } from "@budibase/backend-core"
import { BudibaseInternalDB, getTableParams } from "../../../db/utils"
import {
BudibaseInternalDB,
getMultiIDParams,
getTableParams,
} from "../../../db/utils"
import {
breakExternalTableId,
isExternalTable,
isSQL,
} from "../../../integrations/utils"
import {
AllDocsResponse,
Database,
Table,
TableResponse,
@ -15,6 +20,14 @@ import datasources from "../datasources"
import { populateExternalTableSchemas } from "./validation"
import sdk from "../../../sdk"
function processInternalTables(docs: AllDocsResponse<Table[]>): Table[] {
return docs.rows.map((tableDoc: any) => ({
...tableDoc.doc,
type: "internal",
sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id,
}))
}
async function getAllInternalTables(db?: Database): Promise<Table[]> {
if (!db) {
db = context.getAppDB()
@ -24,15 +37,49 @@ async function getAllInternalTables(db?: Database): Promise<Table[]> {
include_docs: true,
})
)
return internalTables.rows.map((tableDoc: any) => ({
...tableDoc.doc,
type: "internal",
sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id,
}))
return processInternalTables(internalTables)
}
async function getAllExternalTables(
datasourceId: any
async function getAllExternalTables(): Promise<Table[]> {
const datasources = await sdk.datasources.fetch({ enriched: true })
const allEntities = datasources.map(datasource => datasource.entities)
let final = []
for (let entities of allEntities) {
final = final.concat(Object.values(entities))
}
return final
}
async function getAllTables() {
const [internal, external] = await Promise.all([
getAllInternalTables(),
getAllExternalTables(),
])
return [...internal, external]
}
async function getTables(tableIds: string[]): Promise<Table[]> {
const externalTableIds = tableIds.filter(tableId => isExternalTable(tableId)),
internalTableIds = tableIds.filter(tableId => !isExternalTable(tableId))
let tables = []
if (externalTableIds.length) {
const externalTables = await getAllExternalTables()
tables = tables.concat(
externalTables.filter(
table => externalTableIds.indexOf(table._id!) !== -1
)
)
}
if (internalTableIds.length) {
const db = context.getAppDB()
const internalTables = await db.allDocs(getMultiIDParams(internalTableIds))
tables = tables.concat(processInternalTables(internalTables))
}
return tables
}
async function getExternalTablesInDatasource(
datasourceId: string
): Promise<Record<string, Table>> {
const datasource = await datasources.get(datasourceId, { enriched: true })
if (!datasource || !datasource.entities) {
@ -42,14 +89,14 @@ async function getAllExternalTables(
}
async function getExternalTable(
datasourceId: any,
tableName: any
datasourceId: string,
tableName: string
): Promise<Table> {
const entities = await getAllExternalTables(datasourceId)
const entities = await getExternalTablesInDatasource(datasourceId)
return entities[tableName]
}
async function getTable(tableId: any): Promise<Table> {
async function getTable(tableId: string): Promise<Table> {
const db = context.getAppDB()
if (isExternalTable(tableId)) {
let { datasourceId, tableName } = breakExternalTableId(tableId)
@ -86,9 +133,11 @@ async function saveTable(table: Table) {
export default {
getAllInternalTables,
getAllExternalTables,
getExternalTablesInDatasource,
getExternalTable,
getTable,
getAllTables,
getTables,
populateExternalTableSchemas,
enrichViewSchemas,
saveTable,