MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #11981 from Budibase/fix/internal-db-enrich-perf 

Internal DB enrichment performance improvements
This commit is contained in:
Michael Drury 2023-10-18 17:42:46 +01:00 committed by GitHub
parent 5cd22672b3 13dadbcc7d
commit 4ae2ae1682
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 213 additions and 222 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
packages/frontend-core/src/api/relationships.js
View File

@ -9,7 +9,9 @@ export const buildRelationshipEndpoints = API => ({
if (!tableId || !rowId) {
return []
}
const response = await API.get({ url: `/api/${tableId}/${rowId}/enrich` })
const response = await API.get({
url: `/api/${tableId}/${rowId}/enrich?field=${fieldName}`,
})
if (!fieldName) {
return response || []
} else {

3
packages/server/src/api/controllers/row/index.ts
View File

@ -26,6 +26,7 @@ import { fixRow } from "../public/rows"
import sdk from "../../../sdk"
import * as exporters from "../view/exporters"
import { apiFileReturn } from "../../../utilities/fileSystem"
import { Format } from "../view/exporters"
export * as views from "./views"
function pickApi(tableId: any) {
@ -267,7 +268,7 @@ export const exportRows = async (
async () => {
const { fileName, content } = await sdk.rows.exportRows({
tableId,
format,
format: format as Format,
rowIds: rows,
columns,
query,

77
packages/server/src/api/controllers/row/internal.ts
View File

@ -1,6 +1,7 @@
import * as linkRows from "../../../db/linkedRows"
import {
generateRowID,
getMultiIDParams,
getTableIDFromRowID,
InternalTables,
} from "../../../db/utils"
@ -24,6 +25,8 @@ import {
UserCtx,
} from "@budibase/types"
import sdk from "../../../sdk"
import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils"
import { flatten } from "lodash"
export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
const tableId = utils.getTableId(ctx)
@ -154,7 +157,7 @@ export async function destroy(ctx: UserCtx) {
if (row.tableId !== tableId) {
throw "Supplied tableId doesn't match the row's tableId"
}
const table = await sdk.tables.getTable(row.tableId)
const table = await sdk.tables.getTable(tableId)
// update the row to include full relationships before deleting them
row = await outputProcessing(table, row, {
squash: false,
@ -164,7 +167,7 @@ export async function destroy(ctx: UserCtx) {
await linkRows.updateLinks({
eventType: linkRows.EventType.ROW_DELETE,
row,
tableId: row.tableId,
tableId,
})
// remove any attachments that were on the row from object storage
await cleanupAttachments(table, { row })
@ -225,60 +228,52 @@ export async function bulkDestroy(ctx: UserCtx) {
}
export async function fetchEnrichedRow(ctx: UserCtx) {
const fieldName = ctx.request.query.field as string | undefined
const db = context.getAppDB()
const tableId = utils.getTableId(ctx)
const rowId = ctx.params.rowId
// need table to work out where links go in row
let [table, row] = await Promise.all([
const rowId = ctx.params.rowId as string
// need table to work out where links go in row, as well as the link docs
let response = await Promise.all([
sdk.tables.getTable(tableId),
utils.findRow(ctx, tableId, rowId),
linkRows.getLinkDocuments({ tableId, rowId, fieldName }),
])
// get the link docs
const linkVals = (await linkRows.getLinkDocuments({
tableId,
rowId,
})) as LinkDocumentValue[]
const table = response[0] as Table
const row = response[1] as Row
const linkVals = response[2] as LinkDocumentValue[]
// look up the actual rows based on the ids
let response = (
await db.allDocs({
include_docs: true,
keys: linkVals.map(linkVal => linkVal.id),
const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id))
let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc)
// get the linked tables
const linkTableIds = getLinkedTableIDs(table as Table)
const linkTables = await sdk.tables.getTables(linkTableIds)
// perform output processing
let final: Promise<Row[]>[] = []
for (let linkTable of linkTables) {
const relatedRows = linkedRows.filter(row => row.tableId === linkTable._id)
// include the row being enriched for performance reasons, don't need to fetch it to include
final = final.concat(
outputProcessing(linkTable, relatedRows, {
// have to clone to avoid JSON cycle
fromRow: cloneDeep(row),
squash: true,
})
).rows.map(row => row.doc)
// group responses by table
let groups: any = {},
tables: Record<string, Table> = {}
for (let row of response) {
if (!row.tableId) {
row.tableId = getTableIDFromRowID(row._id)
}
const linkedTableId = row.tableId
if (groups[linkedTableId] == null) {
groups[linkedTableId] = [row]
tables[linkedTableId] = await db.get(linkedTableId)
} else {
groups[linkedTableId].push(row)
}
}
let linkedRows: Row[] = []
for (let [tableId, rows] of Object.entries(groups)) {
// need to include the IDs in these rows for any links they may have
linkedRows = linkedRows.concat(
await outputProcessing(tables[tableId], rows as Row[])
)
}
// finalise the promises
linkedRows = flatten(await Promise.all(final))
// insert the link rows in the correct place throughout the main row
for (let fieldName of Object.keys(table.schema)) {
let field = table.schema[fieldName]
if (field.type === FieldTypes.LINK) {
// find the links that pertain to this field, get their indexes
const linkIndexes = linkVals
.filter(link => link.fieldName === fieldName)
.map(link => linkVals.indexOf(link))
// find the links that pertain to this field
const links = linkVals.filter(link => link.fieldName === fieldName)
// find the rows that the links state are linked to this field
row[fieldName] = linkedRows.filter((linkRow, index) =>
linkIndexes.includes(index)
row[fieldName] = linkedRows.filter(linkRow =>
links.find(link => link.id === linkRow._id)
)
}
}

2
packages/server/src/api/controllers/row/staticFormula.ts
View File

@ -149,7 +149,7 @@ export async function finaliseRow(
await db.put(table)
} catch (err: any) {
if (err.status === 409) {
const updatedTable = await sdk.tables.getTable(table._id)
const updatedTable = await sdk.tables.getTable(table._id!)
let response = processAutoColumn(null, updatedTable, row, {
reprocessing: true,
})

123
packages/server/src/api/controllers/row/utils.ts
View File

@ -17,20 +17,6 @@ import sdk from "../../../sdk"
import validateJs from "validate.js"
import { cloneDeep } from "lodash/fp"
function isForeignKey(key: string, table: Table) {
const relationships = Object.values(table.schema).filter(
column => column.type === FieldType.LINK
)
return relationships.some(
relationship =>
(
relationship as
| OneToManyRelationshipFieldMetadata
| ManyToOneRelationshipFieldMetadata
).foreignKey === key
)
}
validateJs.extend(validateJs.validators.datetime, {
parse: function (value: string) {
return new Date(value).getTime()
@ -60,7 +46,7 @@ export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
return row
}
export function getTableId(ctx: Ctx) {
export function getTableId(ctx: Ctx): string {
// top priority, use the URL first
if (ctx.params?.sourceId) {
return ctx.params.sourceId
@ -77,112 +63,7 @@ export function getTableId(ctx: Ctx) {
if (ctx.params?.viewName) {
return ctx.params.viewName
}
}
export async function validate({
tableId,
row,
table,
}: {
tableId?: string
row: Row
table?: Table
}) {
let fetchedTable: Table
if (!table) {
fetchedTable = await sdk.tables.getTable(tableId)
} else {
fetchedTable = table
}
const errors: any = {}
for (let fieldName of Object.keys(fetchedTable.schema)) {
const column = fetchedTable.schema[fieldName]
const constraints = cloneDeep(column.constraints)
const type = column.type
// foreign keys are likely to be enriched
if (isForeignKey(fieldName, fetchedTable)) {
continue
}
// formulas shouldn't validated, data will be deleted anyway
if (type === FieldTypes.FORMULA || column.autocolumn) {
continue
}
// special case for options, need to always allow unselected (empty)
if (type === FieldTypes.OPTIONS && constraints?.inclusion) {
constraints.inclusion.push(null as any, "")
}
let res
// Validate.js doesn't seem to handle array
if (type === FieldTypes.ARRAY && row[fieldName]) {
if (row[fieldName].length) {
if (!Array.isArray(row[fieldName])) {
row[fieldName] = row[fieldName].split(",")
}
row[fieldName].map((val: any) => {
if (
!constraints?.inclusion?.includes(val) &&
constraints?.inclusion?.length !== 0
) {
errors[fieldName] = "Field not in list"
}
})
} else if (constraints?.presence && row[fieldName].length === 0) {
// non required MultiSelect creates an empty array, which should not throw errors
errors[fieldName] = [`${fieldName} is required`]
}
} else if (
(type === FieldTypes.ATTACHMENT || type === FieldTypes.JSON) &&
typeof row[fieldName] === "string"
) {
// this should only happen if there is an error
try {
const json = JSON.parse(row[fieldName])
if (type === FieldTypes.ATTACHMENT) {
if (Array.isArray(json)) {
row[fieldName] = json
} else {
errors[fieldName] = [`Must be an array`]
}
}
} catch (err) {
errors[fieldName] = [`Contains invalid JSON`]
}
} else {
res = validateJs.single(row[fieldName], constraints)
}
if (res) errors[fieldName] = res
}
return { valid: Object.keys(errors).length === 0, errors }
}
// don't do a pure falsy check, as 0 is included
// https://github.com/Budibase/budibase/issues/10118
export function removeEmptyFilters(filters: SearchFilters) {
for (let filterField of NoEmptyFilterStrings) {
if (!filters[filterField]) {
continue
}
for (let filterType of Object.keys(filters)) {
if (filterType !== filterField) {
continue
}
// don't know which one we're checking, type could be anything
const value = filters[filterType] as unknown
if (typeof value === "object") {
for (let [key, value] of Object.entries(
filters[filterType] as object
)) {
if (value == null || value === "") {
// @ts-ignore
delete filters[filterField][key]
}
}
}
}
}
return filters
throw new Error("Unable to find table ID in request")
}
export function isUserMetadataTable(tableId: string) {

4
packages/server/src/api/controllers/table/index.ts
View File

@ -16,6 +16,7 @@ import {
Table,
TableResponse,
UserCtx,
Row,
} from "@budibase/types"
import sdk from "../../../sdk"
import { jsonFromCsvString } from "../../../utilities/csv"
@ -139,8 +140,7 @@ export async function validateNewTableImport(ctx: UserCtx) {
}
export async function validateExistingTableImport(ctx: UserCtx) {
const { rows, tableId }: { rows: unknown; tableId: unknown } =
ctx.request.body
const { rows, tableId }: { rows: Row[]; tableId?: string } = ctx.request.body
let schema = null
if (tableId) {

6
packages/server/src/api/controllers/user.ts
View File

@ -1,13 +1,11 @@
import { generateUserFlagID } from "../../db/utils"
import { InternalTables } from "../../db/utils"
import { generateUserFlagID, InternalTables } from "../../db/utils"
import { getFullUser } from "../../utilities/users"
import { context } from "@budibase/backend-core"
import { Ctx, UserCtx } from "@budibase/types"
import sdk from "../../sdk"
export async function fetchMetadata(ctx: Ctx) {
const users = await sdk.users.fetchMetadata()
ctx.body = users
ctx.body = await sdk.users.fetchMetadata()
}
export async function updateSelfMetadata(ctx: UserCtx) {

51
packages/server/src/db/linkedRows/index.ts
View File

@ -9,13 +9,13 @@ import {
getLinkedTable,
} from "./linkUtils"
import flatten from "lodash/flatten"
import { FieldTypes } from "../../constants"
import { getMultiIDParams, USER_METDATA_PREFIX } from "../utils"
import partition from "lodash/partition"
import { getGlobalUsersFromMetadata } from "../../utilities/global"
import { processFormulas } from "../../utilities/rowProcessor"
import { context } from "@budibase/backend-core"
import { Table, Row, LinkDocumentValue } from "@budibase/types"
import { Table, Row, LinkDocumentValue, FieldType } from "@budibase/types"
import sdk from "../../sdk"
export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils"
@ -35,7 +35,7 @@ export const EventType = {
function clearRelationshipFields(table: Table, rows: Row[]) {
for (let [key, field] of Object.entries(table.schema)) {
if (field.type === FieldTypes.LINK) {
if (field.type === FieldType.LINK) {
rows = rows.map(row => {
delete row[key]
return row
@ -45,7 +45,7 @@ function clearRelationshipFields(table: Table, rows: Row[]) {
return rows
}
async function getLinksForRows(rows: Row[]) {
async function getLinksForRows(rows: Row[]): Promise<LinkDocumentValue[]> {
const tableIds = [...new Set(rows.map(el => el.tableId))]
// start by getting all the link values for performance reasons
const promises = tableIds.map(tableId =>
@ -146,32 +146,57 @@ export async function updateLinks(args: {
* This is required for formula fields, this may only be utilised internally (for now).
* @param table The table from which the rows originated.
* @param rows The rows which are to be enriched.
* @param opts optional - options like passing in a base row to use for enrichment.
* @return returns the rows with all of the enriched relationships on it.
*/
export async function attachFullLinkedDocs(table: Table, rows: Row[]) {
export async function attachFullLinkedDocs(
table: Table,
rows: Row[],
opts?: { fromRow?: Row }
) {
const linkedTableIds = getLinkedTableIDs(table)
if (linkedTableIds.length === 0) {
return rows
}
// get all the links
const links = (await getLinksForRows(rows)).filter(link =>
// get tables and links
let response = await Promise.all([
getLinksForRows(rows),
sdk.tables.getTables(linkedTableIds),
])
// find the links that pertain to one of the rows that is being enriched
const links = (response[0] as LinkDocumentValue[]).filter(link =>
rows.some(row => row._id === link.thisId)
)
// if fromRow has been passed in, then we don't need to fetch it (optimisation)
let linksWithoutFromRow = links
if (opts?.fromRow) {
linksWithoutFromRow = links.filter(link => link.id !== opts?.fromRow?._id)
}
const linkedTables = response[1] as Table[]
// clear any existing links that could be dupe'd
rows = clearRelationshipFields(table, rows)
// now get the docs and combine into the rows
let linked = await getFullLinkedDocs(links)
const linkedTables: Table[] = []
let linked = []
if (linksWithoutFromRow.length > 0) {
linked = await getFullLinkedDocs(linksWithoutFromRow)
}
for (let row of rows) {
for (let link of links.filter(link => link.thisId === row._id)) {
if (row[link.fieldName] == null) {
row[link.fieldName] = []
}
const linkedRow = linked.find(row => row._id === link.id)
let linkedRow: Row
if (opts?.fromRow && opts?.fromRow?._id === link.id) {
linkedRow = opts.fromRow!
} else {
linkedRow = linked.find(row => row._id === link.id)
}
if (linkedRow) {
const linkedTableId =
linkedRow.tableId || getRelatedTableForField(table, link.fieldName)
const linkedTable = await getLinkedTable(linkedTableId, linkedTables)
const linkedTable = linkedTables.find(
table => table._id === linkedTableId
)
if (linkedTable) {
row[link.fieldName].push(processFormulas(linkedTable, linkedRow))
}
@ -199,13 +224,13 @@ export async function squashLinksToPrimaryDisplay(
// this only fetches the table if its not already in array
const rowTable = await getLinkedTable(row.tableId!, linkedTables)
for (let [column, schema] of Object.entries(rowTable?.schema || {})) {
if (schema.type !== FieldTypes.LINK || !Array.isArray(row[column])) {
if (schema.type !== FieldType.LINK || !Array.isArray(row[column])) {
continue
}
const newLinks = []
for (let link of row[column]) {
const linkTblId = link.tableId || getRelatedTableForField(table, column)
const linkedTable = await getLinkedTable(linkTblId, linkedTables)
const linkedTable = await getLinkedTable(linkTblId!, linkedTables)
const obj: any = { _id: link._id }
if (linkedTable?.primaryDisplay && link[linkedTable.primaryDisplay]) {
obj.primaryDisplay = link[linkedTable.primaryDisplay]

21
packages/server/src/db/linkedRows/linkUtils.ts
View File

@ -31,19 +31,22 @@ export const IncludeDocs = {
export async function getLinkDocuments(args: {
tableId?: string
rowId?: string
includeDocs?: any
fieldName?: string
includeDocs?: boolean
}): Promise<LinkDocumentValue[] | LinkDocument[]> {
const { tableId, rowId, includeDocs } = args
const { tableId, rowId, fieldName, includeDocs } = args
const db = context.getAppDB()
let params: any
if (rowId != null) {
if (rowId) {
params = { key: [tableId, rowId] }
}
// only table is known
else {
params = { startKey: [tableId], endKey: [tableId, {}] }
}
params.include_docs = !!includeDocs
if (includeDocs) {
params.include_docs = true
}
try {
let linkRows = (await db.query(getQueryIndex(ViewName.LINK), params)).rows
// filter to get unique entries
@ -63,6 +66,14 @@ export async function getLinkDocuments(args: {
return unique
})
// filter down to just the required field name
if (fieldName) {
linkRows = linkRows.filter(link => {
const value = link.value as LinkDocumentValue
return value.fieldName === fieldName
})
}
// return docs if docs requested, otherwise just the value information
if (includeDocs) {
return linkRows.map(row => row.doc) as LinkDocument[]
} else {
@ -87,7 +98,7 @@ export function getUniqueByProp(array: any[], prop: string) {
})
}
export function getLinkedTableIDs(table: Table) {
export function getLinkedTableIDs(table: Table): string[] {
return Object.values(table.schema)
.filter(isRelationshipColumn)
.map(column => column.tableId)

4
packages/server/src/middleware/tests/trimViewRowInfo.spec.ts
View File

@ -16,6 +16,7 @@ jest.mock("../../sdk", () => ({
import sdk from "../../sdk"
import { Next } from "koa"
const tableId = utils.generateTableID()
const mockGetView = sdk.views.get as jest.MockedFunction<typeof sdk.views.get>
const mockGetTable = sdk.tables.getTable as jest.MockedFunction<
typeof sdk.tables.getTable
@ -41,6 +42,7 @@ class TestConfiguration {
body: ctxRequestBody,
}
this.params.viewId = viewId
this.params.sourceId = tableId
return this.middleware(
{
request: this.request as any,
@ -69,7 +71,7 @@ describe("trimViewRowInfo middleware", () => {
})
const table: Table = {
_id: utils.generateTableID(),
_id: tableId,
name: generator.word(),
type: "table",
schema: {

29
packages/server/src/sdk/app/datasources/datasources.ts
View File

@ -23,10 +23,13 @@ import {
getTableParams,
} from "../../../db/utils"
import sdk from "../../index"
import datasource from "../../../api/routes/datasource"
const ENV_VAR_PREFIX = "env."
export async function fetch() {
export async function fetch(opts?: {
enriched: boolean
}): Promise<Datasource[]> {
// Get internal tables
const db = context.getAppDB()
const internalTables = await db.allDocs(
@ -44,7 +47,7 @@ export async function fetch() {
const bbInternalDb = {
...BudibaseInternalDB,
}
} as Datasource
// Get external datasources
const datasources = (
@ -66,7 +69,18 @@ export async function fetch() {
}
}
if (opts?.enriched) {
const envVars = await getEnvironmentVariables()
const promises = datasources.map(datasource =>
enrichDatasourceWithValues(datasource, envVars)
)
const enriched = (await Promise.all(promises)).map(
result => result.datasource
)
return [bbInternalDb, ...enriched]
} else {
return [bbInternalDb, ...datasources]
}
}
export function areRESTVariablesValid(datasource: Datasource) {
@ -107,9 +121,12 @@ export function checkDatasourceTypes(schema: Integration, config: any) {
return config
}
async function enrichDatasourceWithValues(datasource: Datasource) {
async function enrichDatasourceWithValues(
datasource: Datasource,
variables?: Record<string, string>
) {
const cloned = cloneDeep(datasource)
const env = await getEnvironmentVariables()
const env = variables ? variables : await getEnvironmentVariables()
//Do not process entities, as we do not want to process formulas
const { entities, ...clonedWithoutEntities } = cloned
const processed = processObjectSync(
@ -235,9 +252,9 @@ export function mergeConfigs(update: Datasource, old: Datasource) {
if (value !== PASSWORD_REPLACEMENT) {
continue
}
if (old.config?.[key]) {
if (update.config && old.config && old.config?.[key]) {
update.config[key] = old.config?.[key]
} else {
} else if (update.config) {
delete update.config[key]
}
}

9
packages/server/src/sdk/app/rows/utils.ts
View File

@ -69,12 +69,15 @@ export async function validate({
valid: boolean
errors: Record<string, any>
}> {
let fetchedTable: Table
if (!table) {
let fetchedTable: Table | undefined
if (!table && tableId) {
fetchedTable = await sdk.tables.getTable(tableId)
} else {
} else if (table) {
fetchedTable = table
}
if (fetchedTable === undefined) {
throw new Error("Unable to fetch table for validation")
}
const errors: Record<string, any> = {}
for (let fieldName of Object.keys(fetchedTable.schema)) {
const column = fetchedTable.schema[fieldName]

93
packages/server/src/sdk/app/tables/index.ts
View File

@ -1,11 +1,16 @@
import { context } from "@budibase/backend-core"
import { BudibaseInternalDB, getTableParams } from "../../../db/utils"
import {
BudibaseInternalDB,
getMultiIDParams,
getTableParams,
} from "../../../db/utils"
import {
breakExternalTableId,
isExternalTable,
isSQL,
} from "../../../integrations/utils"
import {
AllDocsResponse,
Database,
Table,
TableResponse,
@ -15,24 +20,70 @@ import datasources from "../datasources"
import { populateExternalTableSchemas } from "./validation"
import sdk from "../../../sdk"
async function getAllInternalTables(db?: Database): Promise<Table[]> {
if (!db) {
db = context.getAppDB()
}
const internalTables = await db.allDocs(
getTableParams(null, {
include_docs: true,
})
)
return internalTables.rows.map((tableDoc: any) => ({
function processInternalTables(docs: AllDocsResponse<Table[]>): Table[] {
return docs.rows.map((tableDoc: any) => ({
...tableDoc.doc,
type: "internal",
sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id,
}))
}
async function getAllExternalTables(
datasourceId: any
async function getAllInternalTables(db?: Database): Promise<Table[]> {
if (!db) {
db = context.getAppDB()
}
const internalTableDocs = await db.allDocs<Table[]>(
getTableParams(null, {
include_docs: true,
})
)
return processInternalTables(internalTableDocs)
}
async function getAllExternalTables(): Promise<Table[]> {
const datasources = await sdk.datasources.fetch({ enriched: true })
const allEntities = datasources.map(datasource => datasource.entities)
let final: Table[] = []
for (let entities of allEntities) {
if (entities) {
final = final.concat(Object.values(entities))
}
}
return final
}
async function getAllTables() {
const [internal, external] = await Promise.all([
getAllInternalTables(),
getAllExternalTables(),
])
return [...internal, external]
}
async function getTables(tableIds: string[]): Promise<Table[]> {
const externalTableIds = tableIds.filter(tableId => isExternalTable(tableId)),
internalTableIds = tableIds.filter(tableId => !isExternalTable(tableId))
let tables: Table[] = []
if (externalTableIds.length) {
const externalTables = await getAllExternalTables()
tables = tables.concat(
externalTables.filter(
table => externalTableIds.indexOf(table._id!) !== -1
)
)
}
if (internalTableIds.length) {
const db = context.getAppDB()
const internalTableDocs = await db.allDocs<Table[]>(
getMultiIDParams(internalTableIds)
)
tables = tables.concat(processInternalTables(internalTableDocs))
}
return tables
}
async function getExternalTablesInDatasource(
datasourceId: string
): Promise<Record<string, Table>> {
const datasource = await datasources.get(datasourceId, { enriched: true })
if (!datasource || !datasource.entities) {
@ -42,22 +93,22 @@ async function getAllExternalTables(
}
async function getExternalTable(
datasourceId: any,
tableName: any
datasourceId: string,
tableName: string
): Promise<Table> {
const entities = await getAllExternalTables(datasourceId)
const entities = await getExternalTablesInDatasource(datasourceId)
return entities[tableName]
}
async function getTable(tableId: any): Promise<Table> {
async function getTable(tableId: string): Promise<Table> {
const db = context.getAppDB()
if (isExternalTable(tableId)) {
let { datasourceId, tableName } = breakExternalTableId(tableId)
const datasource = await datasources.get(datasourceId!)
const table = await getExternalTable(datasourceId, tableName)
const table = await getExternalTable(datasourceId!, tableName!)
return { ...table, sql: isSQL(datasource) }
} else {
return db.get(tableId)
return db.get<Table>(tableId)
}
}
@ -86,9 +137,11 @@ async function saveTable(table: Table) {
export default {
getAllInternalTables,
getAllExternalTables,
getExternalTablesInDatasource,
getExternalTable,
getTable,
getAllTables,
getTables,
populateExternalTableSchemas,
enrichViewSchemas,
saveTable,

5
packages/server/src/utilities/rowProcessor/index.ts
View File

@ -210,6 +210,7 @@ export async function outputProcessing<T extends Row[] | Row>(
opts: {
squash?: boolean
preserveLinks?: boolean
fromRow?: Row
skipBBReferences?: boolean
} = {
squash: true,
@ -227,7 +228,9 @@ export async function outputProcessing<T extends Row[] | Row>(
}
// attach any linked row information
let enriched = !opts.preserveLinks
? await linkRows.attachFullLinkedDocs(table, safeRows)
? await linkRows.attachFullLinkedDocs(table, safeRows, {
fromRow: opts?.fromRow,
})
: safeRows
// process complex types: attachements, bb references...