Merge pull request #11981 from Budibase/fix/internal-db-enrich-perf

Internal DB enrichment performance improvements

packages/frontend-core/src/api/relationships.js

@@ -9,7 +9,9 @@ export const buildRelationshipEndpoints = API => ({
     if (!tableId || !rowId) {
       return []
     }
-    const response = await API.get({ url: `/api/${tableId}/${rowId}/enrich` })
+    const response = await API.get({
+      url: `/api/${tableId}/${rowId}/enrich?field=${fieldName}`,
+    })
     if (!fieldName) {
       return response || []
     } else {

packages/server/src/api/controllers/row/index.ts

@@ -26,6 +26,7 @@ import { fixRow } from "../public/rows"
 import sdk from "../../../sdk"
 import * as exporters from "../view/exporters"
 import { apiFileReturn } from "../../../utilities/fileSystem"
+import { Format } from "../view/exporters"
 export * as views from "./views"
 
 function pickApi(tableId: any) {
@@ -267,7 +268,7 @@ export const exportRows = async (
     async () => {
       const { fileName, content } = await sdk.rows.exportRows({
         tableId,
-        format,
+        format: format as Format,
         rowIds: rows,
         columns,
         query,

packages/server/src/api/controllers/row/internal.ts

@@ -1,6 +1,7 @@
 import * as linkRows from "../../../db/linkedRows"
 import {
   generateRowID,
+  getMultiIDParams,
   getTableIDFromRowID,
   InternalTables,
 } from "../../../db/utils"
@@ -24,6 +25,8 @@ import {
   UserCtx,
 } from "@budibase/types"
 import sdk from "../../../sdk"
+import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils"
+import { flatten } from "lodash"
 
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const tableId = utils.getTableId(ctx)
@@ -154,7 +157,7 @@ export async function destroy(ctx: UserCtx) {
   if (row.tableId !== tableId) {
     throw "Supplied tableId doesn't match the row's tableId"
   }
-  const table = await sdk.tables.getTable(row.tableId)
+  const table = await sdk.tables.getTable(tableId)
   // update the row to include full relationships before deleting them
   row = await outputProcessing(table, row, {
     squash: false,
@@ -164,7 +167,7 @@ export async function destroy(ctx: UserCtx) {
   await linkRows.updateLinks({
     eventType: linkRows.EventType.ROW_DELETE,
     row,
-    tableId: row.tableId,
+    tableId,
   })
   // remove any attachments that were on the row from object storage
   await cleanupAttachments(table, { row })
@@ -225,60 +228,52 @@ export async function bulkDestroy(ctx: UserCtx) {
 }
 
 export async function fetchEnrichedRow(ctx: UserCtx) {
+  const fieldName = ctx.request.query.field as string | undefined
   const db = context.getAppDB()
   const tableId = utils.getTableId(ctx)
-  const rowId = ctx.params.rowId
+  const rowId = ctx.params.rowId as string
-  // need table to work out where links go in row
+  // need table to work out where links go in row, as well as the link docs
-  let [table, row] = await Promise.all([
+  let response = await Promise.all([
     sdk.tables.getTable(tableId),
     utils.findRow(ctx, tableId, rowId),
+    linkRows.getLinkDocuments({ tableId, rowId, fieldName }),
   ])
-  // get the link docs
+  const table = response[0] as Table
-  const linkVals = (await linkRows.getLinkDocuments({
+  const row = response[1] as Row
-    tableId,
+  const linkVals = response[2] as LinkDocumentValue[]
-    rowId,
-  })) as LinkDocumentValue[]
   // look up the actual rows based on the ids
-  let response = (
+  const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id))
-    await db.allDocs({
+  let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc)
-      include_docs: true,
+
-      keys: linkVals.map(linkVal => linkVal.id),
+  // get the linked tables
+  const linkTableIds = getLinkedTableIDs(table as Table)
+  const linkTables = await sdk.tables.getTables(linkTableIds)
+
+  // perform output processing
+  let final: Promise<Row[]>[] = []
+  for (let linkTable of linkTables) {
+    const relatedRows = linkedRows.filter(row => row.tableId === linkTable._id)
+    // include the row being enriched for performance reasons, don't need to fetch it to include
+    final = final.concat(
+      outputProcessing(linkTable, relatedRows, {
+        // have to clone to avoid JSON cycle
+        fromRow: cloneDeep(row),
+        squash: true,
       })
-  ).rows.map(row => row.doc)
-  // group responses by table
-  let groups: any = {},
-    tables: Record<string, Table> = {}
-  for (let row of response) {
-    if (!row.tableId) {
-      row.tableId = getTableIDFromRowID(row._id)
-    }
-    const linkedTableId = row.tableId
-    if (groups[linkedTableId] == null) {
-      groups[linkedTableId] = [row]
-      tables[linkedTableId] = await db.get(linkedTableId)
-    } else {
-      groups[linkedTableId].push(row)
-    }
-  }
-  let linkedRows: Row[] = []
-  for (let [tableId, rows] of Object.entries(groups)) {
-    // need to include the IDs in these rows for any links they may have
-    linkedRows = linkedRows.concat(
-      await outputProcessing(tables[tableId], rows as Row[])
     )
   }
+  // finalise the promises
+  linkedRows = flatten(await Promise.all(final))
 
   // insert the link rows in the correct place throughout the main row
   for (let fieldName of Object.keys(table.schema)) {
     let field = table.schema[fieldName]
     if (field.type === FieldTypes.LINK) {
-      // find the links that pertain to this field, get their indexes
+      // find the links that pertain to this field
-      const linkIndexes = linkVals
+      const links = linkVals.filter(link => link.fieldName === fieldName)
-        .filter(link => link.fieldName === fieldName)
-        .map(link => linkVals.indexOf(link))
       // find the rows that the links state are linked to this field
-      row[fieldName] = linkedRows.filter((linkRow, index) =>
+      row[fieldName] = linkedRows.filter(linkRow =>
-        linkIndexes.includes(index)
+        links.find(link => link.id === linkRow._id)
       )
     }
   }

packages/server/src/api/controllers/row/staticFormula.ts

@@ -149,7 +149,7 @@ export async function finaliseRow(
       await db.put(table)
     } catch (err: any) {
       if (err.status === 409) {
-        const updatedTable = await sdk.tables.getTable(table._id)
+        const updatedTable = await sdk.tables.getTable(table._id!)
         let response = processAutoColumn(null, updatedTable, row, {
           reprocessing: true,
         })

packages/server/src/api/controllers/row/utils.ts

@@ -17,20 +17,6 @@ import sdk from "../../../sdk"
 import validateJs from "validate.js"
 import { cloneDeep } from "lodash/fp"
 
-function isForeignKey(key: string, table: Table) {
-  const relationships = Object.values(table.schema).filter(
-    column => column.type === FieldType.LINK
-  )
-  return relationships.some(
-    relationship =>
-      (
-        relationship as
-          | OneToManyRelationshipFieldMetadata
-          | ManyToOneRelationshipFieldMetadata
-      ).foreignKey === key
-  )
-}
-
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {
     return new Date(value).getTime()
@@ -60,7 +46,7 @@ export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
   return row
 }
 
-export function getTableId(ctx: Ctx) {
+export function getTableId(ctx: Ctx): string {
   // top priority, use the URL first
   if (ctx.params?.sourceId) {
     return ctx.params.sourceId
@@ -77,112 +63,7 @@ export function getTableId(ctx: Ctx) {
   if (ctx.params?.viewName) {
     return ctx.params.viewName
   }
-}
+  throw new Error("Unable to find table ID in request")
-
-export async function validate({
-  tableId,
-  row,
-  table,
-}: {
-  tableId?: string
-  row: Row
-  table?: Table
-}) {
-  let fetchedTable: Table
-  if (!table) {
-    fetchedTable = await sdk.tables.getTable(tableId)
-  } else {
-    fetchedTable = table
-  }
-  const errors: any = {}
-  for (let fieldName of Object.keys(fetchedTable.schema)) {
-    const column = fetchedTable.schema[fieldName]
-    const constraints = cloneDeep(column.constraints)
-    const type = column.type
-    // foreign keys are likely to be enriched
-    if (isForeignKey(fieldName, fetchedTable)) {
-      continue
-    }
-    // formulas shouldn't validated, data will be deleted anyway
-    if (type === FieldTypes.FORMULA || column.autocolumn) {
-      continue
-    }
-    // special case for options, need to always allow unselected (empty)
-    if (type === FieldTypes.OPTIONS && constraints?.inclusion) {
-      constraints.inclusion.push(null as any, "")
-    }
-    let res
-
-    // Validate.js doesn't seem to handle array
-    if (type === FieldTypes.ARRAY && row[fieldName]) {
-      if (row[fieldName].length) {
-        if (!Array.isArray(row[fieldName])) {
-          row[fieldName] = row[fieldName].split(",")
-        }
-        row[fieldName].map((val: any) => {
-          if (
-            !constraints?.inclusion?.includes(val) &&
-            constraints?.inclusion?.length !== 0
-          ) {
-            errors[fieldName] = "Field not in list"
-          }
-        })
-      } else if (constraints?.presence && row[fieldName].length === 0) {
-        // non required MultiSelect creates an empty array, which should not throw errors
-        errors[fieldName] = [`${fieldName} is required`]
-      }
-    } else if (
-      (type === FieldTypes.ATTACHMENT || type === FieldTypes.JSON) &&
-      typeof row[fieldName] === "string"
-    ) {
-      // this should only happen if there is an error
-      try {
-        const json = JSON.parse(row[fieldName])
-        if (type === FieldTypes.ATTACHMENT) {
-          if (Array.isArray(json)) {
-            row[fieldName] = json
-          } else {
-            errors[fieldName] = [`Must be an array`]
-          }
-        }
-      } catch (err) {
-        errors[fieldName] = [`Contains invalid JSON`]
-      }
-    } else {
-      res = validateJs.single(row[fieldName], constraints)
-    }
-    if (res) errors[fieldName] = res
-  }
-  return { valid: Object.keys(errors).length === 0, errors }
-}
-
-// don't do a pure falsy check, as 0 is included
-// https://github.com/Budibase/budibase/issues/10118
-export function removeEmptyFilters(filters: SearchFilters) {
-  for (let filterField of NoEmptyFilterStrings) {
-    if (!filters[filterField]) {
-      continue
-    }
-
-    for (let filterType of Object.keys(filters)) {
-      if (filterType !== filterField) {
-        continue
-      }
-      // don't know which one we're checking, type could be anything
-      const value = filters[filterType] as unknown
-      if (typeof value === "object") {
-        for (let [key, value] of Object.entries(
-          filters[filterType] as object
-        )) {
-          if (value == null || value === "") {
-            // @ts-ignore
-            delete filters[filterField][key]
-          }
-        }
-      }
-    }
-  }
-  return filters
 }
 
 export function isUserMetadataTable(tableId: string) {

packages/server/src/api/controllers/table/index.ts

@@ -16,6 +16,7 @@ import {
   Table,
   TableResponse,
   UserCtx,
+  Row,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { jsonFromCsvString } from "../../../utilities/csv"
@@ -139,8 +140,7 @@ export async function validateNewTableImport(ctx: UserCtx) {
 }
 
 export async function validateExistingTableImport(ctx: UserCtx) {
-  const { rows, tableId }: { rows: unknown; tableId: unknown } =
+  const { rows, tableId }: { rows: Row[]; tableId?: string } = ctx.request.body
-    ctx.request.body
 
   let schema = null
   if (tableId) {

packages/server/src/api/controllers/user.ts

@@ -1,13 +1,11 @@
-import { generateUserFlagID } from "../../db/utils"
+import { generateUserFlagID, InternalTables } from "../../db/utils"
-import { InternalTables } from "../../db/utils"
 import { getFullUser } from "../../utilities/users"
 import { context } from "@budibase/backend-core"
 import { Ctx, UserCtx } from "@budibase/types"
 import sdk from "../../sdk"
 
 export async function fetchMetadata(ctx: Ctx) {
-  const users = await sdk.users.fetchMetadata()
+  ctx.body = await sdk.users.fetchMetadata()
-  ctx.body = users
 }
 
 export async function updateSelfMetadata(ctx: UserCtx) {

packages/server/src/db/linkedRows/index.ts

@@ -9,13 +9,13 @@ import {
   getLinkedTable,
 } from "./linkUtils"
 import flatten from "lodash/flatten"
-import { FieldTypes } from "../../constants"
 import { getMultiIDParams, USER_METDATA_PREFIX } from "../utils"
 import partition from "lodash/partition"
 import { getGlobalUsersFromMetadata } from "../../utilities/global"
 import { processFormulas } from "../../utilities/rowProcessor"
 import { context } from "@budibase/backend-core"
-import { Table, Row, LinkDocumentValue } from "@budibase/types"
+import { Table, Row, LinkDocumentValue, FieldType } from "@budibase/types"
+import sdk from "../../sdk"
 
 export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils"
 
@@ -35,7 +35,7 @@ export const EventType = {
 
 function clearRelationshipFields(table: Table, rows: Row[]) {
   for (let [key, field] of Object.entries(table.schema)) {
-    if (field.type === FieldTypes.LINK) {
+    if (field.type === FieldType.LINK) {
       rows = rows.map(row => {
         delete row[key]
         return row
@@ -45,7 +45,7 @@ function clearRelationshipFields(table: Table, rows: Row[]) {
   return rows
 }
 
-async function getLinksForRows(rows: Row[]) {
+async function getLinksForRows(rows: Row[]): Promise<LinkDocumentValue[]> {
   const tableIds = [...new Set(rows.map(el => el.tableId))]
   // start by getting all the link values for performance reasons
   const promises = tableIds.map(tableId =>
@@ -146,32 +146,57 @@ export async function updateLinks(args: {
  * This is required for formula fields, this may only be utilised internally (for now).
  * @param table The table from which the rows originated.
  * @param rows The rows which are to be enriched.
+ * @param opts optional - options like passing in a base row to use for enrichment.
  * @return returns the rows with all of the enriched relationships on it.
  */
-export async function attachFullLinkedDocs(table: Table, rows: Row[]) {
+export async function attachFullLinkedDocs(
+  table: Table,
+  rows: Row[],
+  opts?: { fromRow?: Row }
+) {
   const linkedTableIds = getLinkedTableIDs(table)
   if (linkedTableIds.length === 0) {
     return rows
   }
-  // get all the links
+  // get tables and links
-  const links = (await getLinksForRows(rows)).filter(link =>
+  let response = await Promise.all([
+    getLinksForRows(rows),
+    sdk.tables.getTables(linkedTableIds),
+  ])
+  // find the links that pertain to one of the rows that is being enriched
+  const links = (response[0] as LinkDocumentValue[]).filter(link =>
     rows.some(row => row._id === link.thisId)
   )
+  // if fromRow has been passed in, then we don't need to fetch it (optimisation)
+  let linksWithoutFromRow = links
+  if (opts?.fromRow) {
+    linksWithoutFromRow = links.filter(link => link.id !== opts?.fromRow?._id)
+  }
+  const linkedTables = response[1] as Table[]
   // clear any existing links that could be dupe'd
   rows = clearRelationshipFields(table, rows)
   // now get the docs and combine into the rows
-  let linked = await getFullLinkedDocs(links)
+  let linked = []
-  const linkedTables: Table[] = []
+  if (linksWithoutFromRow.length > 0) {
+    linked = await getFullLinkedDocs(linksWithoutFromRow)
+  }
   for (let row of rows) {
     for (let link of links.filter(link => link.thisId === row._id)) {
       if (row[link.fieldName] == null) {
         row[link.fieldName] = []
       }
-      const linkedRow = linked.find(row => row._id === link.id)
+      let linkedRow: Row
+      if (opts?.fromRow && opts?.fromRow?._id === link.id) {
+        linkedRow = opts.fromRow!
+      } else {
+        linkedRow = linked.find(row => row._id === link.id)
+      }
       if (linkedRow) {
         const linkedTableId =
           linkedRow.tableId || getRelatedTableForField(table, link.fieldName)
-        const linkedTable = await getLinkedTable(linkedTableId, linkedTables)
+        const linkedTable = linkedTables.find(
+          table => table._id === linkedTableId
+        )
         if (linkedTable) {
           row[link.fieldName].push(processFormulas(linkedTable, linkedRow))
         }
@@ -199,13 +224,13 @@ export async function squashLinksToPrimaryDisplay(
     // this only fetches the table if its not already in array
     const rowTable = await getLinkedTable(row.tableId!, linkedTables)
     for (let [column, schema] of Object.entries(rowTable?.schema || {})) {
-      if (schema.type !== FieldTypes.LINK || !Array.isArray(row[column])) {
+      if (schema.type !== FieldType.LINK || !Array.isArray(row[column])) {
         continue
       }
       const newLinks = []
       for (let link of row[column]) {
         const linkTblId = link.tableId || getRelatedTableForField(table, column)
-        const linkedTable = await getLinkedTable(linkTblId, linkedTables)
+        const linkedTable = await getLinkedTable(linkTblId!, linkedTables)
         const obj: any = { _id: link._id }
         if (linkedTable?.primaryDisplay && link[linkedTable.primaryDisplay]) {
           obj.primaryDisplay = link[linkedTable.primaryDisplay]

packages/server/src/db/linkedRows/linkUtils.ts

@@ -31,19 +31,22 @@ export const IncludeDocs = {
 export async function getLinkDocuments(args: {
   tableId?: string
   rowId?: string
-  includeDocs?: any
+  fieldName?: string
+  includeDocs?: boolean
 }): Promise<LinkDocumentValue[] | LinkDocument[]> {
-  const { tableId, rowId, includeDocs } = args
+  const { tableId, rowId, fieldName, includeDocs } = args
   const db = context.getAppDB()
   let params: any
-  if (rowId != null) {
+  if (rowId) {
     params = { key: [tableId, rowId] }
   }
   // only table is known
   else {
     params = { startKey: [tableId], endKey: [tableId, {}] }
   }
-  params.include_docs = !!includeDocs
+  if (includeDocs) {
+    params.include_docs = true
+  }
   try {
     let linkRows = (await db.query(getQueryIndex(ViewName.LINK), params)).rows
     // filter to get unique entries
@@ -63,6 +66,14 @@ export async function getLinkDocuments(args: {
       return unique
     })
 
+    // filter down to just the required field name
+    if (fieldName) {
+      linkRows = linkRows.filter(link => {
+        const value = link.value as LinkDocumentValue
+        return value.fieldName === fieldName
+      })
+    }
+    // return docs if docs requested, otherwise just the value information
     if (includeDocs) {
       return linkRows.map(row => row.doc) as LinkDocument[]
     } else {
@@ -87,7 +98,7 @@ export function getUniqueByProp(array: any[], prop: string) {
   })
 }
 
-export function getLinkedTableIDs(table: Table) {
+export function getLinkedTableIDs(table: Table): string[] {
   return Object.values(table.schema)
     .filter(isRelationshipColumn)
     .map(column => column.tableId)

packages/server/src/middleware/tests/trimViewRowInfo.spec.ts

@@ -16,6 +16,7 @@ jest.mock("../../sdk", () => ({
 import sdk from "../../sdk"
 import { Next } from "koa"
 
+const tableId = utils.generateTableID()
 const mockGetView = sdk.views.get as jest.MockedFunction<typeof sdk.views.get>
 const mockGetTable = sdk.tables.getTable as jest.MockedFunction<
   typeof sdk.tables.getTable
@@ -41,6 +42,7 @@ class TestConfiguration {
       body: ctxRequestBody,
     }
     this.params.viewId = viewId
+    this.params.sourceId = tableId
     return this.middleware(
       {
         request: this.request as any,
@@ -69,7 +71,7 @@ describe("trimViewRowInfo middleware", () => {
   })
 
   const table: Table = {
-    _id: utils.generateTableID(),
+    _id: tableId,
     name: generator.word(),
     type: "table",
     schema: {

packages/server/src/sdk/app/datasources/datasources.ts

@@ -23,10 +23,13 @@ import {
   getTableParams,
 } from "../../../db/utils"
 import sdk from "../../index"
+import datasource from "../../../api/routes/datasource"
 
 const ENV_VAR_PREFIX = "env."
 
-export async function fetch() {
+export async function fetch(opts?: {
+  enriched: boolean
+}): Promise<Datasource[]> {
   // Get internal tables
   const db = context.getAppDB()
   const internalTables = await db.allDocs(
@@ -44,7 +47,7 @@ export async function fetch() {
 
   const bbInternalDb = {
     ...BudibaseInternalDB,
-  }
+  } as Datasource
 
   // Get external datasources
   const datasources = (
@@ -66,8 +69,19 @@ export async function fetch() {
     }
   }
 
+  if (opts?.enriched) {
+    const envVars = await getEnvironmentVariables()
+    const promises = datasources.map(datasource =>
+      enrichDatasourceWithValues(datasource, envVars)
+    )
+    const enriched = (await Promise.all(promises)).map(
+      result => result.datasource
+    )
+    return [bbInternalDb, ...enriched]
+  } else {
     return [bbInternalDb, ...datasources]
   }
+}
 
 export function areRESTVariablesValid(datasource: Datasource) {
   const restConfig = datasource.config as RestConfig
@@ -107,9 +121,12 @@ export function checkDatasourceTypes(schema: Integration, config: any) {
   return config
 }
 
-async function enrichDatasourceWithValues(datasource: Datasource) {
+async function enrichDatasourceWithValues(
+  datasource: Datasource,
+  variables?: Record<string, string>
+) {
   const cloned = cloneDeep(datasource)
-  const env = await getEnvironmentVariables()
+  const env = variables ? variables : await getEnvironmentVariables()
   //Do not process entities, as we do not want to process formulas
   const { entities, ...clonedWithoutEntities } = cloned
   const processed = processObjectSync(
@@ -235,9 +252,9 @@ export function mergeConfigs(update: Datasource, old: Datasource) {
     if (value !== PASSWORD_REPLACEMENT) {
       continue
     }
-    if (old.config?.[key]) {
+    if (update.config && old.config && old.config?.[key]) {
       update.config[key] = old.config?.[key]
-    } else {
+    } else if (update.config) {
       delete update.config[key]
     }
   }

packages/server/src/sdk/app/rows/utils.ts

@@ -69,12 +69,15 @@ export async function validate({
   valid: boolean
   errors: Record<string, any>
 }> {
-  let fetchedTable: Table
+  let fetchedTable: Table | undefined
-  if (!table) {
+  if (!table && tableId) {
     fetchedTable = await sdk.tables.getTable(tableId)
-  } else {
+  } else if (table) {
     fetchedTable = table
   }
+  if (fetchedTable === undefined) {
+    throw new Error("Unable to fetch table for validation")
+  }
   const errors: Record<string, any> = {}
   for (let fieldName of Object.keys(fetchedTable.schema)) {
     const column = fetchedTable.schema[fieldName]

packages/server/src/sdk/app/tables/index.ts

@@ -1,11 +1,16 @@
 import { context } from "@budibase/backend-core"
-import { BudibaseInternalDB, getTableParams } from "../../../db/utils"
+import {
+  BudibaseInternalDB,
+  getMultiIDParams,
+  getTableParams,
+} from "../../../db/utils"
 import {
   breakExternalTableId,
   isExternalTable,
   isSQL,
 } from "../../../integrations/utils"
 import {
+  AllDocsResponse,
   Database,
   Table,
   TableResponse,
@@ -15,24 +20,70 @@ import datasources from "../datasources"
 import { populateExternalTableSchemas } from "./validation"
 import sdk from "../../../sdk"
 
-async function getAllInternalTables(db?: Database): Promise<Table[]> {
+function processInternalTables(docs: AllDocsResponse<Table[]>): Table[] {
-  if (!db) {
+  return docs.rows.map((tableDoc: any) => ({
-    db = context.getAppDB()
-  }
-  const internalTables = await db.allDocs(
-    getTableParams(null, {
-      include_docs: true,
-    })
-  )
-  return internalTables.rows.map((tableDoc: any) => ({
     ...tableDoc.doc,
     type: "internal",
     sourceId: tableDoc.doc.sourceId || BudibaseInternalDB._id,
   }))
 }
 
-async function getAllExternalTables(
+async function getAllInternalTables(db?: Database): Promise<Table[]> {
-  datasourceId: any
+  if (!db) {
+    db = context.getAppDB()
+  }
+  const internalTableDocs = await db.allDocs<Table[]>(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+  return processInternalTables(internalTableDocs)
+}
+
+async function getAllExternalTables(): Promise<Table[]> {
+  const datasources = await sdk.datasources.fetch({ enriched: true })
+  const allEntities = datasources.map(datasource => datasource.entities)
+  let final: Table[] = []
+  for (let entities of allEntities) {
+    if (entities) {
+      final = final.concat(Object.values(entities))
+    }
+  }
+  return final
+}
+
+async function getAllTables() {
+  const [internal, external] = await Promise.all([
+    getAllInternalTables(),
+    getAllExternalTables(),
+  ])
+  return [...internal, external]
+}
+
+async function getTables(tableIds: string[]): Promise<Table[]> {
+  const externalTableIds = tableIds.filter(tableId => isExternalTable(tableId)),
+    internalTableIds = tableIds.filter(tableId => !isExternalTable(tableId))
+  let tables: Table[] = []
+  if (externalTableIds.length) {
+    const externalTables = await getAllExternalTables()
+    tables = tables.concat(
+      externalTables.filter(
+        table => externalTableIds.indexOf(table._id!) !== -1
+      )
+    )
+  }
+  if (internalTableIds.length) {
+    const db = context.getAppDB()
+    const internalTableDocs = await db.allDocs<Table[]>(
+      getMultiIDParams(internalTableIds)
+    )
+    tables = tables.concat(processInternalTables(internalTableDocs))
+  }
+  return tables
+}
+
+async function getExternalTablesInDatasource(
+  datasourceId: string
 ): Promise<Record<string, Table>> {
   const datasource = await datasources.get(datasourceId, { enriched: true })
   if (!datasource || !datasource.entities) {
@@ -42,22 +93,22 @@ async function getAllExternalTables(
 }
 
 async function getExternalTable(
-  datasourceId: any,
+  datasourceId: string,
-  tableName: any
+  tableName: string
 ): Promise<Table> {
-  const entities = await getAllExternalTables(datasourceId)
+  const entities = await getExternalTablesInDatasource(datasourceId)
   return entities[tableName]
 }
 
-async function getTable(tableId: any): Promise<Table> {
+async function getTable(tableId: string): Promise<Table> {
   const db = context.getAppDB()
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
     const datasource = await datasources.get(datasourceId!)
-    const table = await getExternalTable(datasourceId, tableName)
+    const table = await getExternalTable(datasourceId!, tableName!)
     return { ...table, sql: isSQL(datasource) }
   } else {
-    return db.get(tableId)
+    return db.get<Table>(tableId)
   }
 }
 
@@ -86,9 +137,11 @@ async function saveTable(table: Table) {
 
 export default {
   getAllInternalTables,
-  getAllExternalTables,
+  getExternalTablesInDatasource,
   getExternalTable,
   getTable,
+  getAllTables,
+  getTables,
   populateExternalTableSchemas,
   enrichViewSchemas,
   saveTable,

packages/server/src/utilities/rowProcessor/index.ts

@@ -210,6 +210,7 @@ export async function outputProcessing<T extends Row[] | Row>(
   opts: {
     squash?: boolean
     preserveLinks?: boolean
+    fromRow?: Row
     skipBBReferences?: boolean
   } = {
     squash: true,
@@ -227,7 +228,9 @@ export async function outputProcessing<T extends Row[] | Row>(
   }
   // attach any linked row information
   let enriched = !opts.preserveLinks
-    ? await linkRows.attachFullLinkedDocs(table, safeRows)
+    ? await linkRows.attachFullLinkedDocs(table, safeRows, {
+        fromRow: opts?.fromRow,
+      })
     : safeRows
 
   // process complex types: attachements, bb references...