Merge pull request #12354 from Budibase/fix/12154-invalid-table-fetches

Fix issues with data UI loading some tables

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -109,7 +109,7 @@ export class DatabaseImpl implements Database {
     }
   }
 
-  async get<T>(id?: string): Promise<T | any> {
+  async get<T extends Document>(id?: string): Promise<T> {
     const db = await this.checkSetup()
     if (!id) {
       throw new Error("Unable to get doc without a valid _id.")
@@ -117,6 +117,28 @@ export class DatabaseImpl implements Database {
     return this.updateOutput(() => db.get(id))
   }
 
+  async getMultiple<T extends Document>(
+    ids: string[],
+    opts?: { allowMissing?: boolean }
+  ): Promise<T[]> {
+    // get unique
+    ids = [...new Set(ids)]
+    const response = await this.allDocs<T>({
+      keys: ids,
+      include_docs: true,
+    })
+    const NOT_FOUND = "not_found"
+    const rows = response.rows.filter(row => row.error !== NOT_FOUND)
+    const someMissing = rows.length !== response.rows.length
+    // some were filtered out - means some missing
+    if (!opts?.allowMissing && someMissing) {
+      const missing = response.rows.filter(row => row.error === NOT_FOUND)
+      const missingIds = missing.map(row => row.key).join(", ")
+      throw new Error(`Unable to get documents: ${missingIds}`)
+    }
+    return rows.map(row => row.doc!)
+  }
+
   async remove(idOrDoc: string | Document, rev?: string) {
     const db = await this.checkSetup()
     let _id: string

packages/server/src/api/controllers/row/internal.ts

@@ -1,9 +1,5 @@
 import * as linkRows from "../../../db/linkedRows"
-import {
+import { generateRowID, InternalTables } from "../../../db/utils"
-  generateRowID,
-  getMultiIDParams,
-  InternalTables,
-} from "../../../db/utils"
 import * as userController from "../user"
 import {
   cleanupAttachments,
@@ -240,8 +236,10 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
   const linkVals = links as LinkDocumentValue[]
 
   // look up the actual rows based on the ids
-  const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id))
+  let linkedRows = await db.getMultiple<Row>(
-  let linkedRows = (await db.allDocs<Row>(params)).rows.map(row => row.doc!)
+    linkVals.map(linkVal => linkVal.id),
+    { allowMissing: true }
+  )
 
   // get the linked tables
   const linkTableIds = getLinkedTableIDs(table as Table)

packages/server/src/api/controllers/row/utils.ts

@@ -1,21 +1,9 @@
 import { InternalTables } from "../../../db/utils"
 import * as userController from "../user"
 import { context } from "@budibase/backend-core"
-import {
+import { Ctx, Row, UserCtx } from "@budibase/types"
-  Ctx,
-  FieldType,
-  ManyToOneRelationshipFieldMetadata,
-  OneToManyRelationshipFieldMetadata,
-  Row,
-  SearchFilters,
-  Table,
-  UserCtx,
-} from "@budibase/types"
-import { FieldTypes, NoEmptyFilterStrings } from "../../../constants"
-import sdk from "../../../sdk"
 
 import validateJs from "validate.js"
-import { cloneDeep } from "lodash/fp"
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {

packages/server/src/db/linkedRows/index.ts

@@ -8,7 +8,7 @@ import {
   getLinkedTable,
 } from "./linkUtils"
 import flatten from "lodash/flatten"
-import { getMultiIDParams, USER_METDATA_PREFIX } from "../utils"
+import { USER_METDATA_PREFIX } from "../utils"
 import partition from "lodash/partition"
 import { getGlobalUsersFromMetadata } from "../../utilities/global"
 import { processFormulas } from "../../utilities/rowProcessor"
@@ -79,9 +79,7 @@ async function getFullLinkedDocs(links: LinkDocumentValue[]) {
   const db = context.getAppDB()
   const linkedRowIds = links.map(link => link.id)
   const uniqueRowIds = [...new Set(linkedRowIds)]
-  let dbRows = (await db.allDocs<Row>(getMultiIDParams(uniqueRowIds))).rows.map(
+  let dbRows = await db.getMultiple<Row>(uniqueRowIds, { allowMissing: true })
-    row => row.doc!
-  )
   // convert the unique db rows back to a full list of linked rows
   const linked = linkedRowIds
     .map(id => dbRows.find(row => row && row._id === id))

packages/server/src/db/utils.ts

@@ -283,16 +283,6 @@ export function generatePluginID(name: string) {
   return `${DocumentType.PLUGIN}${SEPARATOR}${name}`
 }
 
-/**
- * This can be used with the db.allDocs to get a list of IDs
- */
-export function getMultiIDParams(ids: string[]) {
-  return {
-    keys: ids,
-    include_docs: true,
-  }
-}
-
 /**
  * Generates a new view ID.
  * @returns The new view ID which the view doc can be stored under.

packages/server/src/sdk/app/tables/getters.ts

@@ -1,5 +1,5 @@
 import { context } from "@budibase/backend-core"
-import { getMultiIDParams, getTableParams } from "../../../db/utils"
+import { getTableParams } from "../../../db/utils"
 import {
   breakExternalTableId,
   isExternalTableID,
@@ -17,6 +17,9 @@ import datasources from "../datasources"
 import sdk from "../../../sdk"
 
 export function processTable(table: Table): Table {
+  if (!table) {
+    return table
+  }
   if (table._id && isExternalTableID(table._id)) {
     return {
       ...table,
@@ -73,6 +76,9 @@ export async function getExternalTable(
   tableName: string
 ): Promise<Table> {
   const entities = await getExternalTablesInDatasource(datasourceId)
+  if (!entities[tableName]) {
+    throw new Error(`Unable to find table named "${tableName}"`)
+  }
   return processTable(entities[tableName])
 }
 
@@ -124,10 +130,10 @@ export async function getTables(tableIds: string[]): Promise<Table[]> {
   }
   if (internalTableIds.length) {
     const db = context.getAppDB()
-    const internalTableDocs = await db.allDocs<Table>(
+    const internalTables = await db.getMultiple<Table>(internalTableIds, {
-      getMultiIDParams(internalTableIds)
+      allowMissing: true,
-    )
+    })
-    tables = tables.concat(internalTableDocs.rows.map(row => row.doc!))
+    tables = tables.concat(internalTables)
   }
   return processTables(tables)
 }

packages/server/src/sdk/tests/tables.spec.ts

@@ -0,0 +1,39 @@
+import TestConfig from "../../tests/utilities/TestConfiguration"
+import { basicTable } from "../../tests/utilities/structures"
+import { Table } from "@budibase/types"
+import sdk from "../"
+
+describe("tables", () => {
+  const config = new TestConfig()
+  let table: Table
+
+  beforeAll(async () => {
+    await config.init()
+    table = await config.api.table.create(basicTable())
+  })
+
+  describe("getTables", () => {
+    it("should be able to retrieve tables", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tables = await sdk.tables.getTables([table._id!])
+        expect(tables.length).toBe(1)
+        expect(tables[0]._id).toBe(table._id)
+        expect(tables[0].name).toBe(table.name)
+      })
+    })
+
+    it("shouldn't fail when retrieving tables that don't exist", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tables = await sdk.tables.getTables(["unknown"])
+        expect(tables.length).toBe(0)
+      })
+    })
+
+    it("should de-duplicate the IDs", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tables = await sdk.tables.getTables([table._id!, table._id!])
+        expect(tables.length).toBe(1)
+      })
+    })
+  })
+})

packages/server/src/utilities/global.ts

@@ -1,4 +1,4 @@
-import { getMultiIDParams, getGlobalIDFromUserMetadataID } from "../db/utils"
+import { getGlobalIDFromUserMetadataID } from "../db/utils"
 import {
   roles,
   db as dbCore,
@@ -96,9 +96,7 @@ export async function getRawGlobalUsers(userIds?: string[]): Promise<User[]> {
   const db = tenancy.getGlobalDB()
   let globalUsers: User[]
   if (userIds) {
-    globalUsers = (await db.allDocs<User>(getMultiIDParams(userIds))).rows.map(
+    globalUsers = await db.getMultiple<User>(userIds, { allowMissing: true })
-      row => row.doc!
-    )
   } else {
     globalUsers = (
       await db.allDocs<User>(

packages/types/src/sdk/db.ts

@@ -122,7 +122,11 @@ export interface Database {
 
   exists(): Promise<boolean>
   checkSetup(): Promise<Nano.DocumentScope<any>>
-  get<T>(id?: string): Promise<T>
+  get<T extends Document>(id?: string): Promise<T>
+  getMultiple<T extends Document>(
+    ids: string[],
+    opts?: { allowMissing?: boolean }
+  ): Promise<T[]>
   remove(
     id: string | Document,
     rev?: string