Implement and test external field selector

packages/backend-core/src/db/constants.ts

@@ -6,3 +6,5 @@ export const CONSTANT_INTERNAL_ROW_COLS = [
   "updatedAt",
   "tableId",
 ] as const
+
+export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const

packages/backend-core/src/db/couch/index.ts

@@ -2,4 +2,4 @@ export * from "./connections"
 export * from "./DatabaseImpl"
 export * from "./utils"
 export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB"
-export * from "./constants"
+export * from "../constants"

packages/backend-core/tests/core/utilities/jestUtils.ts

@@ -20,3 +20,11 @@ export const expectAnyInternalColsAttributes: {
   createdAt: expect.anything(),
   updatedAt: expect.anything(),
 }
+
+export const expectAnyExternalColsAttributes: {
+  [K in (typeof db.CONSTANT_EXTERNAL_ROW_COLS)[number]]: any
+} = {
+  tableId: expect.anything(),
+  _id: expect.anything(),
+  _rev: expect.anything(),
+}

packages/server/src/sdk/app/rows/search/external.ts

@@ -14,7 +14,8 @@ import { breakExternalTableId } from "../../../../integrations/utils"
 import { cleanExportRows } from "../utils"
 import { utils } from "@budibase/shared-core"
 import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
-import { HTTPError } from "@budibase/backend-core"
+import { HTTPError, db } from "@budibase/backend-core"
+import pick from "lodash/pick"
 
 export async function search(options: SearchParams) {
   const { tableId } = options
@@ -48,7 +49,7 @@ export async function search(options: SearchParams) {
     }
   }
   try {
-    const rows = (await handleRequest(Operation.READ, tableId, {
+    let rows = (await handleRequest(Operation.READ, tableId, {
       filters: query,
       sort,
       paginate: paginateObj as PaginationJson,
@@ -67,6 +68,12 @@ export async function search(options: SearchParams) {
       })) as Row[]
       hasNextPage = nextRows.length > 0
     }
+
+    if (options.fields) {
+      const fields = [...options.fields, ...db.CONSTANT_EXTERNAL_ROW_COLS]
+      rows = rows.map((r: any) => pick(r, fields))
+    }
+
     // need wrapper object for bookmarks etc when paginating
     return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 }
   } catch (err: any) {

packages/server/src/sdk/app/rows/search/tests/external.spec.ts

@@ -0,0 +1,141 @@
+import { GenericContainer } from "testcontainers"
+
+import { Datasource, FieldType, Row, SourceName, Table } from "@budibase/types"
+import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
+import { SearchParams } from "../../search"
+import { search } from "../external"
+import {
+  expectAnyExternalColsAttributes,
+  generator,
+} from "@budibase/backend-core/tests"
+
+jest.unmock("mysql2/promise")
+
+describe("external", () => {
+  const config = new TestConfiguration()
+
+  let externalDatasource: Datasource
+
+  const tableData: Table = {
+    name: generator.word(),
+    type: "external",
+    primary: ["id"],
+    schema: {
+      id: {
+        name: "id",
+        type: FieldType.AUTO,
+        autocolumn: true,
+      },
+      name: {
+        name: "name",
+        type: FieldType.STRING,
+      },
+      surname: {
+        name: "surname",
+        type: FieldType.STRING,
+      },
+      age: {
+        name: "age",
+        type: FieldType.NUMBER,
+      },
+      address: {
+        name: "address",
+        type: FieldType.STRING,
+      },
+    },
+  }
+
+  beforeAll(async () => {
+    const container = await new GenericContainer("mysql")
+      .withExposedPorts(3306)
+      .withEnv("MYSQL_ROOT_PASSWORD", "admin")
+      .withEnv("MYSQL_DATABASE", "db")
+      .withEnv("MYSQL_USER", "user")
+      .withEnv("MYSQL_PASSWORD", "password")
+      .start()
+
+    const host = container.getContainerIpAddress()
+    const port = container.getMappedPort(3306)
+
+    await config.init()
+
+    externalDatasource = await config.createDatasource({
+      datasource: {
+        type: "datasource",
+        name: "Test",
+        source: SourceName.MYSQL,
+        plus: true,
+        config: {
+          host,
+          port,
+          user: "user",
+          database: "db",
+          password: "password",
+          rejectUnauthorized: true,
+        },
+      },
+    })
+  })
+
+  describe("search", () => {
+    const rows: Row[] = []
+    beforeAll(async () => {
+      const table = await config.createTable({
+        ...tableData,
+        sourceId: externalDatasource._id,
+      })
+      for (let i = 0; i < 10; i++) {
+        rows.push(
+          await config.createRow({
+            tableId: table._id,
+            name: generator.first(),
+            surname: generator.last(),
+            age: generator.age(),
+            address: generator.address(),
+          })
+        )
+      }
+    })
+
+    it("default search returns all the data", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tableId = config.table!._id!
+
+        const searchParams: SearchParams = {
+          tableId,
+          query: {},
+        }
+        const result = await search(searchParams)
+
+        expect(result.rows).toHaveLength(10)
+        expect(result.rows).toEqual(
+          expect.arrayContaining(rows.map(r => expect.objectContaining(r)))
+        )
+      })
+    })
+
+    it("querying by fields will always return data attribute columns", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tableId = config.table!._id!
+
+        const searchParams: SearchParams = {
+          tableId,
+          query: {},
+          fields: ["name", "age"],
+        }
+        const result = await search(searchParams)
+
+        expect(result.rows).toHaveLength(10)
+        expect(result.rows).toEqual(
+          expect.arrayContaining(
+            rows.map(r => ({
+              ...expectAnyExternalColsAttributes,
+              name: r.name,
+              age: r.age,
+            }))
+          )
+        )
+      })
+    })
+  })
+})