diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts new file mode 100644 index 0000000000..aea485e3e3 --- /dev/null +++ b/packages/backend-core/src/db/constants.ts @@ -0,0 +1,10 @@ +export const CONSTANT_INTERNAL_ROW_COLS = [ + "_id", + "_rev", + "type", + "createdAt", + "updatedAt", + "tableId", +] as const + +export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const diff --git a/packages/backend-core/src/db/couch/index.ts b/packages/backend-core/src/db/couch/index.ts index c731d20d6c..932efed3f7 100644 --- a/packages/backend-core/src/db/couch/index.ts +++ b/packages/backend-core/src/db/couch/index.ts @@ -2,3 +2,4 @@ export * from "./connections" export * from "./DatabaseImpl" export * from "./utils" export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB" +export * from "../constants" diff --git a/packages/backend-core/tests/core/utilities/jestUtils.ts b/packages/backend-core/tests/core/utilities/jestUtils.ts index d84eac548c..4a3da8db8c 100644 --- a/packages/backend-core/tests/core/utilities/jestUtils.ts +++ b/packages/backend-core/tests/core/utilities/jestUtils.ts @@ -1,3 +1,5 @@ +import { db } from "../../../src" + export function expectFunctionWasCalledTimesWith( jestFunction: any, times: number, @@ -7,3 +9,22 @@ export function expectFunctionWasCalledTimesWith( jestFunction.mock.calls.filter((call: any) => call[0] === argument).length ).toBe(times) } + +export const expectAnyInternalColsAttributes: { + [K in (typeof db.CONSTANT_INTERNAL_ROW_COLS)[number]]: any +} = { + tableId: expect.anything(), + type: expect.anything(), + _id: expect.anything(), + _rev: expect.anything(), + createdAt: expect.anything(), + updatedAt: expect.anything(), +} + +export const expectAnyExternalColsAttributes: { + [K in (typeof db.CONSTANT_EXTERNAL_ROW_COLS)[number]]: any +} = { + tableId: expect.anything(), + _id: expect.anything(), + _rev: expect.anything(), +} diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index d7279148c7..6a5cfa77a2 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -5,7 +5,7 @@ tk.freeze(timestamp) import { outputProcessing } from "../../../utilities/rowProcessor" import * as setup from "./utilities" const { basicRow } = setup.structures -import { context, tenancy } from "@budibase/backend-core" +import { context, db, tenancy } from "@budibase/backend-core" import { quotas } from "@budibase/pro" import { QuotaUsageType, @@ -17,7 +17,11 @@ import { SortType, SortOrder, } from "@budibase/types" -import { generator, structures } from "@budibase/backend-core/tests" +import { + expectAnyInternalColsAttributes, + generator, + structures, +} from "@budibase/backend-core/tests" describe("/rows", () => { let request = setup.getRequest() @@ -969,7 +973,7 @@ describe("/rows", () => { } ) - it("when schema is defined, no other columns are returned", async () => { + it("when schema is defined, defined columns and row attributes are returned", async () => { const table = await config.createTable(userTable()) const rows = [] for (let i = 0; i < 10; i++) { @@ -989,7 +993,12 @@ describe("/rows", () => { expect(response.body.rows).toHaveLength(10) expect(response.body.rows).toEqual( - expect.arrayContaining(rows.map(r => ({ name: r.name }))) + expect.arrayContaining( + rows.map(r => ({ + ...expectAnyInternalColsAttributes, + name: r.name, + })) + ) ) }) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 4937460686..a523d828f3 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -3,7 +3,6 @@ import { isExternalTable } from "../../../integrations/utils" import * as internal from "./search/internal" import * as external from "./search/external" import { Format } from "../../../api/controllers/view/exporters" -import _ from "lodash" export interface SearchParams { tableId: string @@ -37,12 +36,7 @@ export async function search(options: SearchParams): Promise<{ hasNextPage?: boolean bookmark?: number | null }> { - const result = await pickApi(options.tableId).search(options) - - if (options.fields) { - result.rows = result.rows.map((r: any) => _.pick(r, options.fields!)) - } - return result + return pickApi(options.tableId).search(options) } export interface ExportRowsParams { diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index a9da764e88..159f3d84fd 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -14,7 +14,8 @@ import { breakExternalTableId } from "../../../../integrations/utils" import { cleanExportRows } from "../utils" import { utils } from "@budibase/shared-core" import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search" -import { HTTPError } from "@budibase/backend-core" +import { HTTPError, db } from "@budibase/backend-core" +import pick from "lodash/pick" export async function search(options: SearchParams) { const { tableId } = options @@ -48,7 +49,7 @@ export async function search(options: SearchParams) { } } try { - const rows = (await handleRequest(Operation.READ, tableId, { + let rows = (await handleRequest(Operation.READ, tableId, { filters: query, sort, paginate: paginateObj as PaginationJson, @@ -67,6 +68,12 @@ export async function search(options: SearchParams) { })) as Row[] hasNextPage = nextRows.length > 0 } + + if (options.fields) { + const fields = [...options.fields, ...db.CONSTANT_EXTERNAL_ROW_COLS] + rows = rows.map((r: any) => pick(r, fields)) + } + // need wrapper object for bookmarks etc when paginating return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 } } catch (err: any) { diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index fb0e35b650..5a29541705 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -1,5 +1,6 @@ import { context, + db, SearchParams as InternalSearchParams, } from "@budibase/backend-core" import env from "../../../../environment" @@ -28,6 +29,7 @@ import { } from "../../../../api/controllers/view/utils" import sdk from "../../../../sdk" import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search" +import pick from "lodash/pick" export async function search(options: SearchParams) { const { tableId } = options @@ -72,6 +74,12 @@ export async function search(options: SearchParams) { response.rows = await getGlobalUsersFromMetadata(response.rows) } table = table || (await sdk.tables.getTable(tableId)) + + if (options.fields) { + const fields = [...options.fields, ...db.CONSTANT_INTERNAL_ROW_COLS] + response.rows = response.rows.map((r: any) => pick(r, fields)) + } + response.rows = await outputProcessing(table, response.rows) } diff --git a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts new file mode 100644 index 0000000000..e2b71ac81e --- /dev/null +++ b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts @@ -0,0 +1,143 @@ +import { GenericContainer } from "testcontainers" + +import { Datasource, FieldType, Row, SourceName, Table } from "@budibase/types" +import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" +import { SearchParams } from "../../search" +import { search } from "../external" +import { + expectAnyExternalColsAttributes, + generator, +} from "@budibase/backend-core/tests" + +jest.unmock("mysql2/promise") + +jest.setTimeout(30000) + +describe("external", () => { + const config = new TestConfiguration() + + let externalDatasource: Datasource + + const tableData: Table = { + name: generator.word(), + type: "external", + primary: ["id"], + schema: { + id: { + name: "id", + type: FieldType.AUTO, + autocolumn: true, + }, + name: { + name: "name", + type: FieldType.STRING, + }, + surname: { + name: "surname", + type: FieldType.STRING, + }, + age: { + name: "age", + type: FieldType.NUMBER, + }, + address: { + name: "address", + type: FieldType.STRING, + }, + }, + } + + beforeAll(async () => { + const container = await new GenericContainer("mysql") + .withExposedPorts(3306) + .withEnv("MYSQL_ROOT_PASSWORD", "admin") + .withEnv("MYSQL_DATABASE", "db") + .withEnv("MYSQL_USER", "user") + .withEnv("MYSQL_PASSWORD", "password") + .start() + + const host = container.getContainerIpAddress() + const port = container.getMappedPort(3306) + + await config.init() + + externalDatasource = await config.createDatasource({ + datasource: { + type: "datasource", + name: "Test", + source: SourceName.MYSQL, + plus: true, + config: { + host, + port, + user: "user", + database: "db", + password: "password", + rejectUnauthorized: true, + }, + }, + }) + }) + + describe("search", () => { + const rows: Row[] = [] + beforeAll(async () => { + const table = await config.createTable({ + ...tableData, + sourceId: externalDatasource._id, + }) + for (let i = 0; i < 10; i++) { + rows.push( + await config.createRow({ + tableId: table._id, + name: generator.first(), + surname: generator.last(), + age: generator.age(), + address: generator.address(), + }) + ) + } + }) + + it("default search returns all the data", async () => { + await config.doInContext(config.appId, async () => { + const tableId = config.table!._id! + + const searchParams: SearchParams = { + tableId, + query: {}, + } + const result = await search(searchParams) + + expect(result.rows).toHaveLength(10) + expect(result.rows).toEqual( + expect.arrayContaining(rows.map(r => expect.objectContaining(r))) + ) + }) + }) + + it("querying by fields will always return data attribute columns", async () => { + await config.doInContext(config.appId, async () => { + const tableId = config.table!._id! + + const searchParams: SearchParams = { + tableId, + query: {}, + fields: ["name", "age"], + } + const result = await search(searchParams) + + expect(result.rows).toHaveLength(10) + expect(result.rows).toEqual( + expect.arrayContaining( + rows.map(r => ({ + ...expectAnyExternalColsAttributes, + name: r.name, + age: r.age, + })) + ) + ) + }) + }) + }) +}) diff --git a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts new file mode 100644 index 0000000000..a58c368cea --- /dev/null +++ b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts @@ -0,0 +1,109 @@ +import { FieldType, Row, Table } from "@budibase/types" +import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" +import { SearchParams } from "../../search" +import { search } from "../internal" +import { + expectAnyInternalColsAttributes, + generator, +} from "@budibase/backend-core/tests" + +describe("internal", () => { + const config = new TestConfiguration() + + const tableData: Table = { + name: generator.word(), + type: "table", + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + type: FieldType.STRING, + }, + }, + surname: { + name: "surname", + type: FieldType.STRING, + constraints: { + type: FieldType.STRING, + }, + }, + age: { + name: "age", + type: FieldType.NUMBER, + constraints: { + type: FieldType.NUMBER, + }, + }, + address: { + name: "address", + type: FieldType.STRING, + constraints: { + type: FieldType.STRING, + }, + }, + }, + } + + beforeAll(async () => { + await config.init() + }) + + describe("search", () => { + const rows: Row[] = [] + beforeAll(async () => { + await config.createTable(tableData) + for (let i = 0; i < 10; i++) { + rows.push( + await config.createRow({ + name: generator.first(), + surname: generator.last(), + age: generator.age(), + address: generator.address(), + }) + ) + } + }) + + it("default search returns all the data", async () => { + await config.doInContext(config.appId, async () => { + const tableId = config.table!._id! + + const searchParams: SearchParams = { + tableId, + query: {}, + } + const result = await search(searchParams) + + expect(result.rows).toHaveLength(10) + expect(result.rows).toEqual( + expect.arrayContaining(rows.map(r => expect.objectContaining(r))) + ) + }) + }) + + it("querying by fields will always return data attribute columns", async () => { + await config.doInContext(config.appId, async () => { + const tableId = config.table!._id! + + const searchParams: SearchParams = { + tableId, + query: {}, + fields: ["name", "age"], + } + const result = await search(searchParams) + + expect(result.rows).toHaveLength(10) + expect(result.rows).toEqual( + expect.arrayContaining( + rows.map(r => ({ + ...expectAnyInternalColsAttributes, + name: r.name, + age: r.age, + })) + ) + ) + }) + }) + }) +})