From d93c99b947e6ad6c60437d59dfd0de8e95ac5c7e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 25 Jul 2023 16:12:04 +0200 Subject: [PATCH 1/6] Return data when schema is defined --- .../backend-core/src/db/couch/constants.ts | 8 +++++++ packages/backend-core/src/db/couch/index.ts | 1 + .../server/src/api/routes/tests/row.spec.ts | 22 ++++++++++++++++--- packages/server/src/sdk/app/rows/search.ts | 8 +------ .../src/sdk/app/rows/search/internal.ts | 8 +++++++ 5 files changed, 37 insertions(+), 10 deletions(-) create mode 100644 packages/backend-core/src/db/couch/constants.ts diff --git a/packages/backend-core/src/db/couch/constants.ts b/packages/backend-core/src/db/couch/constants.ts new file mode 100644 index 0000000000..0b13f8f7b3 --- /dev/null +++ b/packages/backend-core/src/db/couch/constants.ts @@ -0,0 +1,8 @@ +export const CONSTANT_INTERNAL_ROW_COLS = [ + "_id", + "_rev", + "type", + "createdAt", + "updatedAt", + "tableId", +] as const diff --git a/packages/backend-core/src/db/couch/index.ts b/packages/backend-core/src/db/couch/index.ts index c731d20d6c..e7dec00d0e 100644 --- a/packages/backend-core/src/db/couch/index.ts +++ b/packages/backend-core/src/db/couch/index.ts @@ -2,3 +2,4 @@ export * from "./connections" export * from "./DatabaseImpl" export * from "./utils" export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB" +export * from "./constants" diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index d7279148c7..7ce9c12bac 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -5,7 +5,7 @@ tk.freeze(timestamp) import { outputProcessing } from "../../../utilities/rowProcessor" import * as setup from "./utilities" const { basicRow } = setup.structures -import { context, tenancy } from "@budibase/backend-core" +import { context, db, tenancy } from "@budibase/backend-core" import { quotas } from "@budibase/pro" import { QuotaUsageType, @@ -969,7 +969,7 @@ describe("/rows", () => { } ) - it("when schema is defined, no other columns are returned", async () => { + it("when schema is defined, defined columns and row attributes are returned", async () => { const table = await config.createTable(userTable()) const rows = [] for (let i = 0; i < 10; i++) { @@ -987,9 +987,25 @@ describe("/rows", () => { }) const response = await config.api.viewV2.search(createViewResponse.id) + const anyRowAttributes: { + [K in (typeof db.CONSTANT_INTERNAL_ROW_COLS)[number]]: any + } = { + tableId: expect.anything(), + type: expect.anything(), + _id: expect.anything(), + _rev: expect.anything(), + createdAt: expect.anything(), + updatedAt: expect.anything(), + } + expect(response.body.rows).toHaveLength(10) expect(response.body.rows).toEqual( - expect.arrayContaining(rows.map(r => ({ name: r.name }))) + expect.arrayContaining( + rows.map(r => ({ + ...anyRowAttributes, + name: r.name, + })) + ) ) }) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 4937460686..a523d828f3 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -3,7 +3,6 @@ import { isExternalTable } from "../../../integrations/utils" import * as internal from "./search/internal" import * as external from "./search/external" import { Format } from "../../../api/controllers/view/exporters" -import _ from "lodash" export interface SearchParams { tableId: string @@ -37,12 +36,7 @@ export async function search(options: SearchParams): Promise<{ hasNextPage?: boolean bookmark?: number | null }> { - const result = await pickApi(options.tableId).search(options) - - if (options.fields) { - result.rows = result.rows.map((r: any) => _.pick(r, options.fields!)) - } - return result + return pickApi(options.tableId).search(options) } export interface ExportRowsParams { diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index fb0e35b650..5a29541705 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -1,5 +1,6 @@ import { context, + db, SearchParams as InternalSearchParams, } from "@budibase/backend-core" import env from "../../../../environment" @@ -28,6 +29,7 @@ import { } from "../../../../api/controllers/view/utils" import sdk from "../../../../sdk" import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search" +import pick from "lodash/pick" export async function search(options: SearchParams) { const { tableId } = options @@ -72,6 +74,12 @@ export async function search(options: SearchParams) { response.rows = await getGlobalUsersFromMetadata(response.rows) } table = table || (await sdk.tables.getTable(tableId)) + + if (options.fields) { + const fields = [...options.fields, ...db.CONSTANT_INTERNAL_ROW_COLS] + response.rows = response.rows.map((r: any) => pick(r, fields)) + } + response.rows = await outputProcessing(table, response.rows) } From 3c28d112b525afb0ddd6011fb6b43e0ea98d1963 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 25 Jul 2023 16:26:25 +0200 Subject: [PATCH 2/6] Add internal sdk tests --- .../app/rows/search/tests/internal.spec.ts | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 packages/server/src/sdk/app/rows/search/tests/internal.spec.ts diff --git a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts new file mode 100644 index 0000000000..c91726b2d4 --- /dev/null +++ b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts @@ -0,0 +1,82 @@ +import { FieldType, Row, Table } from "@budibase/types" +import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" +import { SearchParams } from "../../search" +import { search } from "../internal" +import { generator } from "@budibase/backend-core/tests" + +describe("internal", () => { + const config = new TestConfiguration() + + const tableData: Table = { + name: generator.word(), + type: "table", + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + type: FieldType.STRING, + }, + }, + surname: { + name: "surname", + type: FieldType.STRING, + constraints: { + type: FieldType.STRING, + }, + }, + age: { + name: "age", + type: FieldType.NUMBER, + constraints: { + type: FieldType.NUMBER, + }, + }, + address: { + name: "address", + type: FieldType.STRING, + constraints: { + type: FieldType.STRING, + }, + }, + }, + } + + beforeAll(async () => { + await config.init() + }) + + describe("search", () => { + const rows: Row[] = [] + beforeAll(async () => { + await config.createTable(tableData) + for (let i = 0; i < 10; i++) { + rows.push( + await config.createRow({ + name: generator.first(), + surname: generator.last(), + age: generator.age(), + address: generator.address(), + }) + ) + } + }) + + it("default search returns all the data", async () => { + await config.doInContext(config.appId, async () => { + const tableId = config.table!._id! + + const searchParams: SearchParams = { + tableId, + query: {}, + } + const result = await search(searchParams) + + expect(result.rows).toHaveLength(10) + expect(result.rows).toEqual( + expect.arrayContaining(rows.map(r => expect.objectContaining(r))) + ) + }) + }) + }) +}) From 67f502579e1fc509df6578f01ec6d5cd56c13a39 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 25 Jul 2023 16:32:26 +0200 Subject: [PATCH 3/6] Test schema --- .../tests/core/utilities/jestUtils.ts | 13 +++++++++ .../server/src/api/routes/tests/row.spec.ts | 19 ++++-------- .../app/rows/search/tests/internal.spec.ts | 29 ++++++++++++++++++- 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/packages/backend-core/tests/core/utilities/jestUtils.ts b/packages/backend-core/tests/core/utilities/jestUtils.ts index d84eac548c..41b993cd17 100644 --- a/packages/backend-core/tests/core/utilities/jestUtils.ts +++ b/packages/backend-core/tests/core/utilities/jestUtils.ts @@ -1,3 +1,5 @@ +import { db } from "../../../src" + export function expectFunctionWasCalledTimesWith( jestFunction: any, times: number, @@ -7,3 +9,14 @@ export function expectFunctionWasCalledTimesWith( jestFunction.mock.calls.filter((call: any) => call[0] === argument).length ).toBe(times) } + +export const expectAnyInternalColsAttributes: { + [K in (typeof db.CONSTANT_INTERNAL_ROW_COLS)[number]]: any +} = { + tableId: expect.anything(), + type: expect.anything(), + _id: expect.anything(), + _rev: expect.anything(), + createdAt: expect.anything(), + updatedAt: expect.anything(), +} diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 7ce9c12bac..6a5cfa77a2 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -17,7 +17,11 @@ import { SortType, SortOrder, } from "@budibase/types" -import { generator, structures } from "@budibase/backend-core/tests" +import { + expectAnyInternalColsAttributes, + generator, + structures, +} from "@budibase/backend-core/tests" describe("/rows", () => { let request = setup.getRequest() @@ -987,22 +991,11 @@ describe("/rows", () => { }) const response = await config.api.viewV2.search(createViewResponse.id) - const anyRowAttributes: { - [K in (typeof db.CONSTANT_INTERNAL_ROW_COLS)[number]]: any - } = { - tableId: expect.anything(), - type: expect.anything(), - _id: expect.anything(), - _rev: expect.anything(), - createdAt: expect.anything(), - updatedAt: expect.anything(), - } - expect(response.body.rows).toHaveLength(10) expect(response.body.rows).toEqual( expect.arrayContaining( rows.map(r => ({ - ...anyRowAttributes, + ...expectAnyInternalColsAttributes, name: r.name, })) ) diff --git a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts index c91726b2d4..a58c368cea 100644 --- a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts @@ -2,7 +2,10 @@ import { FieldType, Row, Table } from "@budibase/types" import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" import { SearchParams } from "../../search" import { search } from "../internal" -import { generator } from "@budibase/backend-core/tests" +import { + expectAnyInternalColsAttributes, + generator, +} from "@budibase/backend-core/tests" describe("internal", () => { const config = new TestConfiguration() @@ -78,5 +81,29 @@ describe("internal", () => { ) }) }) + + it("querying by fields will always return data attribute columns", async () => { + await config.doInContext(config.appId, async () => { + const tableId = config.table!._id! + + const searchParams: SearchParams = { + tableId, + query: {}, + fields: ["name", "age"], + } + const result = await search(searchParams) + + expect(result.rows).toHaveLength(10) + expect(result.rows).toEqual( + expect.arrayContaining( + rows.map(r => ({ + ...expectAnyInternalColsAttributes, + name: r.name, + age: r.age, + })) + ) + ) + }) + }) }) }) From 3ce92e8034f4be882214a3045aee989cf5d15f96 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 25 Jul 2023 17:14:38 +0200 Subject: [PATCH 4/6] Implement and test external field selector --- .../src/db/{couch => }/constants.ts | 2 + packages/backend-core/src/db/couch/index.ts | 2 +- .../tests/core/utilities/jestUtils.ts | 8 + .../src/sdk/app/rows/search/external.ts | 11 +- .../app/rows/search/tests/external.spec.ts | 141 ++++++++++++++++++ 5 files changed, 161 insertions(+), 3 deletions(-) rename packages/backend-core/src/db/{couch => }/constants.ts (61%) create mode 100644 packages/server/src/sdk/app/rows/search/tests/external.spec.ts diff --git a/packages/backend-core/src/db/couch/constants.ts b/packages/backend-core/src/db/constants.ts similarity index 61% rename from packages/backend-core/src/db/couch/constants.ts rename to packages/backend-core/src/db/constants.ts index 0b13f8f7b3..aea485e3e3 100644 --- a/packages/backend-core/src/db/couch/constants.ts +++ b/packages/backend-core/src/db/constants.ts @@ -6,3 +6,5 @@ export const CONSTANT_INTERNAL_ROW_COLS = [ "updatedAt", "tableId", ] as const + +export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const diff --git a/packages/backend-core/src/db/couch/index.ts b/packages/backend-core/src/db/couch/index.ts index e7dec00d0e..932efed3f7 100644 --- a/packages/backend-core/src/db/couch/index.ts +++ b/packages/backend-core/src/db/couch/index.ts @@ -2,4 +2,4 @@ export * from "./connections" export * from "./DatabaseImpl" export * from "./utils" export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB" -export * from "./constants" +export * from "../constants" diff --git a/packages/backend-core/tests/core/utilities/jestUtils.ts b/packages/backend-core/tests/core/utilities/jestUtils.ts index 41b993cd17..4a3da8db8c 100644 --- a/packages/backend-core/tests/core/utilities/jestUtils.ts +++ b/packages/backend-core/tests/core/utilities/jestUtils.ts @@ -20,3 +20,11 @@ export const expectAnyInternalColsAttributes: { createdAt: expect.anything(), updatedAt: expect.anything(), } + +export const expectAnyExternalColsAttributes: { + [K in (typeof db.CONSTANT_EXTERNAL_ROW_COLS)[number]]: any +} = { + tableId: expect.anything(), + _id: expect.anything(), + _rev: expect.anything(), +} diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index a9da764e88..159f3d84fd 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -14,7 +14,8 @@ import { breakExternalTableId } from "../../../../integrations/utils" import { cleanExportRows } from "../utils" import { utils } from "@budibase/shared-core" import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search" -import { HTTPError } from "@budibase/backend-core" +import { HTTPError, db } from "@budibase/backend-core" +import pick from "lodash/pick" export async function search(options: SearchParams) { const { tableId } = options @@ -48,7 +49,7 @@ export async function search(options: SearchParams) { } } try { - const rows = (await handleRequest(Operation.READ, tableId, { + let rows = (await handleRequest(Operation.READ, tableId, { filters: query, sort, paginate: paginateObj as PaginationJson, @@ -67,6 +68,12 @@ export async function search(options: SearchParams) { })) as Row[] hasNextPage = nextRows.length > 0 } + + if (options.fields) { + const fields = [...options.fields, ...db.CONSTANT_EXTERNAL_ROW_COLS] + rows = rows.map((r: any) => pick(r, fields)) + } + // need wrapper object for bookmarks etc when paginating return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 } } catch (err: any) { diff --git a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts new file mode 100644 index 0000000000..3333452142 --- /dev/null +++ b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts @@ -0,0 +1,141 @@ +import { GenericContainer } from "testcontainers" + +import { Datasource, FieldType, Row, SourceName, Table } from "@budibase/types" +import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" +import { SearchParams } from "../../search" +import { search } from "../external" +import { + expectAnyExternalColsAttributes, + generator, +} from "@budibase/backend-core/tests" + +jest.unmock("mysql2/promise") + +describe("external", () => { + const config = new TestConfiguration() + + let externalDatasource: Datasource + + const tableData: Table = { + name: generator.word(), + type: "external", + primary: ["id"], + schema: { + id: { + name: "id", + type: FieldType.AUTO, + autocolumn: true, + }, + name: { + name: "name", + type: FieldType.STRING, + }, + surname: { + name: "surname", + type: FieldType.STRING, + }, + age: { + name: "age", + type: FieldType.NUMBER, + }, + address: { + name: "address", + type: FieldType.STRING, + }, + }, + } + + beforeAll(async () => { + const container = await new GenericContainer("mysql") + .withExposedPorts(3306) + .withEnv("MYSQL_ROOT_PASSWORD", "admin") + .withEnv("MYSQL_DATABASE", "db") + .withEnv("MYSQL_USER", "user") + .withEnv("MYSQL_PASSWORD", "password") + .start() + + const host = container.getContainerIpAddress() + const port = container.getMappedPort(3306) + + await config.init() + + externalDatasource = await config.createDatasource({ + datasource: { + type: "datasource", + name: "Test", + source: SourceName.MYSQL, + plus: true, + config: { + host, + port, + user: "user", + database: "db", + password: "password", + rejectUnauthorized: true, + }, + }, + }) + }) + + describe("search", () => { + const rows: Row[] = [] + beforeAll(async () => { + const table = await config.createTable({ + ...tableData, + sourceId: externalDatasource._id, + }) + for (let i = 0; i < 10; i++) { + rows.push( + await config.createRow({ + tableId: table._id, + name: generator.first(), + surname: generator.last(), + age: generator.age(), + address: generator.address(), + }) + ) + } + }) + + it("default search returns all the data", async () => { + await config.doInContext(config.appId, async () => { + const tableId = config.table!._id! + + const searchParams: SearchParams = { + tableId, + query: {}, + } + const result = await search(searchParams) + + expect(result.rows).toHaveLength(10) + expect(result.rows).toEqual( + expect.arrayContaining(rows.map(r => expect.objectContaining(r))) + ) + }) + }) + + it("querying by fields will always return data attribute columns", async () => { + await config.doInContext(config.appId, async () => { + const tableId = config.table!._id! + + const searchParams: SearchParams = { + tableId, + query: {}, + fields: ["name", "age"], + } + const result = await search(searchParams) + + expect(result.rows).toHaveLength(10) + expect(result.rows).toEqual( + expect.arrayContaining( + rows.map(r => ({ + ...expectAnyExternalColsAttributes, + name: r.name, + age: r.age, + })) + ) + ) + }) + }) + }) +}) From d07f8405595a85fd877d7e0f4a3d1ea491e2856a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 25 Jul 2023 22:15:45 +0200 Subject: [PATCH 5/6] Increase timeouts --- packages/server/src/sdk/app/rows/search/tests/external.spec.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts index 3333452142..a5c7f8e617 100644 --- a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts @@ -11,6 +11,8 @@ import { jest.unmock("mysql2/promise") +jest.setTimeout(20000) + describe("external", () => { const config = new TestConfiguration() From 17b394dba94ca0e575d274fedf6db02a0be337b4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 26 Jul 2023 12:12:32 +0200 Subject: [PATCH 6/6] Increase timeouts --- packages/server/src/sdk/app/rows/search/tests/external.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts index a5c7f8e617..e2b71ac81e 100644 --- a/packages/server/src/sdk/app/rows/search/tests/external.spec.ts +++ b/packages/server/src/sdk/app/rows/search/tests/external.spec.ts @@ -11,7 +11,7 @@ import { jest.unmock("mysql2/promise") -jest.setTimeout(20000) +jest.setTimeout(30000) describe("external", () => { const config = new TestConfiguration()