From e7f1bcab9e063120795f6c7349358e09eb94309c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 14:16:12 +0200 Subject: [PATCH 1/6] Remove ctx from fetch --- packages/server/src/api/controllers/row/index.ts | 2 +- packages/server/src/sdk/app/rows/search.ts | 4 ++-- packages/server/src/sdk/app/rows/search/external.ts | 7 +++---- packages/server/src/sdk/app/rows/search/internal.ts | 13 +++++++------ 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index cff594d329..a05c640c33 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -72,7 +72,7 @@ export async function fetchView(ctx: any) { export async function fetch(ctx: any) { const tableId = utils.getTableId(ctx) - ctx.body = await quotas.addQuery(() => sdk.rows.fetch(tableId, ctx), { + ctx.body = await quotas.addQuery(() => sdk.rows.fetch(tableId), { datasourceId: tableId, }) } diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index c37494192b..56dea32025 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -29,8 +29,8 @@ export async function exportRows(tableId: string, ctx: Ctx) { return pickApi(tableId).exportRows(ctx) } -export async function fetch(tableId: string, ctx: Ctx) { - return pickApi(tableId).fetch(ctx) +export async function fetch(tableId: string) { + return pickApi(tableId).fetch(tableId) } export async function fetchView(tableId: string, ctx: Ctx) { diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index c48f984a2d..abed06eb31 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -161,8 +161,7 @@ export async function exportRows(ctx: Ctx) { return apiFileReturn(content) } -export async function fetch(ctx: Ctx) { - const tableId = ctx.params.tableId +export async function fetch(tableId: string) { return handleRequest(Operation.READ, tableId, { includeSqlRelationships: IncludeRelationship.INCLUDE, }) @@ -172,6 +171,6 @@ export async function fetchView(ctx: Ctx) { // there are no views in external datasources, shouldn't ever be called // for now just fetch const split = ctx.params.viewName.split("all_") - ctx.params.tableId = split[1] ? split[1] : split[0] - return fetch(ctx) + const tableId = split[1] ? split[1] : split[0] + return fetch(tableId) } diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index 6545761283..d3d15deb53 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -27,12 +27,13 @@ import { import sdk from "../../../../sdk" export async function search(ctx: Ctx) { + const { tableId } = ctx.params + // Fetch the whole table when running in cypress, as search doesn't work if (!env.COUCH_DB_URL && env.isCypress()) { - return { rows: await fetch(ctx) } + return { rows: await fetch(tableId) } } - const { tableId } = ctx.params const db = context.getAppDB() const { paginate, query, ...params } = ctx.request.body params.version = ctx.version @@ -121,13 +122,13 @@ export async function exportRows(ctx: Ctx) { } } -export async function fetch(ctx: Ctx) { +export async function fetch(tableId: string) { const db = context.getAppDB() - const tableId = ctx.params.tableId let table = await db.get(tableId) let rows = await getRawTableData(db, tableId) - return outputProcessing(table, rows) + const result = await outputProcessing(table, rows) + return result } async function getRawTableData(db: Database, tableId: string) { @@ -151,7 +152,7 @@ export async function fetchView(ctx: Ctx) { // if this is a table view being looked for just transfer to that if (viewName.startsWith(DocumentType.TABLE)) { ctx.params.tableId = viewName - return fetch(ctx) + return fetch(viewName) } const db = context.getAppDB() From 1bd8bdf84c285e72dd57733a71b1b99146db3f47 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 15:29:41 +0200 Subject: [PATCH 2/6] Clean ctx from fetchview --- .../server/src/api/controllers/row/index.ts | 18 +++++++++++++++--- .../server/src/api/controllers/view/index.ts | 5 ++++- packages/server/src/sdk/app/rows/search.ts | 14 ++++++++++++-- .../server/src/sdk/app/rows/search/external.ts | 4 ++-- .../server/src/sdk/app/rows/search/internal.ts | 10 +++++----- 5 files changed, 38 insertions(+), 13 deletions(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index a05c640c33..ed441f7784 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -65,9 +65,21 @@ export const save = async (ctx: any) => { } export async function fetchView(ctx: any) { const tableId = utils.getTableId(ctx) - ctx.body = await quotas.addQuery(() => sdk.rows.fetchView(tableId, ctx), { - datasourceId: tableId, - }) + const viewName = decodeURIComponent(ctx.params.viewName) + + const { calculation, group, field } = ctx.query + + ctx.body = await quotas.addQuery( + () => + sdk.rows.fetchView(tableId, viewName, { + calculation, + group, + field, + }), + { + datasourceId: tableId, + } + ) } export async function fetch(ctx: any) { diff --git a/packages/server/src/api/controllers/view/index.ts b/packages/server/src/api/controllers/view/index.ts index 83fbe7a17b..a087634292 100644 --- a/packages/server/src/api/controllers/view/index.ts +++ b/packages/server/src/api/controllers/view/index.ts @@ -162,7 +162,10 @@ export async function exportView(ctx: Ctx) { let rows = ctx.body as Row[] let schema: TableSchema = view && view.meta && view.meta.schema - const tableId = ctx.params.tableId || view.meta.tableId + const tableId = + ctx.params.tableId || + view?.meta?.tableId || + (viewName.startsWith(DocumentType.TABLE) && viewName) const table: Table = await sdk.tables.getTable(tableId) if (!schema) { schema = table.schema diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 56dea32025..f8f50df2e6 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -14,6 +14,12 @@ export interface SearchParams { sortType?: string } +export interface ViewParams { + calculation: string + group: string + field: string +} + function pickApi(tableId: any) { if (isExternalTable(tableId)) { return external @@ -33,6 +39,10 @@ export async function fetch(tableId: string) { return pickApi(tableId).fetch(tableId) } -export async function fetchView(tableId: string, ctx: Ctx) { - return pickApi(tableId).fetchView(ctx) +export async function fetchView( + tableId: string, + viewName: string, + params: ViewParams +) { + return pickApi(tableId).fetchView(viewName, params) } diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index abed06eb31..e6d910b1c9 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -167,10 +167,10 @@ export async function fetch(tableId: string) { }) } -export async function fetchView(ctx: Ctx) { +export async function fetchView(viewName: string) { // there are no views in external datasources, shouldn't ever be called // for now just fetch - const split = ctx.params.viewName.split("all_") + const split = viewName.split("all_") const tableId = split[1] ? split[1] : split[0] return fetch(tableId) } diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index d3d15deb53..7caa5083c9 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -146,17 +146,17 @@ async function getRawTableData(db: Database, tableId: string) { return rows as Row[] } -export async function fetchView(ctx: Ctx) { - const viewName = decodeURIComponent(ctx.params.viewName) - +export async function fetchView( + viewName: string, + options: { calculation: string; group: string; field: string } +) { // if this is a table view being looked for just transfer to that if (viewName.startsWith(DocumentType.TABLE)) { - ctx.params.tableId = viewName return fetch(viewName) } const db = context.getAppDB() - const { calculation, group, field } = ctx.query + const { calculation, group, field } = options const viewInfo = await getView(db, viewName) let response if (env.SELF_HOSTED) { From 90bf4655eaa924408cf095b6fa0dff4d1de3e136 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 15:57:12 +0200 Subject: [PATCH 3/6] Remove ctx from export rows (search not implemented) --- .../server/src/api/controllers/row/index.ts | 34 +++++++++++-- packages/server/src/sdk/app/rows/search.ts | 20 +++++++- .../src/sdk/app/rows/search/external.ts | 50 +++++++++---------- .../src/sdk/app/rows/search/internal.ts | 32 ++++++------ packages/server/src/sdk/app/rows/utils.ts | 3 +- 5 files changed, 92 insertions(+), 47 deletions(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index ed441f7784..78d64610b0 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -6,6 +6,8 @@ import { Ctx } from "@budibase/types" import * as utils from "./utils" import { gridSocket } from "../../../websockets" import sdk from "../../../sdk" +import * as exporters from "../view/exporters" +import { apiFileReturn } from "../../../utilities/fileSystem" function pickApi(tableId: any) { if (isExternalTable(tableId)) { @@ -164,7 +166,33 @@ export async function fetchEnrichedRow(ctx: any) { export const exportRows = async (ctx: any) => { const tableId = utils.getTableId(ctx) - ctx.body = await quotas.addQuery(() => sdk.rows.exportRows(tableId, ctx), { - datasourceId: tableId, - }) + + const format = ctx.query.format + + const { rows, columns, query } = ctx.request.body + if (typeof format !== "string" || !exporters.isFormat(format)) { + ctx.throw( + 400, + `Format ${format} not valid. Valid values: ${Object.values( + exporters.Format + )}` + ) + } + + ctx.body = await quotas.addQuery( + async () => { + const { fileName, content } = await sdk.rows.exportRows({ + tableId, + format, + rowIds: rows, + columns, + query, + }) + ctx.attachment(fileName) + return apiFileReturn(content) + }, + { + datasourceId: tableId, + } + ) } diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index f8f50df2e6..b3b01d5d6c 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -2,6 +2,7 @@ import { Ctx, SearchFilters } from "@budibase/types" import { isExternalTable } from "../../../integrations/utils" import * as internal from "./search/internal" import * as external from "./search/external" +import { Format } from "../../../api/controllers/view/exporters" export interface SearchParams { tableId: string @@ -31,8 +32,23 @@ export async function search(tableId: string, ctx: Ctx) { return pickApi(tableId).search(ctx) } -export async function exportRows(tableId: string, ctx: Ctx) { - return pickApi(tableId).exportRows(ctx) +export interface ExportRowsParams { + tableId: string + format: Format + rowIds: string[] + columns: string[] + query: string +} + +export interface ExportRowsResult { + fileName: string + content: string +} + +export async function exportRows( + options: ExportRowsParams +): Promise { + return pickApi(options.tableId).exportRows(options) } export async function fetch(tableId: string) { diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index e6d910b1c9..fe175d9d53 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -14,6 +14,7 @@ import { breakExternalTableId } from "../../../../integrations/utils" import { cleanExportRows } from "../utils" import { apiFileReturn } from "../../../../utilities/fileSystem" import { utils } from "@budibase/shared-core" +import { ExportRowsParams, ExportRowsResult } from "../search" export async function search(ctx: Ctx) { const tableId = ctx.params.tableId @@ -78,34 +79,30 @@ export async function search(ctx: Ctx) { } } -export async function exportRows(ctx: Ctx) { - const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId) - const format = ctx.query.format as string - const { columns } = ctx.request.body +export async function exportRows( + options: ExportRowsParams +): Promise { + const { tableId, format, columns, rowIds } = options + const { datasourceId, tableName } = breakExternalTableId(tableId) + const datasource = await sdk.datasources.get(datasourceId!) if (!datasource || !datasource.entities) { - ctx.throw(400, "Datasource has not been configured for plus API.") + throw ctx.throw(400, "Datasource has not been configured for plus API.") } - if (!exporters.isFormat(format)) { - ctx.throw( - 400, - `Format ${format} not valid. Valid values: ${Object.values( - exporters.Format - )}` - ) - } - - if (ctx.request.body.rows) { + if (rowIds?.length) { ctx.request.body = { query: { oneOf: { - _id: ctx.request.body.rows.map((row: string) => { + _id: rowIds.map((row: string) => { const ids = JSON.parse( decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",") ) if (ids.length > 1) { - ctx.throw(400, "Export data does not support composite keys.") + throw ctx.throw( + 400, + "Export data does not support composite keys." + ) } return ids[0] }), @@ -131,14 +128,14 @@ export async function exportRows(ctx: Ctx) { } if (!tableName) { - ctx.throw(400, "Could not find table name.") + throw ctx.throw(400, "Could not find table name.") } - let schema = datasource.entities[tableName].schema + const schema = datasource.entities[tableName].schema let exportRows = cleanExportRows(rows, schema, format, columns) let headers = Object.keys(schema) - let content + let content: string switch (format) { case exporters.Format.CSV: content = exporters.csv(headers, exportRows) @@ -150,15 +147,14 @@ export async function exportRows(ctx: Ctx) { content = exporters.jsonWithSchema(schema, exportRows) break default: - utils.unreachable(format) - break + throw utils.unreachable(format) } - const filename = `export.${format}` - - // send down the file - ctx.attachment(filename) - return apiFileReturn(content) + const fileName = `export.${format}` + return { + fileName, + content, + } } export async function fetch(tableId: string) { diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index 7caa5083c9..0407d1b630 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -25,6 +25,7 @@ import { getFromMemoryDoc, } from "../../../../api/controllers/view/utils" import sdk from "../../../../sdk" +import { ExportRowsParams, ExportRowsResult } from "../search" export async function search(ctx: Ctx) { const { tableId } = ctx.params @@ -67,15 +68,12 @@ export async function search(ctx: Ctx) { return response } -export async function exportRows(ctx: Ctx) { +export async function exportRows( + options: ExportRowsParams +): Promise { + const { tableId, format, rowIds, columns, query } = options const db = context.getAppDB() - const table = await db.get(ctx.params.tableId) - const rowIds = ctx.request.body.rows - let format = ctx.query.format - if (typeof format !== "string") { - ctx.throw(400, "Format parameter is not valid") - } - const { columns, query } = ctx.request.body + const table = await db.get(tableId) let result if (rowIds) { @@ -109,14 +107,20 @@ export async function exportRows(ctx: Ctx) { let exportRows = cleanExportRows(rows, schema, format, columns) if (format === Format.CSV) { - ctx.attachment("export.csv") - return apiFileReturn(csv(Object.keys(rows[0]), exportRows)) + return { + fileName: "export.csv", + content: csv(Object.keys(rows[0]), exportRows), + } } else if (format === Format.JSON) { - ctx.attachment("export.json") - return apiFileReturn(json(exportRows)) + return { + fileName: "export.json", + content: json(exportRows), + } } else if (format === Format.JSON_WITH_SCHEMA) { - ctx.attachment("export.json") - return apiFileReturn(jsonWithSchema(schema, exportRows)) + return { + fileName: "export.json", + content: jsonWithSchema(schema, exportRows), + } } else { throw "Format not recognised" } diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 1f5be1479b..d9f64af163 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -1,3 +1,4 @@ +import { TableSchema } from "@budibase/types" import { FieldTypes } from "../../../constants" import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" @@ -11,7 +12,7 @@ export async function getDatasourceAndQuery(json: any) { export function cleanExportRows( rows: any[], - schema: any, + schema: TableSchema, format: string, columns: string[] ) { From ccb5143383412f4efc48974af0eb1dd994bffc33 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 18:45:40 +0200 Subject: [PATCH 4/6] Remove context from search --- .../server/src/api/controllers/row/index.ts | 9 ++- packages/server/src/sdk/app/rows/search.ts | 18 +++--- .../src/sdk/app/rows/search/external.ts | 56 +++++++++---------- .../src/sdk/app/rows/search/internal.ts | 33 +++++++---- 4 files changed, 67 insertions(+), 49 deletions(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 78d64610b0..673f2adb53 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -134,9 +134,14 @@ export async function destroy(ctx: any) { export async function search(ctx: any) { const tableId = utils.getTableId(ctx) - ctx.status = 200 - ctx.body = await quotas.addQuery(() => sdk.rows.search(tableId, ctx), { + const searchParams = { + ...ctx.request.body, + tableId, + } + + ctx.status = 200 + ctx.body = await quotas.addQuery(() => sdk.rows.search(searchParams), { datasourceId: tableId, }) } diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index b3b01d5d6c..b43af78740 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -1,4 +1,4 @@ -import { Ctx, SearchFilters } from "@budibase/types" +import { SearchFilters } from "@budibase/types" import { isExternalTable } from "../../../integrations/utils" import * as internal from "./search/internal" import * as external from "./search/external" @@ -6,13 +6,15 @@ import { Format } from "../../../api/controllers/view/exporters" export interface SearchParams { tableId: string - paginate: boolean - query?: SearchFilters - bookmark?: number - limit: number + paginate?: boolean + query: SearchFilters + bookmark?: string + limit?: number sort?: string sortOrder?: string sortType?: string + version?: string + disableEscaping?: boolean } export interface ViewParams { @@ -28,8 +30,8 @@ function pickApi(tableId: any) { return internal } -export async function search(tableId: string, ctx: Ctx) { - return pickApi(tableId).search(ctx) +export async function search(options: SearchParams) { + return pickApi(options.tableId).search(options) } export interface ExportRowsParams { @@ -37,7 +39,7 @@ export interface ExportRowsParams { format: Format rowIds: string[] columns: string[] - query: string + query: SearchFilters } export interface ExportRowsResult { diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index fe175d9d53..3317b0e398 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -5,22 +5,23 @@ import { PaginationJson, IncludeRelationship, Row, - Ctx, + SearchFilters, } from "@budibase/types" import * as exporters from "../../../../api/controllers/view/exporters" import sdk from "../../../../sdk" import { handleRequest } from "../../../../api/controllers/row/external" import { breakExternalTableId } from "../../../../integrations/utils" import { cleanExportRows } from "../utils" -import { apiFileReturn } from "../../../../utilities/fileSystem" import { utils } from "@budibase/shared-core" -import { ExportRowsParams, ExportRowsResult } from "../search" +import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search" +import { HTTPError } from "@budibase/backend-core" -export async function search(ctx: Ctx) { - const tableId = ctx.params.tableId - const { paginate, query, ...params } = ctx.request.body - let { bookmark, limit } = params - if (!bookmark && paginate) { +export async function search(options: SearchParams) { + const { tableId } = options + const { paginate, query, ...params } = options + const { limit } = params + let bookmark = (params.bookmark && parseInt(params.bookmark)) || undefined + if (paginate && bookmark) { bookmark = 1 } let paginateObj = {} @@ -60,14 +61,14 @@ export async function search(ctx: Ctx) { sort, paginate: { limit: 1, - page: bookmark * limit + 1, + page: bookmark! * limit + 1, }, includeSqlRelationships: IncludeRelationship.INCLUDE, })) as Row[] hasNextPage = nextRows.length > 0 } // need wrapper object for bookmarks etc when paginating - return { rows, hasNextPage, bookmark: bookmark + 1 } + return { rows, hasNextPage, bookmark: (bookmark || 0) + 1 } } catch (err: any) { if (err.message && err.message.includes("does not exist")) { throw new Error( @@ -87,31 +88,30 @@ export async function exportRows( const datasource = await sdk.datasources.get(datasourceId!) if (!datasource || !datasource.entities) { - throw ctx.throw(400, "Datasource has not been configured for plus API.") + throw new HTTPError("Datasource has not been configured for plus API.", 400) } + let query: SearchFilters = {} if (rowIds?.length) { - ctx.request.body = { - query: { - oneOf: { - _id: rowIds.map((row: string) => { - const ids = JSON.parse( - decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",") + query = { + oneOf: { + _id: rowIds.map((row: string) => { + const ids = JSON.parse( + decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",") + ) + if (ids.length > 1) { + throw new HTTPError( + "Export data does not support composite keys.", + 400 ) - if (ids.length > 1) { - throw ctx.throw( - 400, - "Export data does not support composite keys." - ) - } - return ids[0] - }), - }, + } + return ids[0] + }), }, } } - let result = await search(ctx) + let result = await search({ tableId, query }) let rows: Row[] = [] // Filter data to only specified columns if required @@ -128,7 +128,7 @@ export async function exportRows( } if (!tableName) { - throw ctx.throw(400, "Could not find table name.") + throw new HTTPError("Could not find table name.", 400) } const schema = datasource.entities[tableName].schema let exportRows = cleanExportRows(rows, schema, format, columns) diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index 0407d1b630..70274de34a 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -1,4 +1,7 @@ -import { context } from "@budibase/backend-core" +import { + context, + SearchParams as InternalSearchParams, +} from "@budibase/backend-core" import env from "../../../../environment" import { fullSearch, paginatedSearch } from "./internalSearch" import { @@ -8,7 +11,7 @@ import { } from "../../../../db/utils" import { getGlobalUsersFromMetadata } from "../../../../utilities/global" import { outputProcessing } from "../../../../utilities/rowProcessor" -import { Ctx, Database, Row } from "@budibase/types" +import { Database, Row } from "@budibase/types" import { cleanExportRows } from "../utils" import { Format, @@ -16,7 +19,6 @@ import { json, jsonWithSchema, } from "../../../../api/controllers/view/exporters" -import { apiFileReturn } from "../../../../utilities/fileSystem" import * as inMemoryViews from "../../../../db/inMemoryView" import { migrateToInMemoryView, @@ -25,10 +27,10 @@ import { getFromMemoryDoc, } from "../../../../api/controllers/view/utils" import sdk from "../../../../sdk" -import { ExportRowsParams, ExportRowsResult } from "../search" +import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search" -export async function search(ctx: Ctx) { - const { tableId } = ctx.params +export async function search(options: SearchParams) { + const { tableId } = options // Fetch the whole table when running in cypress, as search doesn't work if (!env.COUCH_DB_URL && env.isCypress()) { @@ -36,16 +38,25 @@ export async function search(ctx: Ctx) { } const db = context.getAppDB() - const { paginate, query, ...params } = ctx.request.body - params.version = ctx.version - params.tableId = tableId + const { paginate, query } = options + + const params: InternalSearchParams = { + tableId: options.tableId, + sort: options.sort, + sortOrder: options.sortOrder, + sortType: options.sortType, + limit: options.limit, + bookmark: options.bookmark, + version: options.version, + disableEscaping: options.disableEscaping, + } let table if (params.sort && !params.sortType) { table = await db.get(tableId) const schema = table.schema const sortField = schema[params.sort] - params.sortType = sortField.type == "number" ? "number" : "string" + params.sortType = sortField.type === "number" ? "number" : "string" } let response @@ -86,7 +97,7 @@ export async function exportRows( result = await outputProcessing(table, response) } else if (query) { - let searchResponse = await search(ctx) + let searchResponse = await search({ tableId, query }) result = searchResponse.rows } From 07607c0fd2effe8a6799e5ffebac1239b457756e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 19:07:54 +0200 Subject: [PATCH 5/6] Fix tests --- packages/server/src/sdk/app/rows/search/external.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 3317b0e398..223e1dcd7b 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -20,8 +20,8 @@ export async function search(options: SearchParams) { const { tableId } = options const { paginate, query, ...params } = options const { limit } = params - let bookmark = (params.bookmark && parseInt(params.bookmark)) || undefined - if (paginate && bookmark) { + let bookmark = (params.bookmark && parseInt(params.bookmark)) || null + if (paginate && !bookmark) { bookmark = 1 } let paginateObj = {} @@ -68,7 +68,7 @@ export async function search(options: SearchParams) { hasNextPage = nextRows.length > 0 } // need wrapper object for bookmarks etc when paginating - return { rows, hasNextPage, bookmark: (bookmark || 0) + 1 } + return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 } } catch (err: any) { if (err.message && err.message.includes("does not exist")) { throw new Error( From deb256a013dddb90113c317d81250ae74a8a0162 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 19:50:06 +0200 Subject: [PATCH 6/6] Refactor and clean export tests --- .../src/api/controllers/row/external.ts | 1 + packages/server/src/sdk/app/rows/search.ts | 4 +- .../src/sdk/app/rows/search/external.ts | 10 +- packages/server/src/sdk/app/rows/utils.ts | 2 +- .../server/src/sdk/tests/rows/row.spec.ts | 115 ++++++++---------- 5 files changed, 59 insertions(+), 73 deletions(-) diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 36329b3469..2122ada068 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -49,6 +49,7 @@ export async function handleRequest( } } } + return new ExternalRequest(operation, tableId, opts?.datasource).run( opts || {} ) diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index b43af78740..87a1662a54 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -37,8 +37,8 @@ export async function search(options: SearchParams) { export interface ExportRowsParams { tableId: string format: Format - rowIds: string[] - columns: string[] + rowIds?: string[] + columns?: string[] query: SearchFilters } diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 223e1dcd7b..a9da764e88 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -86,11 +86,6 @@ export async function exportRows( const { tableId, format, columns, rowIds } = options const { datasourceId, tableName } = breakExternalTableId(tableId) - const datasource = await sdk.datasources.get(datasourceId!) - if (!datasource || !datasource.entities) { - throw new HTTPError("Datasource has not been configured for plus API.", 400) - } - let query: SearchFilters = {} if (rowIds?.length) { query = { @@ -111,6 +106,11 @@ export async function exportRows( } } + const datasource = await sdk.datasources.get(datasourceId!) + if (!datasource || !datasource.entities) { + throw new HTTPError("Datasource has not been configured for plus API.", 400) + } + let result = await search({ tableId, query }) let rows: Row[] = [] diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index d9f64af163..6a037a4ade 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -14,7 +14,7 @@ export function cleanExportRows( rows: any[], schema: TableSchema, format: string, - columns: string[] + columns?: string[] ) { let cleanRows = [...rows] diff --git a/packages/server/src/sdk/tests/rows/row.spec.ts b/packages/server/src/sdk/tests/rows/row.spec.ts index 68140345b7..08c5746f2e 100644 --- a/packages/server/src/sdk/tests/rows/row.spec.ts +++ b/packages/server/src/sdk/tests/rows/row.spec.ts @@ -1,6 +1,10 @@ import { exportRows } from "../../app/rows/search/external" import sdk from "../.." import { ExternalRequest } from "../../../api/controllers/row/ExternalRequest" +import { ExportRowsParams } from "../../app/rows/search" +import { Format } from "../../../api/controllers/view/exporters" +import { HTTPError } from "@budibase/backend-core" +import { Operation } from "@budibase/types" const mockDatasourcesGet = jest.fn() sdk.datasources.get = mockDatasourcesGet @@ -16,30 +20,21 @@ jest.mock("../../../api/controllers/view/exporters", () => ({ })) jest.mock("../../../utilities/fileSystem") -function getUserCtx() { - return { - params: { - tableId: "datasource__tablename", - }, - query: { - format: "csv", - }, - request: { - body: {}, - }, - throw: jest.fn(() => { - throw "Err" - }), - attachment: jest.fn(), - } as any -} - -describe("external row controller", () => { +describe("external row sdk", () => { describe("exportRows", () => { + function getExportOptions(): ExportRowsParams { + return { + tableId: "datasource__tablename", + format: Format.CSV, + query: {}, + } + } + + const externalRequestCall = jest.fn() beforeAll(() => { jest .spyOn(ExternalRequest.prototype, "run") - .mockImplementation(() => Promise.resolve([])) + .mockImplementation(externalRequestCall.mockResolvedValue([])) }) afterEach(() => { @@ -47,15 +42,10 @@ describe("external row controller", () => { }) it("should throw a 400 if no datasource entities are present", async () => { - let userCtx = getUserCtx() - try { - await exportRows(userCtx) - } catch (e) { - expect(userCtx.throw).toHaveBeenCalledWith( - 400, - "Datasource has not been configured for plus API." - ) - } + const exportOptions = getExportOptions() + await expect(exportRows(exportOptions)).rejects.toThrowError( + new HTTPError("Datasource has not been configured for plus API.", 400) + ) }) it("should handle single quotes from a row ID", async () => { @@ -66,51 +56,46 @@ describe("external row controller", () => { }, }, })) - let userCtx = getUserCtx() - userCtx.request.body = { - rows: ["['d001']"], - } + const exportOptions = getExportOptions() + exportOptions.rowIds = ["['d001']"] - await exportRows(userCtx) + await exportRows(exportOptions) - expect(userCtx.request.body).toEqual({ - query: { - oneOf: { - _id: ["d001"], + expect(ExternalRequest).toBeCalledTimes(1) + expect(ExternalRequest).toBeCalledWith( + Operation.READ, + exportOptions.tableId, + undefined + ) + + expect(externalRequestCall).toBeCalledTimes(1) + expect(externalRequestCall).toBeCalledWith( + expect.objectContaining({ + filters: { + oneOf: { + _id: ["d001"], + }, }, - }, - }) + }) + ) }) it("should throw a 400 if any composite keys are present", async () => { - let userCtx = getUserCtx() - userCtx.request.body = { - rows: ["[123]", "['d001'%2C'10111']"], - } - try { - await exportRows(userCtx) - } catch (e) { - expect(userCtx.throw).toHaveBeenCalledWith( - 400, - "Export data does not support composite keys." - ) - } + const exportOptions = getExportOptions() + exportOptions.rowIds = ["[123]", "['d001'%2C'10111']"] + await expect(exportRows(exportOptions)).rejects.toThrowError( + new HTTPError("Export data does not support composite keys.", 400) + ) }) it("should throw a 400 if no table name was found", async () => { - let userCtx = getUserCtx() - userCtx.params.tableId = "datasource__" - userCtx.request.body = { - rows: ["[123]"], - } - try { - await exportRows(userCtx) - } catch (e) { - expect(userCtx.throw).toHaveBeenCalledWith( - 400, - "Could not find table name." - ) - } + const exportOptions = getExportOptions() + exportOptions.tableId = "datasource__" + exportOptions.rowIds = ["[123]"] + + await expect(exportRows(exportOptions)).rejects.toThrowError( + new HTTPError("Could not find table name.", 400) + ) }) }) })