import { IncludeRelationship, Operation, PaginationJson, Row, RowSearchParams, SearchFilters, SearchResponse, SortJson, SortOrder, Table, } from "@budibase/types" import * as exporters from "../../../../api/controllers/view/exporters" import { handleRequest } from "../../../../api/controllers/row/external" import { breakExternalTableId, breakRowIdField, } from "../../../../integrations/utils" import { utils, PROTECTED_EXTERNAL_COLUMNS } from "@budibase/shared-core" import { ExportRowsParams, ExportRowsResult } from "./types" import { HTTPError } from "@budibase/backend-core" import pick from "lodash/pick" import { outputProcessing } from "../../../../utilities/rowProcessor" import sdk from "../../../" import { isSearchingByRowID } from "./utils" function getPaginationAndLimitParameters( filters: SearchFilters, paginate: boolean | undefined, bookmark: number | undefined, limit: number | undefined ): PaginationJson | undefined { let paginateObj: PaginationJson | undefined // only try set limits/pagination if we aren't doing a row ID search if (isSearchingByRowID(filters)) { return } if (paginate && !limit) { throw new Error("Cannot paginate query without a limit") } if (paginate && limit) { paginateObj = { // add one so we can track if there is another page limit: limit + 1, } if (bookmark) { paginateObj.offset = limit * bookmark } } else if (limit) { paginateObj = { limit: limit, } } return paginateObj } export async function search( options: RowSearchParams, table: Table ): Promise> { const { tableId } = options const { countRows, paginate, query, ...params } = options const { limit } = params let bookmark = (params.bookmark && parseInt(params.bookmark as string)) || undefined if (paginate && !bookmark) { bookmark = 0 } let paginateObj = getPaginationAndLimitParameters( query, paginate, bookmark, limit ) let sort: SortJson | undefined if (params.sort) { const direction = params.sortOrder === "descending" ? SortOrder.DESCENDING : SortOrder.ASCENDING sort = { [params.sort]: { direction }, } } // Make sure oneOf _id queries decode the Row IDs if (query?.oneOf?._id) { const rowIds = query.oneOf._id query.oneOf._id = rowIds.map((row: string) => { const ids = breakRowIdField(row) return ids[0] }) } try { const parameters = { filters: query, sort, paginate: paginateObj as PaginationJson, includeSqlRelationships: IncludeRelationship.INCLUDE, } const queries: Promise[] = [] queries.push(handleRequest(Operation.READ, tableId, parameters)) if (countRows) { queries.push(handleRequest(Operation.COUNT, tableId, parameters)) } const responses = await Promise.all(queries) let rows = responses[0] as Row[] const totalRows = responses.length > 1 ? (responses[1] as number) : undefined let hasNextPage = false // remove the extra row if it's there if (paginate && limit && rows.length > limit) { rows.pop() hasNextPage = true } if (options.fields) { const fields = [...options.fields, ...PROTECTED_EXTERNAL_COLUMNS] rows = rows.map((r: any) => pick(r, fields)) } rows = await outputProcessing(table, rows, { preserveLinks: true, squash: true, }) // need wrapper object for bookmarks etc when paginating const response: SearchResponse = { rows, hasNextPage } if (hasNextPage && bookmark != null) { response.bookmark = bookmark + 1 } if (totalRows != null) { response.totalRows = totalRows } if (paginate && !hasNextPage) { response.hasNextPage = false } return response } catch (err: any) { if (err.message && err.message.includes("does not exist")) { throw new Error( `Table updated externally, please re-fetch - ${err.message}` ) } else { throw err } } } export async function exportRows( options: ExportRowsParams ): Promise { const { tableId, format, columns, rowIds, query, sort, sortOrder, delimiter, customHeaders, } = options if (!tableId) { throw new HTTPError("No table ID for search provided.", 400) } const { datasourceId, tableName } = breakExternalTableId(tableId) let requestQuery: SearchFilters = {} if (rowIds?.length) { requestQuery = { oneOf: { _id: rowIds.map((row: string) => { const ids = breakRowIdField(row) if (ids.length > 1) { return ids } return ids[0] }), }, } } else { requestQuery = query || {} } const datasource = await sdk.datasources.get(datasourceId) const table = await sdk.tables.getTable(tableId) if (!datasource || !datasource.entities) { throw new HTTPError("Datasource has not been configured for plus API.", 400) } let result = await search( { tableId, query: requestQuery, sort, sortOrder }, table ) let rows: Row[] = [] let headers // Filter data to only specified columns if required if (columns && columns.length) { for (let i = 0; i < result.rows.length; i++) { rows[i] = {} for (let column of columns) { rows[i][column] = result.rows[i][column] } } headers = columns } else { rows = result.rows } const schema = datasource.entities[tableName].schema let exportRows = sdk.rows.utils.cleanExportRows( rows, schema, format, columns, customHeaders ) let content: string switch (format) { case exporters.Format.CSV: content = exporters.csv( headers ?? Object.keys(schema), exportRows, delimiter, customHeaders ) break case exporters.Format.JSON: content = exporters.json(exportRows) break case exporters.Format.JSON_WITH_SCHEMA: content = exporters.jsonWithSchema(schema, exportRows) break default: throw utils.unreachable(format) } const fileName = `export.${format}` return { fileName, content, } } export async function fetch(tableId: string): Promise { const response = await handleRequest( Operation.READ, tableId, { includeSqlRelationships: IncludeRelationship.INCLUDE, } ) const table = await sdk.tables.getTable(tableId) return await outputProcessing(table, response, { preserveLinks: true, squash: true, }) } export async function fetchRaw(tableId: string): Promise { return await handleRequest(Operation.READ, tableId, { includeSqlRelationships: IncludeRelationship.INCLUDE, }) } export async function fetchView(viewName: string) { // there are no views in external datasources, shouldn't ever be called // for now just fetch const split = viewName.split("all_") const tableId = split[1] ? split[1] : split[0] return fetch(tableId) }