From d1b64bcd9cbf6faffdc6f4eb5b1866231721f704 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 14 Jul 2023 17:04:59 +0200 Subject: [PATCH 1/5] Moving files and functions --- .../server/src/api/controllers/datasource.ts | 3 +- .../api/controllers/row/ExternalRequest.ts | 3 +- .../src/api/controllers/row/external.ts | 151 +---------- .../server/src/api/controllers/row/index.ts | 10 +- .../src/api/controllers/row/internal.ts | 234 +---------------- .../server/src/api/controllers/row/utils.ts | 101 ++----- packages/server/src/api/controllers/user.ts | 4 +- .../server/src/api/controllers/view/index.ts | 3 +- packages/server/src/sdk/app/rows/index.ts | 4 + packages/server/src/sdk/app/rows/search.ts | 38 +++ .../src/sdk/app/rows/search/external.ts | 152 +++++++++++ .../src/sdk/app/rows/search/internal.ts | 246 ++++++++++++++++++ .../app/rows/search}/internalSearch.ts | 0 packages/server/src/sdk/app/rows/utils.ts | 47 ++++ 14 files changed, 529 insertions(+), 467 deletions(-) create mode 100644 packages/server/src/sdk/app/rows/search.ts create mode 100644 packages/server/src/sdk/app/rows/search/external.ts create mode 100644 packages/server/src/sdk/app/rows/search/internal.ts rename packages/server/src/{api/controllers/row => sdk/app/rows/search}/internalSearch.ts (100%) create mode 100644 packages/server/src/sdk/app/rows/utils.ts diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 069842a020..65902a542e 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -9,7 +9,6 @@ import { import { destroy as tableDestroy } from "./table/internal" import { BuildSchemaErrors, InvalidColumns } from "../../constants" import { getIntegration } from "../../integrations" -import { getDatasourceAndQuery } from "./row/utils" import { invalidateDynamicVariables } from "../../threads/utils" import { db as dbCore, context, events } from "@budibase/backend-core" import { @@ -442,7 +441,7 @@ export async function find(ctx: UserCtx) { export async function query(ctx: UserCtx) { const queryJson = ctx.request.body try { - ctx.body = await getDatasourceAndQuery(queryJson) + ctx.body = await sdk.rows.utils.getDatasourceAndQuery(queryJson) } catch (err: any) { ctx.throw(400, err) } diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 0139147e35..9378eb534b 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -23,14 +23,13 @@ import { isRowId, isSQL, } from "../../../integrations/utils" -import { getDatasourceAndQuery } from "./utils" +import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils" import { FieldTypes } from "../../../constants" import { processObjectSync } from "@budibase/string-templates" import { cloneDeep } from "lodash/fp" import { processDates, processFormulas } from "../../../utilities/rowProcessor" import { db as dbCore } from "@budibase/backend-core" import sdk from "../../../sdk" -import { isEditableColumn } from "../../../sdk/app/tables/validation" export interface ManyRelationship { tableId?: string diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 44396df67b..5a9647c63d 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -1,30 +1,21 @@ -import { - FieldTypes, - NoEmptyFilterStrings, - SortDirection, -} from "../../../constants" +import { FieldTypes, NoEmptyFilterStrings } from "../../../constants" import { breakExternalTableId, breakRowIdField, } from "../../../integrations/utils" import { ExternalRequest, RunConfig } from "./ExternalRequest" -import * as exporters from "../view/exporters" -import { apiFileReturn } from "../../../utilities/fileSystem" import { + Ctx, Datasource, IncludeRelationship, Operation, - PaginationJson, Row, - SortJson, Table, UserCtx, } from "@budibase/types" import sdk from "../../../sdk" import * as utils from "./utils" -const { cleanExportRows } = require("./utils") - async function getRow( tableId: string, rowId: string, @@ -114,21 +105,6 @@ export async function save(ctx: UserCtx) { } } -export async function fetchView(ctx: UserCtx) { - // there are no views in external datasources, shouldn't ever be called - // for now just fetch - const split = ctx.params.viewName.split("all_") - ctx.params.tableId = split[1] ? split[1] : split[0] - return fetch(ctx) -} - -export async function fetch(ctx: UserCtx) { - const tableId = ctx.params.tableId - return handleRequest(Operation.READ, tableId, { - includeSqlRelationships: IncludeRelationship.INCLUDE, - }) -} - export async function find(ctx: UserCtx) { const id = ctx.params.rowId const tableId = ctx.params.tableId @@ -161,129 +137,6 @@ export async function bulkDestroy(ctx: UserCtx) { return { response: { ok: true }, rows: responses.map(resp => resp.row) } } -export async function search(ctx: UserCtx) { - const tableId = ctx.params.tableId - const { paginate, query, ...params } = ctx.request.body - let { bookmark, limit } = params - if (!bookmark && paginate) { - bookmark = 1 - } - let paginateObj = {} - - if (paginate) { - paginateObj = { - // add one so we can track if there is another page - limit: limit, - page: bookmark, - } - } else if (params && limit) { - paginateObj = { - limit: limit, - } - } - let sort: SortJson | undefined - if (params.sort) { - const direction = - params.sortOrder === "descending" - ? SortDirection.DESCENDING - : SortDirection.ASCENDING - sort = { - [params.sort]: { direction }, - } - } - try { - const rows = (await handleRequest(Operation.READ, tableId, { - filters: query, - sort, - paginate: paginateObj as PaginationJson, - includeSqlRelationships: IncludeRelationship.INCLUDE, - })) as Row[] - let hasNextPage = false - if (paginate && rows.length === limit) { - const nextRows = (await handleRequest(Operation.READ, tableId, { - filters: query, - sort, - paginate: { - limit: 1, - page: bookmark * limit + 1, - }, - includeSqlRelationships: IncludeRelationship.INCLUDE, - })) as Row[] - hasNextPage = nextRows.length > 0 - } - // need wrapper object for bookmarks etc when paginating - return { rows, hasNextPage, bookmark: bookmark + 1 } - } catch (err: any) { - if (err.message && err.message.includes("does not exist")) { - throw new Error( - `Table updated externally, please re-fetch - ${err.message}` - ) - } else { - throw err - } - } -} - -export async function exportRows(ctx: UserCtx) { - const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId) - const format = ctx.query.format - const { columns } = ctx.request.body - const datasource = await sdk.datasources.get(datasourceId!) - if (!datasource || !datasource.entities) { - ctx.throw(400, "Datasource has not been configured for plus API.") - } - - if (ctx.request.body.rows) { - ctx.request.body = { - query: { - oneOf: { - _id: ctx.request.body.rows.map((row: string) => { - const ids = JSON.parse( - decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",") - ) - if (ids.length > 1) { - ctx.throw(400, "Export data does not support composite keys.") - } - return ids[0] - }), - }, - }, - } - } - - let result = await search(ctx) - let rows: Row[] = [] - - // Filter data to only specified columns if required - - if (columns && columns.length) { - for (let i = 0; i < result.rows.length; i++) { - rows[i] = {} - for (let column of columns) { - rows[i][column] = result.rows[i][column] - } - } - } else { - rows = result.rows - } - - if (!tableName) { - ctx.throw(400, "Could not find table name.") - } - let schema = datasource.entities[tableName].schema - let exportRows = cleanExportRows(rows, schema, format, columns) - - let headers = Object.keys(schema) - - // @ts-ignore - const exporter = exporters[format] - const filename = `export.${format}` - - // send down the file - ctx.attachment(filename) - return apiFileReturn(exporter(headers, exportRows)) -} - export async function fetchEnrichedRow(ctx: UserCtx) { const id = ctx.params.rowId const tableId = ctx.params.tableId diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 91270429a4..cff594d329 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -5,6 +5,7 @@ import { isExternalTable } from "../../../integrations/utils" import { Ctx } from "@budibase/types" import * as utils from "./utils" import { gridSocket } from "../../../websockets" +import sdk from "../../../sdk" function pickApi(tableId: any) { if (isExternalTable(tableId)) { @@ -64,14 +65,14 @@ export const save = async (ctx: any) => { } export async function fetchView(ctx: any) { const tableId = utils.getTableId(ctx) - ctx.body = await quotas.addQuery(() => pickApi(tableId).fetchView(ctx), { + ctx.body = await quotas.addQuery(() => sdk.rows.fetchView(tableId, ctx), { datasourceId: tableId, }) } export async function fetch(ctx: any) { const tableId = utils.getTableId(ctx) - ctx.body = await quotas.addQuery(() => pickApi(tableId).fetch(ctx), { + ctx.body = await quotas.addQuery(() => sdk.rows.fetch(tableId, ctx), { datasourceId: tableId, }) } @@ -120,7 +121,8 @@ export async function destroy(ctx: any) { export async function search(ctx: any) { const tableId = utils.getTableId(ctx) ctx.status = 200 - ctx.body = await quotas.addQuery(() => pickApi(tableId).search(ctx), { + + ctx.body = await quotas.addQuery(() => sdk.rows.search(tableId, ctx), { datasourceId: tableId, }) } @@ -150,7 +152,7 @@ export async function fetchEnrichedRow(ctx: any) { export const exportRows = async (ctx: any) => { const tableId = utils.getTableId(ctx) - ctx.body = await quotas.addQuery(() => pickApi(tableId).exportRows(ctx), { + ctx.body = await quotas.addQuery(() => sdk.rows.exportRows(tableId, ctx), { datasourceId: tableId, }) } diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index 87d8cd7e9a..9808e93dfb 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -1,7 +1,6 @@ import * as linkRows from "../../../db/linkedRows" import { generateRowID, - getRowParams, getTableIDFromRowID, DocumentType, InternalTables, @@ -14,8 +13,6 @@ import { } from "../../../utilities/rowProcessor" import { FieldTypes } from "../../../constants" import * as utils from "./utils" -import { fullSearch, paginatedSearch } from "./internalSearch" -import { getGlobalUsersFromMetadata } from "../../../utilities/global" import * as inMemoryViews from "../../../db/inMemoryView" import env from "../../../environment" import { @@ -27,65 +24,7 @@ import { import { cloneDeep } from "lodash/fp" import { context, db as dbCore } from "@budibase/backend-core" import { finaliseRow, updateRelatedFormula } from "./staticFormula" -import { csv, json, jsonWithSchema, Format } from "../view/exporters" -import { apiFileReturn } from "../../../utilities/fileSystem" -import { - UserCtx, - Database, - LinkDocumentValue, - Row, - Table, -} from "@budibase/types" - -import { cleanExportRows } from "./utils" - -const CALCULATION_TYPES = { - SUM: "sum", - COUNT: "count", - STATS: "stats", -} - -async function getView(db: Database, viewName: string) { - let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc - let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc - let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView - let viewInfo, - migrate = false - try { - viewInfo = await mainGetter(db, viewName) - } catch (err: any) { - // check if it can be retrieved from design doc (needs migrated) - if (err.status !== 404) { - viewInfo = null - } else { - viewInfo = await secondaryGetter(db, viewName) - migrate = !!viewInfo - } - } - if (migrate) { - await migration(db, viewName) - } - if (!viewInfo) { - throw "View does not exist." - } - return viewInfo -} - -async function getRawTableData(ctx: UserCtx, db: Database, tableId: string) { - let rows - if (tableId === InternalTables.USER_METADATA) { - await userController.fetchMetadata(ctx) - rows = ctx.body - } else { - const response = await db.allDocs( - getRowParams(tableId, null, { - include_docs: true, - }) - ) - rows = response.rows.map(row => row.doc) - } - return rows as Row[] -} +import { UserCtx, LinkDocumentValue, Row, Table } from "@budibase/types" export async function patch(ctx: UserCtx) { const db = context.getAppDB() @@ -195,82 +134,6 @@ export async function save(ctx: UserCtx) { }) } -export async function fetchView(ctx: UserCtx) { - const viewName = decodeURIComponent(ctx.params.viewName) - - // if this is a table view being looked for just transfer to that - if (viewName.startsWith(DocumentType.TABLE)) { - ctx.params.tableId = viewName - return fetch(ctx) - } - - const db = context.getAppDB() - const { calculation, group, field } = ctx.query - const viewInfo = await getView(db, viewName) - let response - if (env.SELF_HOSTED) { - response = await db.query(`database/${viewName}`, { - include_docs: !calculation, - group: !!group, - }) - } else { - const tableId = viewInfo.meta.tableId - const data = await getRawTableData(ctx, db, tableId) - response = await inMemoryViews.runView( - viewInfo, - calculation as string, - !!group, - data - ) - } - - let rows - if (!calculation) { - response.rows = response.rows.map(row => row.doc) - let table - try { - table = await db.get(viewInfo.meta.tableId) - } catch (err) { - /* istanbul ignore next */ - table = { - schema: {}, - } - } - rows = await outputProcessing(table, response.rows) - } - - if (calculation === CALCULATION_TYPES.STATS) { - response.rows = response.rows.map(row => ({ - group: row.key, - field, - ...row.value, - avg: row.value.sum / row.value.count, - })) - rows = response.rows - } - - if ( - calculation === CALCULATION_TYPES.COUNT || - calculation === CALCULATION_TYPES.SUM - ) { - rows = response.rows.map(row => ({ - group: row.key, - field, - value: row.value, - })) - } - return rows -} - -export async function fetch(ctx: UserCtx) { - const db = context.getAppDB() - - const tableId = ctx.params.tableId - let table = await db.get(tableId) - let rows = await getRawTableData(ctx, db, tableId) - return outputProcessing(table, rows) -} - export async function find(ctx: UserCtx) { const db = dbCore.getDB(ctx.appId) const table = await db.get(ctx.params.tableId) @@ -354,101 +217,6 @@ export async function bulkDestroy(ctx: UserCtx) { return { response: { ok: true }, rows: processedRows } } -export async function search(ctx: UserCtx) { - // Fetch the whole table when running in cypress, as search doesn't work - if (!env.COUCH_DB_URL && env.isCypress()) { - return { rows: await fetch(ctx) } - } - - const { tableId } = ctx.params - const db = context.getAppDB() - const { paginate, query, ...params } = ctx.request.body - params.version = ctx.version - params.tableId = tableId - - let table - if (params.sort && !params.sortType) { - table = await db.get(tableId) - const schema = table.schema - const sortField = schema[params.sort] - params.sortType = sortField.type == "number" ? "number" : "string" - } - - let response - if (paginate) { - response = await paginatedSearch(query, params) - } else { - response = await fullSearch(query, params) - } - - // Enrich search results with relationships - if (response.rows && response.rows.length) { - // enrich with global users if from users table - if (tableId === InternalTables.USER_METADATA) { - response.rows = await getGlobalUsersFromMetadata(response.rows) - } - table = table || (await db.get(tableId)) - response.rows = await outputProcessing(table, response.rows) - } - - return response -} - -export async function exportRows(ctx: UserCtx) { - const db = context.getAppDB() - const table = await db.get(ctx.params.tableId) - const rowIds = ctx.request.body.rows - let format = ctx.query.format - if (typeof format !== "string") { - ctx.throw(400, "Format parameter is not valid") - } - const { columns, query } = ctx.request.body - - let result - if (rowIds) { - let response = ( - await db.allDocs({ - include_docs: true, - keys: rowIds, - }) - ).rows.map(row => row.doc) - - result = await outputProcessing(table, response) - } else if (query) { - let searchResponse = await search(ctx) - result = searchResponse.rows - } - - let rows: Row[] = [] - let schema = table.schema - - // Filter data to only specified columns if required - if (columns && columns.length) { - for (let i = 0; i < result.length; i++) { - rows[i] = {} - for (let column of columns) { - rows[i][column] = result[i][column] - } - } - } else { - rows = result - } - - let exportRows = cleanExportRows(rows, schema, format, columns) - if (format === Format.CSV) { - ctx.attachment("export.csv") - return apiFileReturn(csv(Object.keys(rows[0]), exportRows)) - } else if (format === Format.JSON) { - ctx.attachment("export.json") - return apiFileReturn(json(exportRows)) - } else if (format === Format.JSON_WITH_SCHEMA) { - ctx.attachment("export.json") - return apiFileReturn(jsonWithSchema(schema, exportRows)) - } else { - throw "Format not recognised" - } -} - export async function fetchEnrichedRow(ctx: UserCtx) { const db = context.getAppDB() const tableId = ctx.params.tableId diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index a213f14a08..6ff90f2b25 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -1,14 +1,19 @@ import { InternalTables } from "../../../db/utils" import * as userController from "../user" -import { FieldTypes } from "../../../constants" import { context } from "@budibase/backend-core" -import { makeExternalQuery } from "../../../integrations/base/query" -import { FieldType, Row, Table, UserCtx } from "@budibase/types" -import { Format } from "../view/exporters" +import { Ctx, FieldType, Row, Table, UserCtx } from "@budibase/types" +import { FieldTypes } from "../../../constants" import sdk from "../../../sdk" -const validateJs = require("validate.js") -const { cloneDeep } = require("lodash/fp") +import validateJs from "validate.js" +import { cloneDeep } from "lodash/fp" + +function isForeignKey(key: string, table: Table) { + const relationships = Object.values(table.schema).filter( + column => column.type === FieldType.LINK + ) + return relationships.some(relationship => relationship.foreignKey === key) +} validateJs.extend(validateJs.validators.datetime, { parse: function (value: string) { @@ -20,19 +25,6 @@ validateJs.extend(validateJs.validators.datetime, { }, }) -function isForeignKey(key: string, table: Table) { - const relationships = Object.values(table.schema).filter( - column => column.type === FieldType.LINK - ) - return relationships.some(relationship => relationship.foreignKey === key) -} - -export async function getDatasourceAndQuery(json: any) { - const datasourceId = json.endpoint.datasourceId - const datasource = await sdk.datasources.get(datasourceId) - return makeExternalQuery(datasource, json) -} - export async function findRow(ctx: UserCtx, tableId: string, rowId: string) { const db = context.getAppDB() let row @@ -52,6 +44,18 @@ export async function findRow(ctx: UserCtx, tableId: string, rowId: string) { return row } +export function getTableId(ctx: Ctx) { + if (ctx.request.body && ctx.request.body.tableId) { + return ctx.request.body.tableId + } + if (ctx.params && ctx.params.tableId) { + return ctx.params.tableId + } + if (ctx.params && ctx.params.viewName) { + return ctx.params.viewName + } +} + export async function validate({ tableId, row, @@ -81,8 +85,8 @@ export async function validate({ continue } // special case for options, need to always allow unselected (empty) - if (type === FieldTypes.OPTIONS && constraints.inclusion) { - constraints.inclusion.push(null, "") + if (type === FieldTypes.OPTIONS && constraints?.inclusion) { + constraints.inclusion.push(null as any, "") } let res @@ -94,13 +98,13 @@ export async function validate({ } row[fieldName].map((val: any) => { if ( - !constraints.inclusion.includes(val) && - constraints.inclusion.length !== 0 + !constraints?.inclusion?.includes(val) && + constraints?.inclusion?.length !== 0 ) { errors[fieldName] = "Field not in list" } }) - } else if (constraints.presence && row[fieldName].length === 0) { + } else if (constraints?.presence && row[fieldName].length === 0) { // non required MultiSelect creates an empty array, which should not throw errors errors[fieldName] = [`${fieldName} is required`] } @@ -128,52 +132,3 @@ export async function validate({ } return { valid: Object.keys(errors).length === 0, errors } } - -export function cleanExportRows( - rows: any[], - schema: any, - format: string, - columns: string[] -) { - let cleanRows = [...rows] - - const relationships = Object.entries(schema) - .filter((entry: any[]) => entry[1].type === FieldTypes.LINK) - .map(entry => entry[0]) - - relationships.forEach(column => { - cleanRows.forEach(row => { - delete row[column] - }) - delete schema[column] - }) - - if (format === Format.CSV) { - // Intended to append empty values in export - const schemaKeys = Object.keys(schema) - for (let key of schemaKeys) { - if (columns?.length && columns.indexOf(key) > 0) { - continue - } - for (let row of cleanRows) { - if (row[key] == null) { - row[key] = undefined - } - } - } - } - - return cleanRows -} - -export function getTableId(ctx: any) { - if (ctx.request.body && ctx.request.body.tableId) { - return ctx.request.body.tableId - } - if (ctx.params && ctx.params.tableId) { - return ctx.params.tableId - } - if (ctx.params && ctx.params.viewName) { - return ctx.params.viewName - } -} diff --git a/packages/server/src/api/controllers/user.ts b/packages/server/src/api/controllers/user.ts index b66f11bc1c..d4b3fe7a02 100644 --- a/packages/server/src/api/controllers/user.ts +++ b/packages/server/src/api/controllers/user.ts @@ -3,10 +3,10 @@ import { InternalTables } from "../../db/utils" import { getGlobalUsers } from "../../utilities/global" import { getFullUser } from "../../utilities/users" import { context } from "@budibase/backend-core" -import { UserCtx } from "@budibase/types" +import { Ctx, UserCtx } from "@budibase/types" import sdk from "../../sdk" -export async function fetchMetadata(ctx: UserCtx) { +export async function fetchMetadata(ctx: Ctx) { const global = await getGlobalUsers() const metadata = await sdk.users.rawUserMetadata() const users = [] diff --git a/packages/server/src/api/controllers/view/index.ts b/packages/server/src/api/controllers/view/index.ts index b194f3cb5e..83fbe7a17b 100644 --- a/packages/server/src/api/controllers/view/index.ts +++ b/packages/server/src/api/controllers/view/index.ts @@ -15,7 +15,6 @@ import { TableSchema, View, } from "@budibase/types" -import { cleanExportRows } from "../row/utils" import { builderSocket } from "../../../websockets" const { cloneDeep, isEqual } = require("lodash") @@ -169,7 +168,7 @@ export async function exportView(ctx: Ctx) { schema = table.schema } - let exportRows = cleanExportRows(rows, schema, format, []) + let exportRows = sdk.rows.utils.cleanExportRows(rows, schema, format, []) if (format === Format.CSV) { ctx.attachment(`${viewName}.csv`) diff --git a/packages/server/src/sdk/app/rows/index.ts b/packages/server/src/sdk/app/rows/index.ts index 12a44ded67..1ba91b134f 100644 --- a/packages/server/src/sdk/app/rows/index.ts +++ b/packages/server/src/sdk/app/rows/index.ts @@ -1,7 +1,11 @@ import * as attachments from "./attachments" import * as rows from "./rows" +import * as search from "./search" +import * as utils from "./utils" export default { ...attachments, ...rows, + ...search, + utils: utils, } diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts new file mode 100644 index 0000000000..c37494192b --- /dev/null +++ b/packages/server/src/sdk/app/rows/search.ts @@ -0,0 +1,38 @@ +import { Ctx, SearchFilters } from "@budibase/types" +import { isExternalTable } from "../../../integrations/utils" +import * as internal from "./search/internal" +import * as external from "./search/external" + +export interface SearchParams { + tableId: string + paginate: boolean + query?: SearchFilters + bookmark?: number + limit: number + sort?: string + sortOrder?: string + sortType?: string +} + +function pickApi(tableId: any) { + if (isExternalTable(tableId)) { + return external + } + return internal +} + +export async function search(tableId: string, ctx: Ctx) { + return pickApi(tableId).search(ctx) +} + +export async function exportRows(tableId: string, ctx: Ctx) { + return pickApi(tableId).exportRows(ctx) +} + +export async function fetch(tableId: string, ctx: Ctx) { + return pickApi(tableId).fetch(ctx) +} + +export async function fetchView(tableId: string, ctx: Ctx) { + return pickApi(tableId).fetchView(ctx) +} diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts new file mode 100644 index 0000000000..18e9d1a79b --- /dev/null +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -0,0 +1,152 @@ +import { + SortJson, + SortDirection, + Operation, + PaginationJson, + IncludeRelationship, + Row, + Ctx, +} from "@budibase/types" +import sdk from "../../../../sdk" +import { handleRequest } from "../../../../api/controllers/row/external" +import { breakExternalTableId } from "../../../../integrations/utils" +import { cleanExportRows } from "../utils" +import { apiFileReturn } from "../../../../utilities/fileSystem" + +export async function search(ctx: Ctx) { + const tableId = ctx.params.tableId + const { paginate, query, ...params } = ctx.request.body + let { bookmark, limit } = params + if (!bookmark && paginate) { + bookmark = 1 + } + let paginateObj = {} + + if (paginate) { + paginateObj = { + // add one so we can track if there is another page + limit: limit, + page: bookmark, + } + } else if (params && limit) { + paginateObj = { + limit: limit, + } + } + let sort: SortJson | undefined + if (params.sort) { + const direction = + params.sortOrder === "descending" + ? SortDirection.DESCENDING + : SortDirection.ASCENDING + sort = { + [params.sort]: { direction }, + } + } + try { + const rows = (await handleRequest(Operation.READ, tableId, { + filters: query, + sort, + paginate: paginateObj as PaginationJson, + includeSqlRelationships: IncludeRelationship.INCLUDE, + })) as Row[] + let hasNextPage = false + if (paginate && rows.length === limit) { + const nextRows = (await handleRequest(Operation.READ, tableId, { + filters: query, + sort, + paginate: { + limit: 1, + page: bookmark * limit + 1, + }, + includeSqlRelationships: IncludeRelationship.INCLUDE, + })) as Row[] + hasNextPage = nextRows.length > 0 + } + // need wrapper object for bookmarks etc when paginating + return { rows, hasNextPage, bookmark: bookmark + 1 } + } catch (err: any) { + if (err.message && err.message.includes("does not exist")) { + throw new Error( + `Table updated externally, please re-fetch - ${err.message}` + ) + } else { + throw err + } + } +} + +export async function exportRows(ctx: Ctx) { + const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId) + const format = ctx.query.format as string + const { columns } = ctx.request.body + const datasource = await sdk.datasources.get(datasourceId!) + if (!datasource || !datasource.entities) { + ctx.throw(400, "Datasource has not been configured for plus API.") + } + + if (ctx.request.body.rows) { + ctx.request.body = { + query: { + oneOf: { + _id: ctx.request.body.rows.map((row: string) => { + const ids = JSON.parse( + decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",") + ) + if (ids.length > 1) { + ctx.throw(400, "Export data does not support composite keys.") + } + return ids[0] + }), + }, + }, + } + } + + let result = await search(ctx) + let rows: Row[] = [] + + // Filter data to only specified columns if required + + if (columns && columns.length) { + for (let i = 0; i < result.rows.length; i++) { + rows[i] = {} + for (let column of columns) { + rows[i][column] = result.rows[i][column] + } + } + } else { + rows = result.rows + } + + if (!tableName) { + ctx.throw(400, "Could not find table name.") + } + let schema = datasource.entities[tableName].schema + let exportRows = cleanExportRows(rows, schema, format, columns) + + let headers = Object.keys(schema) + + // @ts-ignore + const exporter = exporters[format] + const filename = `export.${format}` + + // send down the file + ctx.attachment(filename) + return apiFileReturn(exporter(headers, exportRows)) +} + +export async function fetch(ctx: Ctx) { + const tableId = ctx.params.tableId + return handleRequest(Operation.READ, tableId, { + includeSqlRelationships: IncludeRelationship.INCLUDE, + }) +} + +export async function fetchView(ctx: Ctx) { + // there are no views in external datasources, shouldn't ever be called + // for now just fetch + const split = ctx.params.viewName.split("all_") + ctx.params.tableId = split[1] ? split[1] : split[0] + return fetch(ctx) +} diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts new file mode 100644 index 0000000000..f79bb691dd --- /dev/null +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -0,0 +1,246 @@ +import { context } from "@budibase/backend-core" +import env from "../../../../environment" +import { fullSearch, paginatedSearch } from "./internalSearch" +import { + InternalTables, + getRowParams, + DocumentType, +} from "../../../../db/utils" +import { getGlobalUsersFromMetadata } from "../../../../utilities/global" +import { outputProcessing } from "../../../../utilities/rowProcessor" +import { Ctx, Database, Row, UserCtx } from "@budibase/types" +import { cleanExportRows } from "../utils" +import { + Format, + csv, + json, + jsonWithSchema, +} from "../../../../api/controllers/view/exporters" +import { apiFileReturn } from "../../../../utilities/fileSystem" +import * as userController from "../../../../api/controllers/user" +import * as inMemoryViews from "../../../../db/inMemoryView" +import { + migrateToInMemoryView, + migrateToDesignView, + getFromDesignDoc, + getFromMemoryDoc, +} from "../../../../api/controllers/view/utils" + +export async function search(ctx: Ctx) { + // Fetch the whole table when running in cypress, as search doesn't work + if (!env.COUCH_DB_URL && env.isCypress()) { + return { rows: await fetch(ctx) } + } + + const { tableId } = ctx.params + const db = context.getAppDB() + const { paginate, query, ...params } = ctx.request.body + params.version = ctx.version + params.tableId = tableId + + let table + if (params.sort && !params.sortType) { + table = await db.get(tableId) + const schema = table.schema + const sortField = schema[params.sort] + params.sortType = sortField.type == "number" ? "number" : "string" + } + + let response + if (paginate) { + response = await paginatedSearch(query, params) + } else { + response = await fullSearch(query, params) + } + + // Enrich search results with relationships + if (response.rows && response.rows.length) { + // enrich with global users if from users table + if (tableId === InternalTables.USER_METADATA) { + response.rows = await getGlobalUsersFromMetadata(response.rows) + } + table = table || (await db.get(tableId)) + response.rows = await outputProcessing(table, response.rows) + } + + return response +} + +export async function exportRows(ctx: Ctx) { + const db = context.getAppDB() + const table = await db.get(ctx.params.tableId) + const rowIds = ctx.request.body.rows + let format = ctx.query.format + if (typeof format !== "string") { + ctx.throw(400, "Format parameter is not valid") + } + const { columns, query } = ctx.request.body + + let result + if (rowIds) { + let response = ( + await db.allDocs({ + include_docs: true, + keys: rowIds, + }) + ).rows.map(row => row.doc) + + result = await outputProcessing(table, response) + } else if (query) { + let searchResponse = await search(ctx) + result = searchResponse.rows + } + + let rows: Row[] = [] + let schema = table.schema + + // Filter data to only specified columns if required + if (columns && columns.length) { + for (let i = 0; i < result.length; i++) { + rows[i] = {} + for (let column of columns) { + rows[i][column] = result[i][column] + } + } + } else { + rows = result + } + + let exportRows = cleanExportRows(rows, schema, format, columns) + if (format === Format.CSV) { + ctx.attachment("export.csv") + return apiFileReturn(csv(Object.keys(rows[0]), exportRows)) + } else if (format === Format.JSON) { + ctx.attachment("export.json") + return apiFileReturn(json(exportRows)) + } else if (format === Format.JSON_WITH_SCHEMA) { + ctx.attachment("export.json") + return apiFileReturn(jsonWithSchema(schema, exportRows)) + } else { + throw "Format not recognised" + } +} + +export async function fetch(ctx: Ctx) { + const db = context.getAppDB() + + const tableId = ctx.params.tableId + let table = await db.get(tableId) + let rows = await getRawTableData(ctx, db, tableId) + return outputProcessing(table, rows) +} + +async function getRawTableData(ctx: Ctx, db: Database, tableId: string) { + let rows + if (tableId === InternalTables.USER_METADATA) { + await userController.fetchMetadata(ctx) + rows = ctx.body + } else { + const response = await db.allDocs( + getRowParams(tableId, null, { + include_docs: true, + }) + ) + rows = response.rows.map(row => row.doc) + } + return rows as Row[] +} + +export async function fetchView(ctx: Ctx) { + const viewName = decodeURIComponent(ctx.params.viewName) + + // if this is a table view being looked for just transfer to that + if (viewName.startsWith(DocumentType.TABLE)) { + ctx.params.tableId = viewName + return fetch(ctx) + } + + const db = context.getAppDB() + const { calculation, group, field } = ctx.query + const viewInfo = await getView(db, viewName) + let response + if (env.SELF_HOSTED) { + response = await db.query(`database/${viewName}`, { + include_docs: !calculation, + group: !!group, + }) + } else { + const tableId = viewInfo.meta.tableId + const data = await getRawTableData(ctx, db, tableId) + response = await inMemoryViews.runView( + viewInfo, + calculation as string, + !!group, + data + ) + } + + let rows + if (!calculation) { + response.rows = response.rows.map(row => row.doc) + let table + try { + table = await db.get(viewInfo.meta.tableId) + } catch (err) { + /* istanbul ignore next */ + table = { + schema: {}, + } + } + rows = await outputProcessing(table, response.rows) + } + + if (calculation === CALCULATION_TYPES.STATS) { + response.rows = response.rows.map(row => ({ + group: row.key, + field, + ...row.value, + avg: row.value.sum / row.value.count, + })) + rows = response.rows + } + + if ( + calculation === CALCULATION_TYPES.COUNT || + calculation === CALCULATION_TYPES.SUM + ) { + rows = response.rows.map(row => ({ + group: row.key, + field, + value: row.value, + })) + } + return rows +} + +const CALCULATION_TYPES = { + SUM: "sum", + COUNT: "count", + STATS: "stats", +} + +async function getView(db: Database, viewName: string) { + let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc + let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc + let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView + let viewInfo, + migrate = false + try { + viewInfo = await mainGetter(db, viewName) + } catch (err: any) { + // check if it can be retrieved from design doc (needs migrated) + if (err.status !== 404) { + viewInfo = null + } else { + viewInfo = await secondaryGetter(db, viewName) + migrate = !!viewInfo + } + } + if (migrate) { + await migration(db, viewName) + } + if (!viewInfo) { + throw "View does not exist." + } + return viewInfo +} diff --git a/packages/server/src/api/controllers/row/internalSearch.ts b/packages/server/src/sdk/app/rows/search/internalSearch.ts similarity index 100% rename from packages/server/src/api/controllers/row/internalSearch.ts rename to packages/server/src/sdk/app/rows/search/internalSearch.ts diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts new file mode 100644 index 0000000000..1f5be1479b --- /dev/null +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -0,0 +1,47 @@ +import { FieldTypes } from "../../../constants" +import { makeExternalQuery } from "../../../integrations/base/query" +import { Format } from "../../../api/controllers/view/exporters" +import sdk from "../.." + +export async function getDatasourceAndQuery(json: any) { + const datasourceId = json.endpoint.datasourceId + const datasource = await sdk.datasources.get(datasourceId) + return makeExternalQuery(datasource, json) +} + +export function cleanExportRows( + rows: any[], + schema: any, + format: string, + columns: string[] +) { + let cleanRows = [...rows] + + const relationships = Object.entries(schema) + .filter((entry: any[]) => entry[1].type === FieldTypes.LINK) + .map(entry => entry[0]) + + relationships.forEach(column => { + cleanRows.forEach(row => { + delete row[column] + }) + delete schema[column] + }) + + if (format === Format.CSV) { + // Intended to append empty values in export + const schemaKeys = Object.keys(schema) + for (let key of schemaKeys) { + if (columns?.length && columns.indexOf(key) > 0) { + continue + } + for (let row of cleanRows) { + if (row[key] == null) { + row[key] = undefined + } + } + } + } + + return cleanRows +} From 9fbad3721839a45812cdf1f9560909c4501cc35a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 10:51:52 +0200 Subject: [PATCH 2/5] Fix tests --- .../src/api/controllers/row/external.ts | 1 - .../src/sdk/app/rows/search/external.ts | 31 ++++++++++++++++-- .../tests => sdk/tests/rows}/row.spec.ts | 32 ++++++++----------- 3 files changed, 42 insertions(+), 22 deletions(-) rename packages/server/src/{api/controllers/tests => sdk/tests/rows}/row.spec.ts (77%) diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 5a9647c63d..36329b3469 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -5,7 +5,6 @@ import { } from "../../../integrations/utils" import { ExternalRequest, RunConfig } from "./ExternalRequest" import { - Ctx, Datasource, IncludeRelationship, Operation, diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 18e9d1a79b..c48f984a2d 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -7,11 +7,13 @@ import { Row, Ctx, } from "@budibase/types" +import * as exporters from "../../../../api/controllers/view/exporters" import sdk from "../../../../sdk" import { handleRequest } from "../../../../api/controllers/row/external" import { breakExternalTableId } from "../../../../integrations/utils" import { cleanExportRows } from "../utils" import { apiFileReturn } from "../../../../utilities/fileSystem" +import { utils } from "@budibase/shared-core" export async function search(ctx: Ctx) { const tableId = ctx.params.tableId @@ -85,6 +87,15 @@ export async function exportRows(ctx: Ctx) { ctx.throw(400, "Datasource has not been configured for plus API.") } + if (!exporters.isFormat(format)) { + ctx.throw( + 400, + `Format ${format} not valid. Valid values: ${Object.values( + exporters.Format + )}` + ) + } + if (ctx.request.body.rows) { ctx.request.body = { query: { @@ -127,13 +138,27 @@ export async function exportRows(ctx: Ctx) { let headers = Object.keys(schema) - // @ts-ignore - const exporter = exporters[format] + let content + switch (format) { + case exporters.Format.CSV: + content = exporters.csv(headers, exportRows) + break + case exporters.Format.JSON: + content = exporters.json(exportRows) + break + case exporters.Format.JSON_WITH_SCHEMA: + content = exporters.jsonWithSchema(schema, exportRows) + break + default: + utils.unreachable(format) + break + } + const filename = `export.${format}` // send down the file ctx.attachment(filename) - return apiFileReturn(exporter(headers, exportRows)) + return apiFileReturn(content) } export async function fetch(ctx: Ctx) { diff --git a/packages/server/src/api/controllers/tests/row.spec.ts b/packages/server/src/sdk/tests/rows/row.spec.ts similarity index 77% rename from packages/server/src/api/controllers/tests/row.spec.ts rename to packages/server/src/sdk/tests/rows/row.spec.ts index 19e86d08be..68140345b7 100644 --- a/packages/server/src/api/controllers/tests/row.spec.ts +++ b/packages/server/src/sdk/tests/rows/row.spec.ts @@ -1,14 +1,14 @@ -import { exportRows } from "../row/external" -import sdk from "../../../sdk" -import { ExternalRequest } from "../row/ExternalRequest" +import { exportRows } from "../../app/rows/search/external" +import sdk from "../.." +import { ExternalRequest } from "../../../api/controllers/row/ExternalRequest" -// @ts-ignore -sdk.datasources = { - get: jest.fn(), -} +const mockDatasourcesGet = jest.fn() +sdk.datasources.get = mockDatasourcesGet -jest.mock("../row/ExternalRequest") -jest.mock("../view/exporters", () => ({ +jest.mock("../../../api/controllers/row/ExternalRequest") + +jest.mock("../../../api/controllers/view/exporters", () => ({ + ...jest.requireActual("../../../api/controllers/view/exporters"), csv: jest.fn(), Format: { CSV: "csv", @@ -31,14 +31,15 @@ function getUserCtx() { throw "Err" }), attachment: jest.fn(), - } + } as any } describe("external row controller", () => { describe("exportRows", () => { beforeAll(() => { - //@ts-ignore - jest.spyOn(ExternalRequest.prototype, "run").mockImplementation(() => []) + jest + .spyOn(ExternalRequest.prototype, "run") + .mockImplementation(() => Promise.resolve([])) }) afterEach(() => { @@ -48,7 +49,6 @@ describe("external row controller", () => { it("should throw a 400 if no datasource entities are present", async () => { let userCtx = getUserCtx() try { - //@ts-ignore await exportRows(userCtx) } catch (e) { expect(userCtx.throw).toHaveBeenCalledWith( @@ -59,8 +59,7 @@ describe("external row controller", () => { }) it("should handle single quotes from a row ID", async () => { - //@ts-ignore - sdk.datasources.get.mockImplementation(() => ({ + mockDatasourcesGet.mockImplementation(async () => ({ entities: { tablename: { schema: {}, @@ -72,7 +71,6 @@ describe("external row controller", () => { rows: ["['d001']"], } - //@ts-ignore await exportRows(userCtx) expect(userCtx.request.body).toEqual({ @@ -90,7 +88,6 @@ describe("external row controller", () => { rows: ["[123]", "['d001'%2C'10111']"], } try { - //@ts-ignore await exportRows(userCtx) } catch (e) { expect(userCtx.throw).toHaveBeenCalledWith( @@ -107,7 +104,6 @@ describe("external row controller", () => { rows: ["[123]"], } try { - //@ts-ignore await exportRows(userCtx) } catch (e) { expect(userCtx.throw).toHaveBeenCalledWith( From e336ba4b5e171961ab0fe593ac18c58c269e0e29 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 11:13:21 +0200 Subject: [PATCH 3/5] Type and fix tests --- .../api/routes/tests/internalSearch.spec.js | 197 -------------- .../api/routes/tests/internalSearch.spec.ts | 248 ++++++++++++++++++ packages/types/src/sdk/search.ts | 2 +- 3 files changed, 249 insertions(+), 198 deletions(-) delete mode 100644 packages/server/src/api/routes/tests/internalSearch.spec.js create mode 100644 packages/server/src/api/routes/tests/internalSearch.spec.ts diff --git a/packages/server/src/api/routes/tests/internalSearch.spec.js b/packages/server/src/api/routes/tests/internalSearch.spec.js deleted file mode 100644 index 8d57d2ee1c..0000000000 --- a/packages/server/src/api/routes/tests/internalSearch.spec.js +++ /dev/null @@ -1,197 +0,0 @@ -const fetch = require("node-fetch") -fetch.mockSearch() -const search = require("../../controllers/row/internalSearch") -// this will be mocked out for _search endpoint -const PARAMS = { - tableId: "ta_12345679abcdef", - version: "1", - bookmark: null, - sort: null, - sortOrder: "ascending", - sortType: "string", -} - -function checkLucene(resp, expected, params = PARAMS) { - const query = resp.rows[0].query - const json = JSON.parse(query) - if (PARAMS.sort) { - expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`) - } - if (PARAMS.bookmark) { - expect(json.bookmark).toBe(PARAMS.bookmark) - } - expect(json.include_docs).toBe(true) - expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`) - expect(json.limit).toBe(params.limit || 50) -} - -describe("internal search", () => { - it("default query", async () => { - const response = await search.paginatedSearch({ - }, PARAMS) - checkLucene(response, `*:*`) - }) - - it("test equal query", async () => { - const response = await search.paginatedSearch({ - equal: { - "column": "1", - } - }, PARAMS) - checkLucene(response, `*:* AND column:"1"`) - }) - - it("test notEqual query", async () => { - const response = await search.paginatedSearch({ - notEqual: { - "column": "1", - } - }, PARAMS) - checkLucene(response, `*:* AND !column:"1"`) - }) - - it("test OR query", async () => { - const response = await search.paginatedSearch({ - allOr: true, - equal: { - "column": "2", - }, - notEqual: { - "column": "1", - } - }, PARAMS) - checkLucene(response, `(column:"2" OR !column:"1")`) - }) - - it("test AND query", async () => { - const response = await search.paginatedSearch({ - equal: { - "column": "2", - }, - notEqual: { - "column": "1", - } - }, PARAMS) - checkLucene(response, `(*:* AND column:"2" AND !column:"1")`) - }) - - it("test pagination query", async () => { - const updatedParams = { - ...PARAMS, - limit: 100, - bookmark: "awd", - sort: "column", - } - const response = await search.paginatedSearch({ - string: { - "column": "2", - }, - }, updatedParams) - checkLucene(response, `*:* AND column:2*`, updatedParams) - }) - - it("test range query", async () => { - const response = await search.paginatedSearch({ - range: { - "column": { low: 1, high: 2 }, - }, - }, PARAMS) - checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS) - }) - - it("test empty query", async () => { - const response = await search.paginatedSearch({ - empty: { - "column": "", - }, - }, PARAMS) - checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS) - }) - - it("test notEmpty query", async () => { - const response = await search.paginatedSearch({ - notEmpty: { - "column": "", - }, - }, PARAMS) - checkLucene(response, `*:* AND column:["" TO *]`, PARAMS) - }) - - it("test oneOf query", async () => { - const response = await search.paginatedSearch({ - oneOf: { - "column": ["a", "b"], - }, - }, PARAMS) - checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS) - }) - - it("test contains query", async () => { - const response = await search.paginatedSearch({ - contains: { - "column": "a", - "colArr": [1, 2, 3], - }, - }, PARAMS) - checkLucene(response, `(*:* AND column:a AND colArr:(1 AND 2 AND 3))`, PARAMS) - }) - - it("test multiple of same column", async () => { - const response = await search.paginatedSearch({ - allOr: true, - equal: { - "1:column": "a", - "2:column": "b", - "3:column": "c", - }, - }, PARAMS) - checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS) - }) - - it("check a weird case for lucene building", async () => { - const response = await search.paginatedSearch({ - equal: { - "1:1:column": "a", - }, - }, PARAMS) - checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS) - }) - - it("test containsAny query", async () => { - const response = await search.paginatedSearch({ - containsAny: { - "column": ["a", "b", "c"] - }, - }, PARAMS) - checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS) - }) - - it("test notContains query", async () => { - const response = await search.paginatedSearch({ - notContains: { - "column": ["a", "b", "c"] - }, - }, PARAMS) - checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS) - }) - - it("test equal without version query", async () => { - PARAMS.version = null - const response = await search.paginatedSearch({ - equal: { - "column": "1", - } - }, PARAMS) - - const query = response.rows[0].query - const json = JSON.parse(query) - if (PARAMS.sort) { - expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`) - } - if (PARAMS.bookmark) { - expect(json.bookmark).toBe(PARAMS.bookmark) - } - expect(json.include_docs).toBe(true) - expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`) - }) -}) \ No newline at end of file diff --git a/packages/server/src/api/routes/tests/internalSearch.spec.ts b/packages/server/src/api/routes/tests/internalSearch.spec.ts new file mode 100644 index 0000000000..fdae513cb6 --- /dev/null +++ b/packages/server/src/api/routes/tests/internalSearch.spec.ts @@ -0,0 +1,248 @@ +const nodeFetch = require("node-fetch") +nodeFetch.mockSearch() +import { SearchParams } from "@budibase/backend-core" +import * as search from "../../../sdk/app/rows/search/internalSearch" +import { Row } from "@budibase/types" + +// this will be mocked out for _search endpoint +const PARAMS: SearchParams = { + tableId: "ta_12345679abcdef", + version: "1", + bookmark: undefined, + sort: undefined, + sortOrder: "ascending", + sortType: "string", +} + +function checkLucene(resp: any, expected: any, params = PARAMS) { + const query = resp.rows[0].query + const json = JSON.parse(query) + if (PARAMS.sort) { + expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`) + } + if (PARAMS.bookmark) { + expect(json.bookmark).toBe(PARAMS.bookmark) + } + expect(json.include_docs).toBe(true) + expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`) + expect(json.limit).toBe(params.limit || 50) +} + +describe("internal search", () => { + it("default query", async () => { + const response = await search.paginatedSearch({}, PARAMS) + checkLucene(response, `*:*`) + }) + + it("test equal query", async () => { + const response = await search.paginatedSearch( + { + equal: { + column: "1", + }, + }, + PARAMS + ) + checkLucene(response, `*:* AND column:"1"`) + }) + + it("test notEqual query", async () => { + const response = await search.paginatedSearch( + { + notEqual: { + column: "1", + }, + }, + PARAMS + ) + checkLucene(response, `*:* AND !column:"1"`) + }) + + it("test OR query", async () => { + const response = await search.paginatedSearch( + { + allOr: true, + equal: { + column: "2", + }, + notEqual: { + column: "1", + }, + }, + PARAMS + ) + checkLucene(response, `(column:"2" OR !column:"1")`) + }) + + it("test AND query", async () => { + const response = await search.paginatedSearch( + { + equal: { + column: "2", + }, + notEqual: { + column: "1", + }, + }, + PARAMS + ) + checkLucene(response, `(*:* AND column:"2" AND !column:"1")`) + }) + + it("test pagination query", async () => { + const updatedParams = { + ...PARAMS, + limit: 100, + bookmark: "awd", + sort: "column", + } + const response = await search.paginatedSearch( + { + string: { + column: "2", + }, + }, + updatedParams + ) + checkLucene(response, `*:* AND column:2*`, updatedParams) + }) + + it("test range query", async () => { + const response = await search.paginatedSearch( + { + range: { + column: { low: 1, high: 2 }, + }, + }, + PARAMS + ) + checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS) + }) + + it("test empty query", async () => { + const response = await search.paginatedSearch( + { + empty: { + column: "", + }, + }, + PARAMS + ) + checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS) + }) + + it("test notEmpty query", async () => { + const response = await search.paginatedSearch( + { + notEmpty: { + column: "", + }, + }, + PARAMS + ) + checkLucene(response, `*:* AND column:["" TO *]`, PARAMS) + }) + + it("test oneOf query", async () => { + const response = await search.paginatedSearch( + { + oneOf: { + column: ["a", "b"], + }, + }, + PARAMS + ) + checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS) + }) + + it("test contains query", async () => { + const response = await search.paginatedSearch( + { + contains: { + column: "a", + colArr: [1, 2, 3], + }, + }, + PARAMS + ) + checkLucene( + response, + `(*:* AND column:a AND colArr:(1 AND 2 AND 3))`, + PARAMS + ) + }) + + it("test multiple of same column", async () => { + const response = await search.paginatedSearch( + { + allOr: true, + equal: { + "1:column": "a", + "2:column": "b", + "3:column": "c", + }, + }, + PARAMS + ) + checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS) + }) + + it("check a weird case for lucene building", async () => { + const response = await search.paginatedSearch( + { + equal: { + "1:1:column": "a", + }, + }, + PARAMS + ) + checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS) + }) + + it("test containsAny query", async () => { + const response = await search.paginatedSearch( + { + containsAny: { + column: ["a", "b", "c"], + }, + }, + PARAMS + ) + checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS) + }) + + it("test notContains query", async () => { + const response = await search.paginatedSearch( + { + notContains: { + column: ["a", "b", "c"], + }, + }, + PARAMS + ) + checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS) + }) + + it("test equal without version query", async () => { + PARAMS.version = undefined + const response = await search.paginatedSearch( + { + equal: { + column: "1", + }, + }, + PARAMS + ) + + const query = response.rows[0].query + const json = JSON.parse(query) + if (PARAMS.sort) { + expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`) + } + if (PARAMS.bookmark) { + expect(json.bookmark).toBe(PARAMS.bookmark) + } + expect(json.include_docs).toBe(true) + expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`) + }) +}) diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index d4c5135038..ae9aec66a2 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -32,7 +32,7 @@ export interface SearchFilters { [key: string]: any[] } contains?: { - [key: string]: any[] + [key: string]: any[] | any } notContains?: { [key: string]: any[] From 18d2802888c5b1465f2157acba9744a9609f4fad Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 11:14:34 +0200 Subject: [PATCH 4/5] Update nodejs version --- .tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index 094292d096..da92e03885 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -nodejs 14.20.1 +nodejs 14.21.3 python 3.10.0 \ No newline at end of file From 20635ae2ff9957d16fe42c28250052b80a0443e4 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 17 Jul 2023 12:18:57 +0200 Subject: [PATCH 5/5] Type and fix test --- .../tests/{queryRows.spec.js => queryRows.spec.ts} | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) rename packages/server/src/automations/tests/{queryRows.spec.js => queryRows.spec.ts} (93%) diff --git a/packages/server/src/automations/tests/queryRows.spec.js b/packages/server/src/automations/tests/queryRows.spec.ts similarity index 93% rename from packages/server/src/automations/tests/queryRows.spec.js rename to packages/server/src/automations/tests/queryRows.spec.ts index 1ce7460806..bee90df08e 100644 --- a/packages/server/src/automations/tests/queryRows.spec.js +++ b/packages/server/src/automations/tests/queryRows.spec.ts @@ -1,6 +1,6 @@ // lucene searching not supported in test due to use of PouchDB -let rows = [] -jest.mock("../../api/controllers/row/internalSearch", () => ({ +let rows: Row[] = [] +jest.mock("../../sdk/app/rows/search/internalSearch", () => ({ fullSearch: jest.fn(() => { return { rows, @@ -8,12 +8,13 @@ jest.mock("../../api/controllers/row/internalSearch", () => ({ }), paginatedSearch: jest.fn(), })) -const setup = require("./utilities") +import { Row, Table } from "@budibase/types" +import * as setup from "./utilities" const NAME = "Test" describe("Test a query step automation", () => { - let table + let table: Table let config = setup.getConfig() beforeAll(async () => { @@ -87,8 +88,8 @@ describe("Test a query step automation", () => { filters: {}, "filters-def": [ { - value: null - } + value: null, + }, ], sortColumn: "name", sortOrder: "ascending",