Improving typing around search, there was duplicates of SearchParams and SearchResponse - which were a little different, bring all of this together under the types library.

This commit is contained in:
mike12345567 2024-03-28 17:57:37 +00:00
parent 12c168e6eb
commit 4c755b3af3
20 changed files with 166 additions and 167 deletions

View File

@ -1,28 +1,16 @@
import fetch from "node-fetch"
import { getCouchInfo } from "./couch"
import { SearchFilters, Row, EmptyFilterOption } from "@budibase/types"
import {
SearchFilters,
Row,
EmptyFilterOption,
SearchResponse,
SearchParams,
WithRequired,
} from "@budibase/types"
const QUERY_START_REGEX = /\d[0-9]*:/g
interface SearchResponse<T> {
rows: T[] | any[]
bookmark?: string
totalRows: number
}
export type SearchParams<T> = {
tableId?: string
sort?: string
sortOrder?: string
sortType?: string
limit?: number
bookmark?: string
version?: string
indexer?: () => Promise<any>
disableEscaping?: boolean
rows?: T | Row[]
}
export function removeKeyNumbering(key: any): string {
if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) {
const parts = key.split(":")
@ -44,7 +32,7 @@ export class QueryBuilder<T> {
#query: SearchFilters
#limit: number
#sort?: string
#bookmark?: string
#bookmark?: string | number
#sortOrder: string
#sortType: string
#includeDocs: boolean
@ -130,7 +118,7 @@ export class QueryBuilder<T> {
return this
}
setBookmark(bookmark?: string) {
setBookmark(bookmark?: string | number) {
if (bookmark != null) {
this.#bookmark = bookmark
}
@ -226,14 +214,20 @@ export class QueryBuilder<T> {
}
}
/**
* Preprocesses a value before going into a lucene search.
* Transforms strings to lowercase and wraps strings and bools in quotes.
* @param value The value to process
* @param options The preprocess options
* @returns {string|*}
*/
preprocess(value: any, { escape, lowercase, wrap, type }: any = {}) {
preprocess(
value: any,
{
escape,
lowercase,
wrap,
type,
}: {
escape?: boolean
lowercase?: boolean
wrap?: boolean
type?: string
} = {}
): string | any {
const hasVersion = !!this.#version
// Determine if type needs wrapped
const originalType = typeof value
@ -561,7 +555,7 @@ async function runQuery<T>(
url: string,
body: any,
cookie: string
): Promise<SearchResponse<T>> {
): Promise<WithRequired<SearchResponse<T>, "totalRows">> {
const response = await fetch(url, {
body: JSON.stringify(body),
method: "POST",
@ -575,7 +569,7 @@ async function runQuery<T>(
}
const json = await response.json()
let output: SearchResponse<T> = {
let output: WithRequired<SearchResponse<T>, "totalRows"> = {
rows: [],
totalRows: 0,
}
@ -613,63 +607,51 @@ async function recursiveSearch<T>(
dbName: string,
index: string,
query: any,
params: any
params: SearchParams
): Promise<any> {
const bookmark = params.bookmark
const rows = params.rows || []
if (rows.length >= params.limit) {
if (params.limit && rows.length >= params.limit) {
return rows
}
let pageSize = QueryBuilder.maxLimit
if (rows.length > params.limit - QueryBuilder.maxLimit) {
if (params.limit && rows.length > params.limit - QueryBuilder.maxLimit) {
pageSize = params.limit - rows.length
}
const page = await new QueryBuilder<T>(dbName, index, query)
const queryBuilder = new QueryBuilder<T>(dbName, index, query)
queryBuilder
.setVersion(params.version)
.setTable(params.tableId)
.setBookmark(bookmark)
.setLimit(pageSize)
.setSort(params.sort)
.setSortOrder(params.sortOrder)
.setSortType(params.sortType)
.run()
if (params.tableId) {
queryBuilder.setTable(params.tableId)
}
const page = await queryBuilder.run()
if (!page.rows.length) {
return rows
}
if (page.rows.length < QueryBuilder.maxLimit) {
return [...rows, ...page.rows]
}
const newParams = {
const newParams: SearchParams = {
...params,
bookmark: page.bookmark,
rows: [...rows, ...page.rows],
rows: [...rows, ...page.rows] as Row[],
}
return await recursiveSearch(dbName, index, query, newParams)
}
/**
* Performs a paginated search. A bookmark will be returned to allow the next
* page to be fetched. There is a max limit off 200 results per page in a
* paginated search.
* @param dbName Which database to run a lucene query on
* @param index Which search index to utilise
* @param query The JSON query structure
* @param params The search params including:
* tableId {string} The table ID to search
* sort {string} The sort column
* sortOrder {string} The sort order ("ascending" or "descending")
* sortType {string} Whether to treat sortable values as strings or
* numbers. ("string" or "number")
* limit {number} The desired page size
* bookmark {string} The bookmark to resume from
* @returns {Promise<{hasNextPage: boolean, rows: *[]}>}
*/
export async function paginatedSearch<T>(
dbName: string,
index: string,
query: SearchFilters,
params: SearchParams<T>
) {
params: SearchParams
): Promise<SearchResponse<T>> {
let limit = params.limit
if (limit == null || isNaN(limit) || limit < 0) {
limit = 50
@ -713,29 +695,12 @@ export async function paginatedSearch<T>(
}
}
/**
* Performs a full search, fetching multiple pages if required to return the
* desired amount of results. There is a limit of 1000 results to avoid
* heavy performance hits, and to avoid client components breaking from
* handling too much data.
* @param dbName Which database to run a lucene query on
* @param index Which search index to utilise
* @param query The JSON query structure
* @param params The search params including:
* tableId {string} The table ID to search
* sort {string} The sort column
* sortOrder {string} The sort order ("ascending" or "descending")
* sortType {string} Whether to treat sortable values as strings or
* numbers. ("string" or "number")
* limit {number} The desired number of results
* @returns {Promise<{rows: *}>}
*/
export async function fullSearch<T>(
dbName: string,
index: string,
query: SearchFilters,
params: SearchParams<T>
) {
params: SearchParams
): Promise<{ rows: Row[] }> {
let limit = params.limit
if (limit == null || isNaN(limit) || limit < 0) {
limit = 1000

View File

@ -1,9 +1,15 @@
import { newid } from "../../docIds/newid"
import { getDB } from "../db"
import { Database, EmptyFilterOption } from "@budibase/types"
import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene"
import {
Database,
EmptyFilterOption,
SortOrder,
SortType,
} from "@budibase/types"
import { fullSearch, paginatedSearch, QueryBuilder } from "../lucene"
const INDEX_NAME = "main"
const TABLE_ID = ""
const index = `function(doc) {
let props = ["property", "number", "array"]
@ -25,8 +31,16 @@ describe("lucene", () => {
dbName = `db-${newid()}`
// create the DB for testing
db = getDB(dbName)
await db.put({ _id: newid(), property: "word", array: ["1", "4"] })
await db.put({ _id: newid(), property: "word2", array: ["3", "1"] })
await db.put({
_id: newid(),
property: "word",
array: ["1", "4"],
})
await db.put({
_id: newid(),
property: "word2",
array: ["3", "1"],
})
await db.put({
_id: newid(),
property: "word3",
@ -338,10 +352,11 @@ describe("lucene", () => {
},
},
{
tableId: TABLE_ID,
limit: 1,
sort: "property",
sortType: "string",
sortOrder: "desc",
sortType: SortType.STRING,
sortOrder: SortOrder.DESCENDING,
}
)
expect(page.rows.length).toBe(1)
@ -360,7 +375,10 @@ describe("lucene", () => {
property: "wo",
},
},
{}
{
tableId: TABLE_ID,
query: {},
}
)
expect(page.rows.length).toBe(3)
})

View File

@ -32,7 +32,6 @@ export { default as env } from "./environment"
export * as blacklist from "./blacklist"
export * as docUpdates from "./docUpdates"
export * from "./utils/Duration"
export { SearchParams } from "./db"
export * as docIds from "./docIds"
export * as security from "./security"
// Add context to tenancy for backwards compatibility

View File

@ -13,7 +13,7 @@ import {
PatchRowRequest,
PatchRowResponse,
Row,
SearchParams,
RowSearchParams,
SearchRowRequest,
SearchRowResponse,
UserCtx,
@ -192,7 +192,7 @@ export async function destroy(ctx: UserCtx<DeleteRowRequest>) {
export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
const tableId = utils.getTableId(ctx)
const searchParams: SearchParams = {
const searchParams: RowSearchParams = {
...ctx.request.body,
tableId,
}

View File

@ -4,8 +4,8 @@ import {
SearchRowResponse,
SearchViewRowRequest,
RequiredKeys,
SearchParams,
SearchFilters,
RowSearchParams,
} from "@budibase/types"
import { dataFilters } from "@budibase/shared-core"
import sdk from "../../../sdk"
@ -57,7 +57,7 @@ export async function searchView(
}
const searchOptions: RequiredKeys<SearchViewRowRequest> &
RequiredKeys<Pick<SearchParams, "tableId" | "query" | "fields">> = {
RequiredKeys<Pick<RowSearchParams, "tableId" | "query" | "fields">> = {
tableId: view.tableId,
query,
fields: viewFields,

View File

@ -1,18 +1,18 @@
const nodeFetch = require("node-fetch")
nodeFetch.mockSearch()
import { SearchParams } from "@budibase/backend-core"
import * as search from "../../../sdk/app/rows/search/internalSearch"
import { Row } from "@budibase/types"
import * as search from "../../../sdk/app/rows/search/utils"
import { RowSearchParams, SortOrder, SortType } from "@budibase/types"
// this will be mocked out for _search endpoint
const PARAMS: SearchParams<Row> = {
const PARAMS: RowSearchParams = {
query: {},
tableId: "ta_12345679abcdef",
version: "1",
bookmark: undefined,
sort: undefined,
sortOrder: "ascending",
sortType: "string",
sortOrder: SortOrder.ASCENDING,
sortType: SortType.STRING,
}
function checkLucene(resp: any, expected: any, params = PARAMS) {

View File

@ -1,7 +1,7 @@
import {
Row,
RowSearchParams,
SearchFilters,
SearchParams,
SearchResponse,
} from "@budibase/types"
import { isExternalTableID } from "../../../integrations/utils"
@ -56,7 +56,9 @@ export function removeEmptyFilters(filters: SearchFilters) {
return filters
}
export async function search(options: SearchParams): Promise<SearchResponse> {
export async function search(
options: RowSearchParams
): Promise<SearchResponse<Row>> {
const isExternalTable = isExternalTableID(options.tableId)
if (isExternalTable) {
return external.search(options)

View File

@ -6,7 +6,7 @@ import {
IncludeRelationship,
Row,
SearchFilters,
SearchParams,
RowSearchParams,
SearchResponse,
} from "@budibase/types"
import * as exporters from "../../../../api/controllers/view/exporters"
@ -23,11 +23,14 @@ import pick from "lodash/pick"
import { outputProcessing } from "../../../../utilities/rowProcessor"
import sdk from "../../../"
export async function search(options: SearchParams): Promise<SearchResponse> {
export async function search(
options: RowSearchParams
): Promise<SearchResponse<Row>> {
const { tableId } = options
const { paginate, query, ...params } = options
const { limit } = params
let bookmark = (params.bookmark && parseInt(params.bookmark)) || undefined
let bookmark =
(params.bookmark && parseInt(params.bookmark as string)) || undefined
if (paginate && !bookmark) {
bookmark = 1
}
@ -92,7 +95,7 @@ export async function search(options: SearchParams): Promise<SearchResponse> {
rows = rows.map((r: any) => pick(r, fields))
}
rows = await outputProcessing(table, rows, {
rows = await outputProcessing<Row[]>(table, rows, {
preserveLinks: true,
squash: true,
})

View File

@ -1,19 +1,16 @@
import {
context,
db,
HTTPError,
SearchParams as InternalSearchParams,
} from "@budibase/backend-core"
import { context, db, HTTPError } from "@budibase/backend-core"
import env from "../../../../environment"
import { fullSearch, paginatedSearch } from "./internalSearch"
import { fullSearch, paginatedSearch, searchInputMapping } from "./utils"
import { getRowParams, InternalTables } from "../../../../db/utils"
import {
Database,
Row,
Table,
SearchParams,
SearchResponse,
DocumentType,
Row,
RowSearchParams,
SearchResponse,
SortType,
Table,
User,
} from "@budibase/types"
import { getGlobalUsersFromMetadata } from "../../../../utilities/global"
import { outputProcessing } from "../../../../utilities/rowProcessor"
@ -32,16 +29,17 @@ import {
} from "../../../../api/controllers/view/utils"
import sdk from "../../../../sdk"
import { ExportRowsParams, ExportRowsResult } from "./types"
import { searchInputMapping } from "./utils"
import pick from "lodash/pick"
import { breakRowIdField } from "../../../../integrations/utils"
export async function search(options: SearchParams): Promise<SearchResponse> {
export async function search(
options: RowSearchParams
): Promise<SearchResponse<Row>> {
const { tableId } = options
const { paginate, query } = options
const params: InternalSearchParams<any> = {
const params: RowSearchParams = {
tableId: options.tableId,
sort: options.sort,
sortOrder: options.sortOrder,
@ -50,6 +48,7 @@ export async function search(options: SearchParams): Promise<SearchResponse> {
bookmark: options.bookmark,
version: options.version,
disableEscaping: options.disableEscaping,
query: {},
}
let table = await sdk.tables.getTable(tableId)
@ -57,7 +56,8 @@ export async function search(options: SearchParams): Promise<SearchResponse> {
if (params.sort && !params.sortType) {
const schema = table.schema
const sortField = schema[params.sort]
params.sortType = sortField.type === "number" ? "number" : "string"
params.sortType =
sortField.type === "number" ? SortType.NUMBER : SortType.STRING
}
let response
@ -71,7 +71,7 @@ export async function search(options: SearchParams): Promise<SearchResponse> {
if (response.rows && response.rows.length) {
// enrich with global users if from users table
if (tableId === InternalTables.USER_METADATA) {
response.rows = await getGlobalUsersFromMetadata(response.rows)
response.rows = await getGlobalUsersFromMetadata(response.rows as User[])
}
if (options.fields) {

View File

@ -1,18 +0,0 @@
import { db as dbCore, context, SearchParams } from "@budibase/backend-core"
import { SearchFilters, Row, SearchIndex } from "@budibase/types"
export async function paginatedSearch(
query: SearchFilters,
params: SearchParams<Row>
) {
const appId = context.getAppId()
return dbCore.paginatedSearch(appId!, SearchIndex.ROWS, query, params)
}
export async function fullSearch(
query: SearchFilters,
params: SearchParams<Row>
) {
const appId = context.getAppId()
return dbCore.fullSearch(appId!, SearchIndex.ROWS, query, params)
}

View File

@ -5,7 +5,7 @@ import {
RelationshipFieldMetadata,
Row,
SearchFilters,
SearchParams,
RowSearchParams,
SearchResponse,
SortDirection,
SortOrder,
@ -93,7 +93,9 @@ function buildTableMap(tables: Table[]) {
return tableMap
}
export async function search(options: SearchParams): Promise<SearchResponse> {
export async function search(
options: RowSearchParams
): Promise<SearchResponse<Row>> {
const { tableId, paginate, query, ...params } = options
const builder = new SqlQueryBuilder(SqlClient.SQL_LITE)

View File

@ -6,7 +6,7 @@ import {
Row,
SourceName,
Table,
SearchParams,
RowSearchParams,
TableSourceType,
} from "@budibase/types"
@ -110,7 +110,7 @@ describe("external search", () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!
const searchParams: SearchParams = {
const searchParams: RowSearchParams = {
tableId,
query: {},
}
@ -127,7 +127,7 @@ describe("external search", () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!
const searchParams: SearchParams = {
const searchParams: RowSearchParams = {
tableId,
query: {},
fields: ["name", "age"],
@ -151,7 +151,7 @@ describe("external search", () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!
const searchParams: SearchParams = {
const searchParams: RowSearchParams = {
tableId,
query: {
oneOf: {

View File

@ -2,7 +2,7 @@ import {
FieldType,
Row,
Table,
SearchParams,
RowSearchParams,
INTERNAL_TABLE_SOURCE_ID,
TableSourceType,
} from "@budibase/types"
@ -77,7 +77,7 @@ describe("internal", () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!
const searchParams: SearchParams = {
const searchParams: RowSearchParams = {
tableId,
query: {},
}
@ -94,7 +94,7 @@ describe("internal", () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!
const searchParams: SearchParams = {
const searchParams: RowSearchParams = {
tableId,
query: {},
fields: ["name", "age"],

View File

@ -4,7 +4,7 @@ import {
FieldType,
FieldTypeSubtypes,
INTERNAL_TABLE_SOURCE_ID,
SearchParams,
RowSearchParams,
Table,
TableSourceType,
} from "@budibase/types"
@ -47,7 +47,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])(
const userMedataId = dbCore.generateUserMetadataID(globalUserId)
it("should be able to map ro_ to global user IDs", () => {
const params: SearchParams = {
const params: RowSearchParams = {
tableId,
query: {
equal: {
@ -60,7 +60,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])(
})
it("should handle array of user IDs", () => {
const params: SearchParams = {
const params: RowSearchParams = {
tableId,
query: {
oneOf: {
@ -77,7 +77,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])(
it("shouldn't change any other input", () => {
const email = "test@example.com"
const params: SearchParams = {
const params: RowSearchParams = {
tableId,
query: {
equal: {

View File

@ -1,17 +1,37 @@
import {
FieldType,
SearchParams,
Table,
DocumentType,
SEPARATOR,
FieldSubtype,
SearchFilters,
SearchIndex,
SearchResponse,
Row,
RowSearchParams,
} from "@budibase/types"
import { db as dbCore } from "@budibase/backend-core"
import { db as dbCore, context } from "@budibase/backend-core"
import { utils } from "@budibase/shared-core"
export async function paginatedSearch(
query: SearchFilters,
params: RowSearchParams
): Promise<SearchResponse<Row>> {
const appId = context.getAppId()
return dbCore.paginatedSearch(appId!, SearchIndex.ROWS, query, params)
}
export async function fullSearch(
query: SearchFilters,
params: RowSearchParams
): Promise<{ rows: Row[] }> {
const appId = context.getAppId()
return dbCore.fullSearch(appId!, SearchIndex.ROWS, query, params)
}
function findColumnInQueries(
column: string,
options: SearchParams,
options: RowSearchParams,
callback: (filter: any) => any
) {
if (!options.query) {
@ -29,7 +49,7 @@ function findColumnInQueries(
}
}
function userColumnMapping(column: string, options: SearchParams) {
function userColumnMapping(column: string, options: RowSearchParams) {
findColumnInQueries(column, options, (filterValue: any): any => {
const isArray = Array.isArray(filterValue),
isString = typeof filterValue === "string"
@ -60,7 +80,7 @@ function userColumnMapping(column: string, options: SearchParams) {
// maps through the search parameters to check if any of the inputs are invalid
// based on the table schema, converts them to something that is valid.
export function searchInputMapping(table: Table, options: SearchParams) {
export function searchInputMapping(table: Table, options: RowSearchParams) {
if (!table?.schema) {
return options
}

View File

@ -58,7 +58,7 @@ import {
RelationshipType,
Row,
Screen,
SearchParams,
RowSearchParams,
SourceName,
Table,
TableSourceType,
@ -733,7 +733,7 @@ export default class TestConfiguration {
return this.api.row.fetch(tableId)
}
async searchRows(tableId: string, searchParams?: SearchParams) {
async searchRows(tableId: string, searchParams?: RowSearchParams) {
if (!tableId && this.table) {
tableId = this.table._id!
}

View File

@ -7,7 +7,7 @@ import {
BulkImportRequest,
BulkImportResponse,
SearchRowResponse,
SearchParams,
RowSearchParams,
DeleteRows,
DeleteRow,
} from "@budibase/types"
@ -135,7 +135,7 @@ export class RowAPI extends TestAPI {
search = async (
sourceId: string,
params?: SearchParams,
params?: RowSearchParams,
expectations?: Expectations
): Promise<SearchRowResponse> => {
return await this._post<SearchRowResponse>(`/api/${sourceId}/search`, {

View File

@ -1,4 +1,4 @@
import { SearchFilters, SearchParams } from "../../../sdk"
import { SearchFilters, RowSearchParams } from "../../../sdk"
import { Row } from "../../../documents"
import { PaginationResponse, SortOrder } from "../../../api"
import { ReadStream } from "fs"
@ -13,7 +13,7 @@ export interface PatchRowRequest extends Row {
export interface PatchRowResponse extends Row {}
export interface SearchRowRequest extends Omit<SearchParams, "tableId"> {}
export interface SearchRowRequest extends Omit<RowSearchParams, "tableId"> {}
export interface SearchViewRowRequest
extends Pick<

View File

@ -22,6 +22,6 @@ export interface PaginationRequest extends BasicPaginationRequest {
}
export interface PaginationResponse {
bookmark: string | undefined
hasNextPage: boolean
bookmark: string | number | undefined
hasNextPage?: boolean
}

View File

@ -1,12 +1,13 @@
import { SortOrder, SortType } from "../api"
import { SearchFilters } from "./search"
import { Row } from "../documents"
import { WithRequired } from "../shared"
export interface SearchParams {
tableId: string
tableId?: string
query?: SearchFilters
paginate?: boolean
query: SearchFilters
bookmark?: string
bookmark?: string | number
limit?: number
sort?: string
sortOrder?: SortOrder
@ -14,10 +15,17 @@ export interface SearchParams {
version?: string
disableEscaping?: boolean
fields?: string[]
indexer?: () => Promise<any>
rows?: Row[]
}
export interface SearchResponse {
rows: Row[]
// when searching for rows we want a more extensive search type that requires certain properties
export interface RowSearchParams
extends WithRequired<SearchParams, "tableId" | "query"> {}
export interface SearchResponse<T> {
rows: T[]
hasNextPage?: boolean
bookmark?: string | number
totalRows?: number
}