diff --git a/packages/backend-core/src/db/index.ts b/packages/backend-core/src/db/index.ts index c47d1793dc..9c2ebf49a9 100644 --- a/packages/backend-core/src/db/index.ts +++ b/packages/backend-core/src/db/index.ts @@ -1,12 +1,14 @@ +import { dataFilters } from "@budibase/shared-core" + export * from "./couch" export * from "./db" export * from "./utils" export * from "./views" export * from "../docIds/conversions" export { default as Replication } from "./Replication" +export const removeKeyNumbering = dataFilters.removeKeyNumbering // exports to support old export structure export * from "../constants/db" export { getGlobalDBName, baseGlobalDBName } from "../context" -export * from "./lucene" export * as searchIndexes from "./searchIndexes" export * from "./errors" diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts deleted file mode 100644 index 9d897553dd..0000000000 --- a/packages/backend-core/src/db/lucene.ts +++ /dev/null @@ -1,721 +0,0 @@ -import fetch from "node-fetch" -import { getCouchInfo } from "./couch" -import { - SearchFilters, - Row, - EmptyFilterOption, - SearchResponse, - SearchParams, - WithRequired, -} from "@budibase/types" -import { dataFilters } from "@budibase/shared-core" - -export const removeKeyNumbering = dataFilters.removeKeyNumbering - -function isEmpty(value: any) { - return value == null || value === "" -} - -/** - * Class to build lucene query URLs. - * Optionally takes a base lucene query object. - */ -export class QueryBuilder { - #dbName: string - #index: string - #query: SearchFilters - #limit: number - #sort?: string - #bookmark?: string | number - #sortOrder: string - #sortType: string - #includeDocs: boolean - #version?: string - #indexBuilder?: () => Promise - #noEscaping = false - #skip?: number - - static readonly maxLimit = 200 - - constructor(dbName: string, index: string, base?: SearchFilters) { - this.#dbName = dbName - this.#index = index - this.#query = { - allOr: false, - onEmptyFilter: EmptyFilterOption.RETURN_ALL, - string: {}, - fuzzy: {}, - range: {}, - equal: {}, - notEqual: {}, - empty: {}, - notEmpty: {}, - oneOf: {}, - contains: {}, - notContains: {}, - containsAny: {}, - ...base, - } - this.#limit = 50 - this.#sortOrder = "ascending" - this.#sortType = "string" - this.#includeDocs = true - } - - disableEscaping() { - this.#noEscaping = true - return this - } - - setIndexBuilder(builderFn: () => Promise) { - this.#indexBuilder = builderFn - return this - } - - setVersion(version?: string) { - if (version != null) { - this.#version = version - } - return this - } - - setTable(tableId: string) { - this.#query.equal!.tableId = tableId - return this - } - - setLimit(limit?: number) { - if (limit != null) { - this.#limit = limit - } - return this - } - - setSort(sort?: string) { - if (sort != null) { - this.#sort = sort - } - return this - } - - setSortOrder(sortOrder?: string) { - if (sortOrder != null) { - this.#sortOrder = sortOrder - } - return this - } - - setSortType(sortType?: string) { - if (sortType != null) { - this.#sortType = sortType - } - return this - } - - setBookmark(bookmark?: string | number) { - if (bookmark != null) { - this.#bookmark = bookmark - } - return this - } - - setSkip(skip: number | undefined) { - this.#skip = skip - return this - } - - excludeDocs() { - this.#includeDocs = false - return this - } - - includeDocs() { - this.#includeDocs = true - return this - } - - addString(key: string, partial: string) { - this.#query.string![key] = partial - return this - } - - addFuzzy(key: string, fuzzy: string) { - this.#query.fuzzy![key] = fuzzy - return this - } - - addRange(key: string, low: string | number, high: string | number) { - this.#query.range![key] = { - low, - high, - } - return this - } - - addEqual(key: string, value: any) { - this.#query.equal![key] = value - return this - } - - addNotEqual(key: string, value: any) { - this.#query.notEqual![key] = value - return this - } - - addEmpty(key: string, value: any) { - this.#query.empty![key] = value - return this - } - - addNotEmpty(key: string, value: any) { - this.#query.notEmpty![key] = value - return this - } - - addOneOf(key: string, value: any) { - this.#query.oneOf![key] = value - return this - } - - addContains(key: string, value: any) { - this.#query.contains![key] = value - return this - } - - addNotContains(key: string, value: any) { - this.#query.notContains![key] = value - return this - } - - addContainsAny(key: string, value: any) { - this.#query.containsAny![key] = value - return this - } - - setAllOr() { - this.#query.allOr = true - } - - setOnEmptyFilter(value: EmptyFilterOption) { - this.#query.onEmptyFilter = value - } - - handleSpaces(input: string) { - if (this.#noEscaping) { - return input - } else { - return input.replace(/ /g, "_") - } - } - - preprocess( - value: any, - { - escape, - lowercase, - wrap, - type, - }: { - escape?: boolean - lowercase?: boolean - wrap?: boolean - type?: string - } = {} - ): string | any { - const hasVersion = !!this.#version - // Determine if type needs wrapped - const originalType = typeof value - // Convert to lowercase - if (value && lowercase) { - value = value.toLowerCase ? value.toLowerCase() : value - } - // Escape characters - if (!this.#noEscaping && escape && originalType === "string") { - value = `${value}`.replace(/[ /#+\-&|!(){}\]^"~*?:\\]/g, "\\$&") - } - - // Wrap in quotes - if (originalType === "string" && !isNaN(value) && !type) { - value = `"${value}"` - } else if (hasVersion && wrap) { - value = originalType === "number" ? value : `"${value}"` - } - return value - } - - isMultiCondition() { - let count = 0 - for (let filters of Object.values(this.#query)) { - // not contains is one massive filter in allOr mode - if (typeof filters === "object") { - count += Object.keys(filters).length - } - } - return count > 1 - } - - compressFilters(filters: Record) { - const compressed: typeof filters = {} - for (let key of Object.keys(filters)) { - const finalKey = removeKeyNumbering(key) - if (compressed[finalKey]) { - compressed[finalKey] = compressed[finalKey].concat(filters[key]) - } else { - compressed[finalKey] = filters[key] - } - } - // add prefixes back - const final: typeof filters = {} - let count = 1 - for (let [key, value] of Object.entries(compressed)) { - final[`${count++}:${key}`] = value - } - return final - } - - buildSearchQuery() { - const builder = this - let allOr = this.#query && this.#query.allOr - let query = allOr ? "" : "*:*" - let allFiltersEmpty = true - const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true } - let tableId = "" - if (this.#query.equal!.tableId) { - tableId = this.#query.equal!.tableId - delete this.#query.equal!.tableId - } - - const equal = (key: string, value: any) => { - if (isEmpty(value)) { - return null - } - return `${key}:${builder.preprocess(value, allPreProcessingOpts)}` - } - - const contains = (key: string, value: any, mode = "AND") => { - if (isEmpty(value)) { - return null - } - if (!Array.isArray(value)) { - return `${key}:${value}` - } - let statement = `${builder.preprocess(value[0], { escape: true })}` - for (let i = 1; i < value.length; i++) { - statement += ` ${mode} ${builder.preprocess(value[i], { - escape: true, - })}` - } - return `${key}:(${statement})` - } - - const fuzzy = (key: string, value: any) => { - if (isEmpty(value)) { - return null - } - value = builder.preprocess(value, { - escape: true, - lowercase: true, - type: "fuzzy", - }) - return `${key}:/.*${value}.*/` - } - - const notContains = (key: string, value: any) => { - const allPrefix = allOr ? "*:* AND " : "" - const mode = allOr ? "AND" : undefined - return allPrefix + "NOT " + contains(key, value, mode) - } - - const containsAny = (key: string, value: any) => { - return contains(key, value, "OR") - } - - const oneOf = (key: string, value: any) => { - if (isEmpty(value)) { - return `*:*` - } - if (!Array.isArray(value)) { - if (typeof value === "string") { - value = value.split(",") - } else { - return "" - } - } - let orStatement = `${builder.preprocess(value[0], allPreProcessingOpts)}` - for (let i = 1; i < value.length; i++) { - orStatement += ` OR ${builder.preprocess( - value[i], - allPreProcessingOpts - )}` - } - return `${key}:(${orStatement})` - } - - function build( - structure: any, - queryFn: (key: string, value: any) => string | null, - opts?: { returnBuilt?: boolean; mode?: string } - ) { - let built = "" - for (let [key, value] of Object.entries(structure)) { - // check for new format - remove numbering if needed - key = removeKeyNumbering(key) - key = builder.preprocess(builder.handleSpaces(key), { - escape: true, - }) - let expression = queryFn(key, value) - if (expression == null) { - continue - } - if (built.length > 0 || query.length > 0) { - const mode = opts?.mode ? opts.mode : allOr ? "OR" : "AND" - built += ` ${mode} ` - } - built += expression - if ( - (typeof value !== "string" && value != null) || - (typeof value === "string" && value !== tableId && value !== "") - ) { - allFiltersEmpty = false - } - } - if (opts?.returnBuilt) { - return built - } else { - query += built - } - } - - // Construct the actual lucene search query string from JSON structure - if (this.#query.string) { - build(this.#query.string, (key: string, value: any) => { - if (isEmpty(value)) { - return null - } - value = builder.preprocess(value, { - escape: true, - lowercase: true, - type: "string", - }) - return `${key}:${value}*` - }) - } - if (this.#query.range) { - build(this.#query.range, (key: string, value: any) => { - if (isEmpty(value)) { - return null - } - if (value.low == null || value.low === "") { - return null - } - if (value.high == null || value.high === "") { - return null - } - const low = builder.preprocess(value.low, allPreProcessingOpts) - const high = builder.preprocess(value.high, allPreProcessingOpts) - return `${key}:[${low} TO ${high}]` - }) - } - if (this.#query.fuzzy) { - build(this.#query.fuzzy, fuzzy) - } - if (this.#query.equal) { - build(this.#query.equal, equal) - } - if (this.#query.notEqual) { - build(this.#query.notEqual, (key: string, value: any) => { - if (isEmpty(value)) { - return null - } - if (typeof value === "boolean") { - return `(*:* AND !${key}:${value})` - } - return `!${key}:${builder.preprocess(value, allPreProcessingOpts)}` - }) - } - if (this.#query.empty) { - build(this.#query.empty, (key: string) => { - // Because the structure of an empty filter looks like this: - // { empty: { someKey: null } } - // - // The check inside of `build` does not set `allFiltersEmpty`, which results - // in weird behaviour when the empty filter is the only filter. We get around - // this by setting `allFiltersEmpty` to false here. - allFiltersEmpty = false - return `(*:* -${key}:["" TO *])` - }) - } - if (this.#query.notEmpty) { - build(this.#query.notEmpty, (key: string) => { - // Because the structure of a notEmpty filter looks like this: - // { notEmpty: { someKey: null } } - // - // The check inside of `build` does not set `allFiltersEmpty`, which results - // in weird behaviour when the empty filter is the only filter. We get around - // this by setting `allFiltersEmpty` to false here. - allFiltersEmpty = false - return `${key}:["" TO *]` - }) - } - if (this.#query.oneOf) { - build(this.#query.oneOf, oneOf) - } - if (this.#query.contains) { - build(this.#query.contains, contains) - } - if (this.#query.notContains) { - build(this.compressFilters(this.#query.notContains), notContains) - } - if (this.#query.containsAny) { - build(this.#query.containsAny, containsAny) - } - // make sure table ID is always added as an AND - if (tableId) { - query = this.isMultiCondition() ? `(${query})` : query - allOr = false - build({ tableId }, equal) - } - if (allFiltersEmpty) { - if (this.#query.onEmptyFilter === EmptyFilterOption.RETURN_NONE) { - return "" - } else if (this.#query?.allOr) { - return query.replace("()", "(*:*)") - } - } - return query - } - - buildSearchBody() { - let body: any = { - q: this.buildSearchQuery(), - limit: Math.min(this.#limit, QueryBuilder.maxLimit), - include_docs: this.#includeDocs, - } - if (this.#bookmark) { - body.bookmark = this.#bookmark - } - if (this.#sort) { - const order = this.#sortOrder === "descending" ? "-" : "" - const type = `<${this.#sortType}>` - body.sort = `${order}${this.handleSpaces(this.#sort)}${type}` - } - return body - } - - async run() { - if (this.#skip) { - await this.#skipItems(this.#skip) - } - return await this.#execute() - } - - /** - * Lucene queries do not support pagination and use bookmarks instead. - * For the given builder, walk through pages using bookmarks until the desired - * page has been met. - */ - async #skipItems(skip: number) { - // Lucene does not support pagination. - // Handle pagination by finding the right bookmark - const prevIncludeDocs = this.#includeDocs - const prevLimit = this.#limit - - this.excludeDocs() - let skipRemaining = skip - let iterationFetched = 0 - do { - const toSkip = Math.min(QueryBuilder.maxLimit, skipRemaining) - this.setLimit(toSkip) - const { bookmark, rows } = await this.#execute() - this.setBookmark(bookmark) - iterationFetched = rows.length - skipRemaining -= rows.length - } while (skipRemaining > 0 && iterationFetched > 0) - - this.#includeDocs = prevIncludeDocs - this.#limit = prevLimit - } - - async #execute() { - const { url, cookie } = getCouchInfo() - const fullPath = `${url}/${this.#dbName}/_design/database/_search/${ - this.#index - }` - const body = this.buildSearchBody() - try { - return await runQuery(fullPath, body, cookie) - } catch (err: any) { - if (err.status === 404 && this.#indexBuilder) { - await this.#indexBuilder() - return await runQuery(fullPath, body, cookie) - } else { - throw err - } - } - } -} - -/** - * Executes a lucene search query. - * @param url The query URL - * @param body The request body defining search criteria - * @param cookie The auth cookie for CouchDB - * @returns {Promise<{rows: []}>} - */ -async function runQuery( - url: string, - body: any, - cookie: string -): Promise, "totalRows">> { - const response = await fetch(url, { - body: JSON.stringify(body), - method: "POST", - headers: { - Authorization: cookie, - }, - }) - - if (response.status === 404) { - throw response - } - const json = await response.json() - - let output: WithRequired, "totalRows"> = { - rows: [], - totalRows: 0, - } - if (json.rows != null && json.rows.length > 0) { - output.rows = json.rows.map((row: any) => row.doc) - } - if (json.bookmark) { - output.bookmark = json.bookmark - } - if (json.total_rows) { - output.totalRows = json.total_rows - } - return output -} - -/** - * Gets round the fixed limit of 200 results from a query by fetching as many - * pages as required and concatenating the results. This recursively operates - * until enough results have been found. - * @param dbName Which database to run a lucene query on - * @param index Which search index to utilise - * @param query The JSON query structure - * @param params The search params including: - * tableId {string} The table ID to search - * sort {string} The sort column - * sortOrder {string} The sort order ("ascending" or "descending") - * sortType {string} Whether to treat sortable values as strings or - * numbers. ("string" or "number") - * limit {number} The number of results to fetch - * bookmark {string|null} Current bookmark in the recursive search - * rows {array|null} Current results in the recursive search - */ -async function recursiveSearch( - dbName: string, - index: string, - query: any, - params: SearchParams -): Promise { - const bookmark = params.bookmark - const rows = params.rows || [] - if (params.limit && rows.length >= params.limit) { - return rows - } - let pageSize = QueryBuilder.maxLimit - if (params.limit && rows.length > params.limit - QueryBuilder.maxLimit) { - pageSize = params.limit - rows.length - } - const queryBuilder = new QueryBuilder(dbName, index, query) - queryBuilder - .setVersion(params.version) - .setBookmark(bookmark) - .setLimit(pageSize) - .setSort(params.sort) - .setSortOrder(params.sortOrder) - .setSortType(params.sortType) - - if (params.tableId) { - queryBuilder.setTable(params.tableId) - } - - const page = await queryBuilder.run() - if (!page.rows.length) { - return rows - } - if (page.rows.length < QueryBuilder.maxLimit) { - return [...rows, ...page.rows] - } - const newParams: SearchParams = { - ...params, - bookmark: page.bookmark, - rows: [...rows, ...page.rows] as Row[], - } - return await recursiveSearch(dbName, index, query, newParams) -} - -export async function paginatedSearch( - dbName: string, - index: string, - query: SearchFilters, - params: SearchParams -): Promise> { - let limit = params.limit - if (limit == null || isNaN(limit) || limit < 0) { - limit = 50 - } - limit = Math.min(limit, QueryBuilder.maxLimit) - const search = new QueryBuilder(dbName, index, query) - if (params.version) { - search.setVersion(params.version) - } - if (params.tableId) { - search.setTable(params.tableId) - } - if (params.sort) { - search - .setSort(params.sort) - .setSortOrder(params.sortOrder) - .setSortType(params.sortType) - } - if (params.indexer) { - search.setIndexBuilder(params.indexer) - } - if (params.disableEscaping) { - search.disableEscaping() - } - const searchResults = await search - .setBookmark(params.bookmark) - .setLimit(limit) - .run() - - // Try fetching 1 row in the next page to see if another page of results - // exists or not - search.setBookmark(searchResults.bookmark).setLimit(1) - if (params.tableId) { - search.setTable(params.tableId) - } - const nextResults = await search.run() - - return { - ...searchResults, - hasNextPage: nextResults.rows && nextResults.rows.length > 0, - } -} - -export async function fullSearch( - dbName: string, - index: string, - query: SearchFilters, - params: SearchParams -): Promise<{ rows: Row[] }> { - let limit = params.limit - if (limit == null || isNaN(limit) || limit < 0) { - limit = 1000 - } - params.limit = Math.min(limit, 1000) - const rows = await recursiveSearch(dbName, index, query, params) - return { rows } -} diff --git a/packages/backend-core/src/db/tests/lucene.spec.ts b/packages/backend-core/src/db/tests/lucene.spec.ts deleted file mode 100644 index c41bdf88d1..0000000000 --- a/packages/backend-core/src/db/tests/lucene.spec.ts +++ /dev/null @@ -1,400 +0,0 @@ -import { newid } from "../../docIds/newid" -import { getDB } from "../db" -import { - Database, - EmptyFilterOption, - SortOrder, - SortType, - DocumentType, - SEPARATOR, -} from "@budibase/types" -import { fullSearch, paginatedSearch, QueryBuilder } from "../lucene" - -const INDEX_NAME = "main" -const TABLE_ID = DocumentType.TABLE + SEPARATOR + newid() - -const index = `function(doc) { - if (!doc._id.startsWith("ro_")) { - return - } - let keys = Object.keys(doc).filter(key => !key.startsWith("_")) - for (let key of keys) { - const value = doc[key] - if (Array.isArray(value)) { - for (let val of value) { - index(key, val) - } - } else if (value) { - index(key, value) - } - } -}` - -function rowId(id?: string) { - return DocumentType.ROW + SEPARATOR + (id || newid()) -} - -describe("lucene", () => { - let db: Database, dbName: string - - beforeAll(async () => { - dbName = `db-${newid()}` - // create the DB for testing - db = getDB(dbName) - await db.put({ - _id: rowId(), - tableId: TABLE_ID, - property: "word", - array: ["1", "4"], - }) - await db.put({ - _id: rowId(), - tableId: TABLE_ID, - property: "word2", - array: ["3", "1"], - }) - await db.put({ - _id: rowId(), - tableId: TABLE_ID, - property: "word3", - number: 1, - array: ["1", "2"], - }) - }) - - it("should be able to create a lucene index", async () => { - const response = await db.put({ - _id: "_design/database", - indexes: { - [INDEX_NAME]: { - index: index, - analyzer: "standard", - }, - }, - }) - expect(response.ok).toBe(true) - }) - - describe("query builder", () => { - it("should be able to perform a basic query", async () => { - const builder = new QueryBuilder(dbName, INDEX_NAME) - builder.setSort("property") - builder.setSortOrder("desc") - builder.setSortType("string") - const resp = await builder.run() - expect(resp.rows.length).toBe(3) - }) - - it("should handle limits", async () => { - const builder = new QueryBuilder(dbName, INDEX_NAME) - builder.setLimit(1) - const resp = await builder.run() - expect(resp.rows.length).toBe(1) - }) - - it("should be able to perform a string search", async () => { - const builder = new QueryBuilder(dbName, INDEX_NAME) - builder.addString("property", "wo") - const resp = await builder.run() - expect(resp.rows.length).toBe(3) - }) - - it("should be able to perform a range search", async () => { - const builder = new QueryBuilder(dbName, INDEX_NAME) - builder.addRange("number", 0, 1) - const resp = await builder.run() - expect(resp.rows.length).toBe(1) - }) - - it("should be able to perform an equal search", async () => { - const builder = new QueryBuilder(dbName, INDEX_NAME) - builder.addEqual("property", "word2") - const resp = await builder.run() - expect(resp.rows.length).toBe(1) - }) - - it("should be able to perform a not equal search", async () => { - const builder = new QueryBuilder(dbName, INDEX_NAME) - builder.addNotEqual("property", "word2") - const resp = await builder.run() - expect(resp.rows.length).toBe(2) - }) - - it("should be able to perform an empty search", async () => { - const builder = new QueryBuilder(dbName, INDEX_NAME) - builder.addEmpty("number", true) - const resp = await builder.run() - expect(resp.rows.length).toBe(2) - }) - - it("should be able to perform a not empty search", async () => { - const builder = new QueryBuilder(dbName, INDEX_NAME) - builder.addNotEmpty("number", true) - const resp = await builder.run() - expect(resp.rows.length).toBe(1) - }) - - it("should be able to perform a one of search", async () => { - const builder = new QueryBuilder(dbName, INDEX_NAME) - builder.addOneOf("property", ["word", "word2"]) - const resp = await builder.run() - expect(resp.rows.length).toBe(2) - }) - - it("should return all rows when doing a one of search against falsey value", async () => { - const builder = new QueryBuilder(dbName, INDEX_NAME) - builder.addOneOf("property", null) - let resp = await builder.run() - expect(resp.rows.length).toBe(3) - - builder.addOneOf("property", undefined) - resp = await builder.run() - expect(resp.rows.length).toBe(3) - - builder.addOneOf("property", "") - resp = await builder.run() - expect(resp.rows.length).toBe(3) - - builder.addOneOf("property", []) - resp = await builder.run() - expect(resp.rows.length).toBe(0) - }) - - it("should be able to perform a contains search", async () => { - const builder = new QueryBuilder(dbName, INDEX_NAME) - builder.addContains("property", ["word"]) - const resp = await builder.run() - expect(resp.rows.length).toBe(1) - }) - - it("should be able to perform a not contains search", async () => { - const builder = new QueryBuilder(dbName, INDEX_NAME) - builder.addNotContains("property", ["word2"]) - const resp = await builder.run() - expect(resp.rows.length).toBe(2) - }) - - it("should be able to perform an or not contains search", async () => { - const builder = new QueryBuilder(dbName, INDEX_NAME) - builder.addNotContains("array", ["1"]) - builder.addNotContains("array", ["2"]) - builder.setAllOr() - const resp = await builder.run() - expect(resp.rows.length).toBe(2) - }) - - describe("empty filters behaviour", () => { - it("should return all rows by default", async () => { - const builder = new QueryBuilder(dbName, INDEX_NAME) - builder.addEqual("property", "") - builder.addEqual("number", null) - builder.addString("property", "") - builder.addFuzzy("property", "") - builder.addNotEqual("number", undefined) - builder.addOneOf("number", null) - builder.addContains("array", undefined) - builder.addNotContains("array", null) - builder.addContainsAny("array", null) - - const resp = await builder.run() - expect(resp.rows.length).toBe(3) - }) - - it("should return all rows when onEmptyFilter is ALL", async () => { - const builder = new QueryBuilder(dbName, INDEX_NAME) - builder.setOnEmptyFilter(EmptyFilterOption.RETURN_ALL) - builder.setAllOr() - builder.addEqual("property", "") - builder.addEqual("number", null) - builder.addString("property", "") - builder.addFuzzy("property", "") - builder.addNotEqual("number", undefined) - builder.addOneOf("number", null) - builder.addContains("array", undefined) - builder.addNotContains("array", null) - builder.addContainsAny("array", null) - - const resp = await builder.run() - expect(resp.rows.length).toBe(3) - }) - - it("should return no rows when onEmptyFilter is NONE", async () => { - const builder = new QueryBuilder(dbName, INDEX_NAME) - builder.setOnEmptyFilter(EmptyFilterOption.RETURN_NONE) - builder.addEqual("property", "") - builder.addEqual("number", null) - builder.addString("property", "") - builder.addFuzzy("property", "") - builder.addNotEqual("number", undefined) - builder.addOneOf("number", null) - builder.addContains("array", undefined) - builder.addNotContains("array", null) - builder.addContainsAny("array", null) - - const resp = await builder.run() - expect(resp.rows.length).toBe(0) - }) - - it("should return all matching rows when onEmptyFilter is NONE, but a filter value is provided", async () => { - const builder = new QueryBuilder(dbName, INDEX_NAME) - builder.setOnEmptyFilter(EmptyFilterOption.RETURN_NONE) - builder.addEqual("property", "") - builder.addEqual("number", 1) - builder.addString("property", "") - builder.addFuzzy("property", "") - builder.addNotEqual("number", undefined) - builder.addOneOf("number", null) - builder.addContains("array", undefined) - builder.addNotContains("array", null) - builder.addContainsAny("array", null) - - const resp = await builder.run() - expect(resp.rows.length).toBe(1) - }) - }) - - describe("skip", () => { - const skipDbName = `db-${newid()}` - let docs: { - _id: string - property: string - array: string[] - }[] - - beforeAll(async () => { - const db = getDB(skipDbName) - - docs = Array(QueryBuilder.maxLimit * 2.5) - .fill(0) - .map((_, i) => ({ - _id: rowId(i.toString().padStart(3, "0")), - tableId: TABLE_ID, - property: `value_${i.toString().padStart(3, "0")}`, - array: [], - })) - await db.bulkDocs(docs) - - await db.put({ - _id: "_design/database", - indexes: { - [INDEX_NAME]: { - index: index, - analyzer: "standard", - }, - }, - }) - }) - - it("should be able to apply skip", async () => { - const builder = new QueryBuilder(skipDbName, INDEX_NAME) - const firstResponse = await builder.run() - builder.setSkip(40) - const secondResponse = await builder.run() - - // Return the default limit - expect(firstResponse.rows.length).toBe(50) - expect(secondResponse.rows.length).toBe(50) - - // Should have the expected overlap - expect(firstResponse.rows.slice(40)).toEqual( - secondResponse.rows.slice(0, 10) - ) - }) - - it("should handle limits", async () => { - const builder = new QueryBuilder(skipDbName, INDEX_NAME) - builder.setLimit(10) - builder.setSkip(50) - builder.setSort("_id") - - const resp = await builder.run() - expect(resp.rows.length).toBe(10) - expect(resp.rows).toEqual( - docs.slice(50, 60).map(expect.objectContaining) - ) - }) - - it("should be able to skip searching through multiple responses", async () => { - const builder = new QueryBuilder(skipDbName, INDEX_NAME) - // Skipping 2 max limits plus a little bit more - const skip = QueryBuilder.maxLimit * 2 + 37 - builder.setSkip(skip) - builder.setSort("_id") - const resp = await builder.run() - - expect(resp.rows.length).toBe(50) - expect(resp.rows).toEqual( - docs.slice(skip, skip + resp.rows.length).map(expect.objectContaining) - ) - }) - - it("should not return results if skipping all docs", async () => { - const builder = new QueryBuilder(skipDbName, INDEX_NAME) - // Skipping 2 max limits plus a little bit more - const skip = docs.length + 1 - builder.setSkip(skip) - - const resp = await builder.run() - - expect(resp.rows.length).toBe(0) - }) - - it("skip should respect with filters", async () => { - const builder = new QueryBuilder(skipDbName, INDEX_NAME) - builder.setLimit(10) - builder.setSkip(50) - builder.addString("property", "value_1") - builder.setSort("property") - - const resp = await builder.run() - expect(resp.rows.length).toBe(10) - expect(resp.rows).toEqual( - docs.slice(150, 160).map(expect.objectContaining) - ) - }) - }) - }) - - describe("paginated search", () => { - it("should be able to perform a paginated search", async () => { - const page = await paginatedSearch( - dbName, - INDEX_NAME, - { - string: { - property: "wo", - }, - }, - { - tableId: TABLE_ID, - limit: 1, - sort: "property", - sortType: SortType.STRING, - sortOrder: SortOrder.DESCENDING, - } - ) - expect(page.rows.length).toBe(1) - expect(page.hasNextPage).toBe(true) - expect(page.bookmark).toBeDefined() - }) - }) - - describe("full search", () => { - it("should be able to perform a full search", async () => { - const page = await fullSearch( - dbName, - INDEX_NAME, - { - string: { - property: "wo", - }, - }, - { - tableId: TABLE_ID, - query: {}, - } - ) - expect(page.rows.length).toBe(3) - }) - }) -}) diff --git a/packages/server/src/sdk/app/rows/search/utils.ts b/packages/server/src/sdk/app/rows/search/utils.ts index c7de7eef37..fe75e493f9 100644 --- a/packages/server/src/sdk/app/rows/search/utils.ts +++ b/packages/server/src/sdk/app/rows/search/utils.ts @@ -5,30 +5,11 @@ import { SEPARATOR, BBReferenceFieldSubType, SearchFilters, - SearchIndex, - SearchResponse, - Row, RowSearchParams, } from "@budibase/types" -import { db as dbCore, context } from "@budibase/backend-core" +import { db as dbCore } from "@budibase/backend-core" import { utils, dataFilters } from "@budibase/shared-core" -export async function paginatedSearch( - query: SearchFilters, - params: RowSearchParams -): Promise> { - const appId = context.getAppId() - return dbCore.paginatedSearch(appId!, SearchIndex.ROWS, query, params) -} - -export async function fullSearch( - query: SearchFilters, - params: RowSearchParams -): Promise<{ rows: Row[] }> { - const appId = context.getAppId() - return dbCore.fullSearch(appId!, SearchIndex.ROWS, query, params) -} - function findColumnInQueries( column: string, filters: SearchFilters,