Initial work to remove Clouseau usage - still need to remove the SCIM index then ready to go.
This commit is contained in:
mike12345567
parent
48dafb42bc
commit
6c64e57531
|
|
@ -1,12 +1,14 @@
|
|||
import { dataFilters } from "@budibase/shared-core"
|
||||
|
||||
export * from "./couch"
|
||||
export * from "./db"
|
||||
export * from "./utils"
|
||||
export * from "./views"
|
||||
export * from "../docIds/conversions"
|
||||
export { default as Replication } from "./Replication"
|
||||
export const removeKeyNumbering = dataFilters.removeKeyNumbering
|
||||
// exports to support old export structure
|
||||
export * from "../constants/db"
|
||||
export { getGlobalDBName, baseGlobalDBName } from "../context"
|
||||
export * from "./lucene"
|
||||
export * as searchIndexes from "./searchIndexes"
|
||||
export * from "./errors"
|
||||
|
|
|
|||
|
|
@ -1,721 +0,0 @@
|
|||
import fetch from "node-fetch"
|
||||
import { getCouchInfo } from "./couch"
|
||||
import {
|
||||
SearchFilters,
|
||||
Row,
|
||||
EmptyFilterOption,
|
||||
SearchResponse,
|
||||
SearchParams,
|
||||
WithRequired,
|
||||
} from "@budibase/types"
|
||||
import { dataFilters } from "@budibase/shared-core"
|
||||
|
||||
export const removeKeyNumbering = dataFilters.removeKeyNumbering
|
||||
|
||||
function isEmpty(value: any) {
|
||||
return value == null || value === ""
|
||||
}
|
||||
|
||||
/**
|
||||
* Class to build lucene query URLs.
|
||||
* Optionally takes a base lucene query object.
|
||||
*/
|
||||
export class QueryBuilder<T> {
|
||||
#dbName: string
|
||||
#index: string
|
||||
#query: SearchFilters
|
||||
#limit: number
|
||||
#sort?: string
|
||||
#bookmark?: string | number
|
||||
#sortOrder: string
|
||||
#sortType: string
|
||||
#includeDocs: boolean
|
||||
#version?: string
|
||||
#indexBuilder?: () => Promise<any>
|
||||
#noEscaping = false
|
||||
#skip?: number
|
||||
|
||||
static readonly maxLimit = 200
|
||||
|
||||
constructor(dbName: string, index: string, base?: SearchFilters) {
|
||||
this.#dbName = dbName
|
||||
this.#index = index
|
||||
this.#query = {
|
||||
allOr: false,
|
||||
onEmptyFilter: EmptyFilterOption.RETURN_ALL,
|
||||
string: {},
|
||||
fuzzy: {},
|
||||
range: {},
|
||||
equal: {},
|
||||
notEqual: {},
|
||||
empty: {},
|
||||
notEmpty: {},
|
||||
oneOf: {},
|
||||
contains: {},
|
||||
notContains: {},
|
||||
containsAny: {},
|
||||
...base,
|
||||
}
|
||||
this.#limit = 50
|
||||
this.#sortOrder = "ascending"
|
||||
this.#sortType = "string"
|
||||
this.#includeDocs = true
|
||||
}
|
||||
|
||||
disableEscaping() {
|
||||
this.#noEscaping = true
|
||||
return this
|
||||
}
|
||||
|
||||
setIndexBuilder(builderFn: () => Promise<any>) {
|
||||
this.#indexBuilder = builderFn
|
||||
return this
|
||||
}
|
||||
|
||||
setVersion(version?: string) {
|
||||
if (version != null) {
|
||||
this.#version = version
|
||||
}
|
||||
return this
|
||||
}
|
||||
|
||||
setTable(tableId: string) {
|
||||
this.#query.equal!.tableId = tableId
|
||||
return this
|
||||
}
|
||||
|
||||
setLimit(limit?: number) {
|
||||
if (limit != null) {
|
||||
this.#limit = limit
|
||||
}
|
||||
return this
|
||||
}
|
||||
|
||||
setSort(sort?: string) {
|
||||
if (sort != null) {
|
||||
this.#sort = sort
|
||||
}
|
||||
return this
|
||||
}
|
||||
|
||||
setSortOrder(sortOrder?: string) {
|
||||
if (sortOrder != null) {
|
||||
this.#sortOrder = sortOrder
|
||||
}
|
||||
return this
|
||||
}
|
||||
|
||||
setSortType(sortType?: string) {
|
||||
if (sortType != null) {
|
||||
this.#sortType = sortType
|
||||
}
|
||||
return this
|
||||
}
|
||||
|
||||
setBookmark(bookmark?: string | number) {
|
||||
if (bookmark != null) {
|
||||
this.#bookmark = bookmark
|
||||
}
|
||||
return this
|
||||
}
|
||||
|
||||
setSkip(skip: number | undefined) {
|
||||
this.#skip = skip
|
||||
return this
|
||||
}
|
||||
|
||||
excludeDocs() {
|
||||
this.#includeDocs = false
|
||||
return this
|
||||
}
|
||||
|
||||
includeDocs() {
|
||||
this.#includeDocs = true
|
||||
return this
|
||||
}
|
||||
|
||||
addString(key: string, partial: string) {
|
||||
this.#query.string![key] = partial
|
||||
return this
|
||||
}
|
||||
|
||||
addFuzzy(key: string, fuzzy: string) {
|
||||
this.#query.fuzzy![key] = fuzzy
|
||||
return this
|
||||
}
|
||||
|
||||
addRange(key: string, low: string | number, high: string | number) {
|
||||
this.#query.range![key] = {
|
||||
low,
|
||||
high,
|
||||
}
|
||||
return this
|
||||
}
|
||||
|
||||
addEqual(key: string, value: any) {
|
||||
this.#query.equal![key] = value
|
||||
return this
|
||||
}
|
||||
|
||||
addNotEqual(key: string, value: any) {
|
||||
this.#query.notEqual![key] = value
|
||||
return this
|
||||
}
|
||||
|
||||
addEmpty(key: string, value: any) {
|
||||
this.#query.empty![key] = value
|
||||
return this
|
||||
}
|
||||
|
||||
addNotEmpty(key: string, value: any) {
|
||||
this.#query.notEmpty![key] = value
|
||||
return this
|
||||
}
|
||||
|
||||
addOneOf(key: string, value: any) {
|
||||
this.#query.oneOf![key] = value
|
||||
return this
|
||||
}
|
||||
|
||||
addContains(key: string, value: any) {
|
||||
this.#query.contains![key] = value
|
||||
return this
|
||||
}
|
||||
|
||||
addNotContains(key: string, value: any) {
|
||||
this.#query.notContains![key] = value
|
||||
return this
|
||||
}
|
||||
|
||||
addContainsAny(key: string, value: any) {
|
||||
this.#query.containsAny![key] = value
|
||||
return this
|
||||
}
|
||||
|
||||
setAllOr() {
|
||||
this.#query.allOr = true
|
||||
}
|
||||
|
||||
setOnEmptyFilter(value: EmptyFilterOption) {
|
||||
this.#query.onEmptyFilter = value
|
||||
}
|
||||
|
||||
handleSpaces(input: string) {
|
||||
if (this.#noEscaping) {
|
||||
return input
|
||||
} else {
|
||||
return input.replace(/ /g, "_")
|
||||
}
|
||||
}
|
||||
|
||||
preprocess(
|
||||
value: any,
|
||||
{
|
||||
escape,
|
||||
lowercase,
|
||||
wrap,
|
||||
type,
|
||||
}: {
|
||||
escape?: boolean
|
||||
lowercase?: boolean
|
||||
wrap?: boolean
|
||||
type?: string
|
||||
} = {}
|
||||
): string | any {
|
||||
const hasVersion = !!this.#version
|
||||
// Determine if type needs wrapped
|
||||
const originalType = typeof value
|
||||
// Convert to lowercase
|
||||
if (value && lowercase) {
|
||||
value = value.toLowerCase ? value.toLowerCase() : value
|
||||
}
|
||||
// Escape characters
|
||||
if (!this.#noEscaping && escape && originalType === "string") {
|
||||
value = `${value}`.replace(/[ /#+\-&|!(){}\]^"~*?:\\]/g, "\\$&")
|
||||
}
|
||||
|
||||
// Wrap in quotes
|
||||
if (originalType === "string" && !isNaN(value) && !type) {
|
||||
value = `"${value}"`
|
||||
} else if (hasVersion && wrap) {
|
||||
value = originalType === "number" ? value : `"${value}"`
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
isMultiCondition() {
|
||||
let count = 0
|
||||
for (let filters of Object.values(this.#query)) {
|
||||
// not contains is one massive filter in allOr mode
|
||||
if (typeof filters === "object") {
|
||||
count += Object.keys(filters).length
|
||||
}
|
||||
}
|
||||
return count > 1
|
||||
}
|
||||
|
||||
compressFilters(filters: Record<string, string[]>) {
|
||||
const compressed: typeof filters = {}
|
||||
for (let key of Object.keys(filters)) {
|
||||
const finalKey = removeKeyNumbering(key)
|
||||
if (compressed[finalKey]) {
|
||||
compressed[finalKey] = compressed[finalKey].concat(filters[key])
|
||||
} else {
|
||||
compressed[finalKey] = filters[key]
|
||||
}
|
||||
}
|
||||
// add prefixes back
|
||||
const final: typeof filters = {}
|
||||
let count = 1
|
||||
for (let [key, value] of Object.entries(compressed)) {
|
||||
final[`${count++}:${key}`] = value
|
||||
}
|
||||
return final
|
||||
}
|
||||
|
||||
buildSearchQuery() {
|
||||
const builder = this
|
||||
let allOr = this.#query && this.#query.allOr
|
||||
let query = allOr ? "" : "*:*"
|
||||
let allFiltersEmpty = true
|
||||
const allPreProcessingOpts = { escape: true, lowercase: true, wrap: true }
|
||||
let tableId = ""
|
||||
if (this.#query.equal!.tableId) {
|
||||
tableId = this.#query.equal!.tableId
|
||||
delete this.#query.equal!.tableId
|
||||
}
|
||||
|
||||
const equal = (key: string, value: any) => {
|
||||
if (isEmpty(value)) {
|
||||
return null
|
||||
}
|
||||
return `${key}:${builder.preprocess(value, allPreProcessingOpts)}`
|
||||
}
|
||||
|
||||
const contains = (key: string, value: any, mode = "AND") => {
|
||||
if (isEmpty(value)) {
|
||||
return null
|
||||
}
|
||||
if (!Array.isArray(value)) {
|
||||
return `${key}:${value}`
|
||||
}
|
||||
let statement = `${builder.preprocess(value[0], { escape: true })}`
|
||||
for (let i = 1; i < value.length; i++) {
|
||||
statement += ` ${mode} ${builder.preprocess(value[i], {
|
||||
escape: true,
|
||||
})}`
|
||||
}
|
||||
return `${key}:(${statement})`
|
||||
}
|
||||
|
||||
const fuzzy = (key: string, value: any) => {
|
||||
if (isEmpty(value)) {
|
||||
return null
|
||||
}
|
||||
value = builder.preprocess(value, {
|
||||
escape: true,
|
||||
lowercase: true,
|
||||
type: "fuzzy",
|
||||
})
|
||||
return `${key}:/.*${value}.*/`
|
||||
}
|
||||
|
||||
const notContains = (key: string, value: any) => {
|
||||
const allPrefix = allOr ? "*:* AND " : ""
|
||||
const mode = allOr ? "AND" : undefined
|
||||
return allPrefix + "NOT " + contains(key, value, mode)
|
||||
}
|
||||
|
||||
const containsAny = (key: string, value: any) => {
|
||||
return contains(key, value, "OR")
|
||||
}
|
||||
|
||||
const oneOf = (key: string, value: any) => {
|
||||
if (isEmpty(value)) {
|
||||
return `*:*`
|
||||
}
|
||||
if (!Array.isArray(value)) {
|
||||
if (typeof value === "string") {
|
||||
value = value.split(",")
|
||||
} else {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
let orStatement = `${builder.preprocess(value[0], allPreProcessingOpts)}`
|
||||
for (let i = 1; i < value.length; i++) {
|
||||
orStatement += ` OR ${builder.preprocess(
|
||||
value[i],
|
||||
allPreProcessingOpts
|
||||
)}`
|
||||
}
|
||||
return `${key}:(${orStatement})`
|
||||
}
|
||||
|
||||
function build(
|
||||
structure: any,
|
||||
queryFn: (key: string, value: any) => string | null,
|
||||
opts?: { returnBuilt?: boolean; mode?: string }
|
||||
) {
|
||||
let built = ""
|
||||
for (let [key, value] of Object.entries(structure)) {
|
||||
// check for new format - remove numbering if needed
|
||||
key = removeKeyNumbering(key)
|
||||
key = builder.preprocess(builder.handleSpaces(key), {
|
||||
escape: true,
|
||||
})
|
||||
let expression = queryFn(key, value)
|
||||
if (expression == null) {
|
||||
continue
|
||||
}
|
||||
if (built.length > 0 || query.length > 0) {
|
||||
const mode = opts?.mode ? opts.mode : allOr ? "OR" : "AND"
|
||||
built += ` ${mode} `
|
||||
}
|
||||
built += expression
|
||||
if (
|
||||
(typeof value !== "string" && value != null) ||
|
||||
(typeof value === "string" && value !== tableId && value !== "")
|
||||
) {
|
||||
allFiltersEmpty = false
|
||||
}
|
||||
}
|
||||
if (opts?.returnBuilt) {
|
||||
return built
|
||||
} else {
|
||||
query += built
|
||||
}
|
||||
}
|
||||
|
||||
// Construct the actual lucene search query string from JSON structure
|
||||
if (this.#query.string) {
|
||||
build(this.#query.string, (key: string, value: any) => {
|
||||
if (isEmpty(value)) {
|
||||
return null
|
||||
}
|
||||
value = builder.preprocess(value, {
|
||||
escape: true,
|
||||
lowercase: true,
|
||||
type: "string",
|
||||
})
|
||||
return `${key}:${value}*`
|
||||
})
|
||||
}
|
||||
if (this.#query.range) {
|
||||
build(this.#query.range, (key: string, value: any) => {
|
||||
if (isEmpty(value)) {
|
||||
return null
|
||||
}
|
||||
if (value.low == null || value.low === "") {
|
||||
return null
|
||||
}
|
||||
if (value.high == null || value.high === "") {
|
||||
return null
|
||||
}
|
||||
const low = builder.preprocess(value.low, allPreProcessingOpts)
|
||||
const high = builder.preprocess(value.high, allPreProcessingOpts)
|
||||
return `${key}:[${low} TO ${high}]`
|
||||
})
|
||||
}
|
||||
if (this.#query.fuzzy) {
|
||||
build(this.#query.fuzzy, fuzzy)
|
||||
}
|
||||
if (this.#query.equal) {
|
||||
build(this.#query.equal, equal)
|
||||
}
|
||||
if (this.#query.notEqual) {
|
||||
build(this.#query.notEqual, (key: string, value: any) => {
|
||||
if (isEmpty(value)) {
|
||||
return null
|
||||
}
|
||||
if (typeof value === "boolean") {
|
||||
return `(*:* AND !${key}:${value})`
|
||||
}
|
||||
return `!${key}:${builder.preprocess(value, allPreProcessingOpts)}`
|
||||
})
|
||||
}
|
||||
if (this.#query.empty) {
|
||||
build(this.#query.empty, (key: string) => {
|
||||
// Because the structure of an empty filter looks like this:
|
||||
// { empty: { someKey: null } }
|
||||
//
|
||||
// The check inside of `build` does not set `allFiltersEmpty`, which results
|
||||
// in weird behaviour when the empty filter is the only filter. We get around
|
||||
// this by setting `allFiltersEmpty` to false here.
|
||||
allFiltersEmpty = false
|
||||
return `(*:* -${key}:["" TO *])`
|
||||
})
|
||||
}
|
||||
if (this.#query.notEmpty) {
|
||||
build(this.#query.notEmpty, (key: string) => {
|
||||
// Because the structure of a notEmpty filter looks like this:
|
||||
// { notEmpty: { someKey: null } }
|
||||
//
|
||||
// The check inside of `build` does not set `allFiltersEmpty`, which results
|
||||
// in weird behaviour when the empty filter is the only filter. We get around
|
||||
// this by setting `allFiltersEmpty` to false here.
|
||||
allFiltersEmpty = false
|
||||
return `${key}:["" TO *]`
|
||||
})
|
||||
}
|
||||
if (this.#query.oneOf) {
|
||||
build(this.#query.oneOf, oneOf)
|
||||
}
|
||||
if (this.#query.contains) {
|
||||
build(this.#query.contains, contains)
|
||||
}
|
||||
if (this.#query.notContains) {
|
||||
build(this.compressFilters(this.#query.notContains), notContains)
|
||||
}
|
||||
if (this.#query.containsAny) {
|
||||
build(this.#query.containsAny, containsAny)
|
||||
}
|
||||
// make sure table ID is always added as an AND
|
||||
if (tableId) {
|
||||
query = this.isMultiCondition() ? `(${query})` : query
|
||||
allOr = false
|
||||
build({ tableId }, equal)
|
||||
}
|
||||
if (allFiltersEmpty) {
|
||||
if (this.#query.onEmptyFilter === EmptyFilterOption.RETURN_NONE) {
|
||||
return ""
|
||||
} else if (this.#query?.allOr) {
|
||||
return query.replace("()", "(*:*)")
|
||||
}
|
||||
}
|
||||
return query
|
||||
}
|
||||
|
||||
buildSearchBody() {
|
||||
let body: any = {
|
||||
q: this.buildSearchQuery(),
|
||||
limit: Math.min(this.#limit, QueryBuilder.maxLimit),
|
||||
include_docs: this.#includeDocs,
|
||||
}
|
||||
if (this.#bookmark) {
|
||||
body.bookmark = this.#bookmark
|
||||
}
|
||||
if (this.#sort) {
|
||||
const order = this.#sortOrder === "descending" ? "-" : ""
|
||||
const type = `<${this.#sortType}>`
|
||||
body.sort = `${order}${this.handleSpaces(this.#sort)}${type}`
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
async run() {
|
||||
if (this.#skip) {
|
||||
await this.#skipItems(this.#skip)
|
||||
}
|
||||
return await this.#execute()
|
||||
}
|
||||
|
||||
/**
|
||||
* Lucene queries do not support pagination and use bookmarks instead.
|
||||
* For the given builder, walk through pages using bookmarks until the desired
|
||||
* page has been met.
|
||||
*/
|
||||
async #skipItems(skip: number) {
|
||||
// Lucene does not support pagination.
|
||||
// Handle pagination by finding the right bookmark
|
||||
const prevIncludeDocs = this.#includeDocs
|
||||
const prevLimit = this.#limit
|
||||
|
||||
this.excludeDocs()
|
||||
let skipRemaining = skip
|
||||
let iterationFetched = 0
|
||||
do {
|
||||
const toSkip = Math.min(QueryBuilder.maxLimit, skipRemaining)
|
||||
this.setLimit(toSkip)
|
||||
const { bookmark, rows } = await this.#execute()
|
||||
this.setBookmark(bookmark)
|
||||
iterationFetched = rows.length
|
||||
skipRemaining -= rows.length
|
||||
} while (skipRemaining > 0 && iterationFetched > 0)
|
||||
|
||||
this.#includeDocs = prevIncludeDocs
|
||||
this.#limit = prevLimit
|
||||
}
|
||||
|
||||
async #execute() {
|
||||
const { url, cookie } = getCouchInfo()
|
||||
const fullPath = `${url}/${this.#dbName}/_design/database/_search/${
|
||||
this.#index
|
||||
}`
|
||||
const body = this.buildSearchBody()
|
||||
try {
|
||||
return await runQuery<T>(fullPath, body, cookie)
|
||||
} catch (err: any) {
|
||||
if (err.status === 404 && this.#indexBuilder) {
|
||||
await this.#indexBuilder()
|
||||
return await runQuery<T>(fullPath, body, cookie)
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes a lucene search query.
|
||||
* @param url The query URL
|
||||
* @param body The request body defining search criteria
|
||||
* @param cookie The auth cookie for CouchDB
|
||||
* @returns {Promise<{rows: []}>}
|
||||
*/
|
||||
async function runQuery<T>(
|
||||
url: string,
|
||||
body: any,
|
||||
cookie: string
|
||||
): Promise<WithRequired<SearchResponse<T>, "totalRows">> {
|
||||
const response = await fetch(url, {
|
||||
body: JSON.stringify(body),
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: cookie,
|
||||
},
|
||||
})
|
||||
|
||||
if (response.status === 404) {
|
||||
throw response
|
||||
}
|
||||
const json = await response.json()
|
||||
|
||||
let output: WithRequired<SearchResponse<T>, "totalRows"> = {
|
||||
rows: [],
|
||||
totalRows: 0,
|
||||
}
|
||||
if (json.rows != null && json.rows.length > 0) {
|
||||
output.rows = json.rows.map((row: any) => row.doc)
|
||||
}
|
||||
if (json.bookmark) {
|
||||
output.bookmark = json.bookmark
|
||||
}
|
||||
if (json.total_rows) {
|
||||
output.totalRows = json.total_rows
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets round the fixed limit of 200 results from a query by fetching as many
|
||||
* pages as required and concatenating the results. This recursively operates
|
||||
* until enough results have been found.
|
||||
* @param dbName Which database to run a lucene query on
|
||||
* @param index Which search index to utilise
|
||||
* @param query The JSON query structure
|
||||
* @param params The search params including:
|
||||
* tableId {string} The table ID to search
|
||||
* sort {string} The sort column
|
||||
* sortOrder {string} The sort order ("ascending" or "descending")
|
||||
* sortType {string} Whether to treat sortable values as strings or
|
||||
* numbers. ("string" or "number")
|
||||
* limit {number} The number of results to fetch
|
||||
* bookmark {string|null} Current bookmark in the recursive search
|
||||
* rows {array|null} Current results in the recursive search
|
||||
*/
|
||||
async function recursiveSearch<T>(
|
||||
dbName: string,
|
||||
index: string,
|
||||
query: any,
|
||||
params: SearchParams
|
||||
): Promise<any> {
|
||||
const bookmark = params.bookmark
|
||||
const rows = params.rows || []
|
||||
if (params.limit && rows.length >= params.limit) {
|
||||
return rows
|
||||
}
|
||||
let pageSize = QueryBuilder.maxLimit
|
||||
if (params.limit && rows.length > params.limit - QueryBuilder.maxLimit) {
|
||||
pageSize = params.limit - rows.length
|
||||
}
|
||||
const queryBuilder = new QueryBuilder<T>(dbName, index, query)
|
||||
queryBuilder
|
||||
.setVersion(params.version)
|
||||
.setBookmark(bookmark)
|
||||
.setLimit(pageSize)
|
||||
.setSort(params.sort)
|
||||
.setSortOrder(params.sortOrder)
|
||||
.setSortType(params.sortType)
|
||||
|
||||
if (params.tableId) {
|
||||
queryBuilder.setTable(params.tableId)
|
||||
}
|
||||
|
||||
const page = await queryBuilder.run()
|
||||
if (!page.rows.length) {
|
||||
return rows
|
||||
}
|
||||
if (page.rows.length < QueryBuilder.maxLimit) {
|
||||
return [...rows, ...page.rows]
|
||||
}
|
||||
const newParams: SearchParams = {
|
||||
...params,
|
||||
bookmark: page.bookmark,
|
||||
rows: [...rows, ...page.rows] as Row[],
|
||||
}
|
||||
return await recursiveSearch(dbName, index, query, newParams)
|
||||
}
|
||||
|
||||
export async function paginatedSearch<T>(
|
||||
dbName: string,
|
||||
index: string,
|
||||
query: SearchFilters,
|
||||
params: SearchParams
|
||||
): Promise<SearchResponse<T>> {
|
||||
let limit = params.limit
|
||||
if (limit == null || isNaN(limit) || limit < 0) {
|
||||
limit = 50
|
||||
}
|
||||
limit = Math.min(limit, QueryBuilder.maxLimit)
|
||||
const search = new QueryBuilder<T>(dbName, index, query)
|
||||
if (params.version) {
|
||||
search.setVersion(params.version)
|
||||
}
|
||||
if (params.tableId) {
|
||||
search.setTable(params.tableId)
|
||||
}
|
||||
if (params.sort) {
|
||||
search
|
||||
.setSort(params.sort)
|
||||
.setSortOrder(params.sortOrder)
|
||||
.setSortType(params.sortType)
|
||||
}
|
||||
if (params.indexer) {
|
||||
search.setIndexBuilder(params.indexer)
|
||||
}
|
||||
if (params.disableEscaping) {
|
||||
search.disableEscaping()
|
||||
}
|
||||
const searchResults = await search
|
||||
.setBookmark(params.bookmark)
|
||||
.setLimit(limit)
|
||||
.run()
|
||||
|
||||
// Try fetching 1 row in the next page to see if another page of results
|
||||
// exists or not
|
||||
search.setBookmark(searchResults.bookmark).setLimit(1)
|
||||
if (params.tableId) {
|
||||
search.setTable(params.tableId)
|
||||
}
|
||||
const nextResults = await search.run()
|
||||
|
||||
return {
|
||||
...searchResults,
|
||||
hasNextPage: nextResults.rows && nextResults.rows.length > 0,
|
||||
}
|
||||
}
|
||||
|
||||
export async function fullSearch<T>(
|
||||
dbName: string,
|
||||
index: string,
|
||||
query: SearchFilters,
|
||||
params: SearchParams
|
||||
): Promise<{ rows: Row[] }> {
|
||||
let limit = params.limit
|
||||
if (limit == null || isNaN(limit) || limit < 0) {
|
||||
limit = 1000
|
||||
}
|
||||
params.limit = Math.min(limit, 1000)
|
||||
const rows = await recursiveSearch<T>(dbName, index, query, params)
|
||||
return { rows }
|
||||
}
|
||||
|
|
@ -1,400 +0,0 @@
|
|||
import { newid } from "../../docIds/newid"
|
||||
import { getDB } from "../db"
|
||||
import {
|
||||
Database,
|
||||
EmptyFilterOption,
|
||||
SortOrder,
|
||||
SortType,
|
||||
DocumentType,
|
||||
SEPARATOR,
|
||||
} from "@budibase/types"
|
||||
import { fullSearch, paginatedSearch, QueryBuilder } from "../lucene"
|
||||
|
||||
const INDEX_NAME = "main"
|
||||
const TABLE_ID = DocumentType.TABLE + SEPARATOR + newid()
|
||||
|
||||
const index = `function(doc) {
|
||||
if (!doc._id.startsWith("ro_")) {
|
||||
return
|
||||
}
|
||||
let keys = Object.keys(doc).filter(key => !key.startsWith("_"))
|
||||
for (let key of keys) {
|
||||
const value = doc[key]
|
||||
if (Array.isArray(value)) {
|
||||
for (let val of value) {
|
||||
index(key, val)
|
||||
}
|
||||
} else if (value) {
|
||||
index(key, value)
|
||||
}
|
||||
}
|
||||
}`
|
||||
|
||||
function rowId(id?: string) {
|
||||
return DocumentType.ROW + SEPARATOR + (id || newid())
|
||||
}
|
||||
|
||||
describe("lucene", () => {
|
||||
let db: Database, dbName: string
|
||||
|
||||
beforeAll(async () => {
|
||||
dbName = `db-${newid()}`
|
||||
// create the DB for testing
|
||||
db = getDB(dbName)
|
||||
await db.put({
|
||||
_id: rowId(),
|
||||
tableId: TABLE_ID,
|
||||
property: "word",
|
||||
array: ["1", "4"],
|
||||
})
|
||||
await db.put({
|
||||
_id: rowId(),
|
||||
tableId: TABLE_ID,
|
||||
property: "word2",
|
||||
array: ["3", "1"],
|
||||
})
|
||||
await db.put({
|
||||
_id: rowId(),
|
||||
tableId: TABLE_ID,
|
||||
property: "word3",
|
||||
number: 1,
|
||||
array: ["1", "2"],
|
||||
})
|
||||
})
|
||||
|
||||
it("should be able to create a lucene index", async () => {
|
||||
const response = await db.put({
|
||||
_id: "_design/database",
|
||||
indexes: {
|
||||
[INDEX_NAME]: {
|
||||
index: index,
|
||||
analyzer: "standard",
|
||||
},
|
||||
},
|
||||
})
|
||||
expect(response.ok).toBe(true)
|
||||
})
|
||||
|
||||
describe("query builder", () => {
|
||||
it("should be able to perform a basic query", async () => {
|
||||
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||
builder.setSort("property")
|
||||
builder.setSortOrder("desc")
|
||||
builder.setSortType("string")
|
||||
const resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(3)
|
||||
})
|
||||
|
||||
it("should handle limits", async () => {
|
||||
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||
builder.setLimit(1)
|
||||
const resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(1)
|
||||
})
|
||||
|
||||
it("should be able to perform a string search", async () => {
|
||||
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||
builder.addString("property", "wo")
|
||||
const resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(3)
|
||||
})
|
||||
|
||||
it("should be able to perform a range search", async () => {
|
||||
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||
builder.addRange("number", 0, 1)
|
||||
const resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(1)
|
||||
})
|
||||
|
||||
it("should be able to perform an equal search", async () => {
|
||||
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||
builder.addEqual("property", "word2")
|
||||
const resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(1)
|
||||
})
|
||||
|
||||
it("should be able to perform a not equal search", async () => {
|
||||
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||
builder.addNotEqual("property", "word2")
|
||||
const resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(2)
|
||||
})
|
||||
|
||||
it("should be able to perform an empty search", async () => {
|
||||
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||
builder.addEmpty("number", true)
|
||||
const resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(2)
|
||||
})
|
||||
|
||||
it("should be able to perform a not empty search", async () => {
|
||||
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||
builder.addNotEmpty("number", true)
|
||||
const resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(1)
|
||||
})
|
||||
|
||||
it("should be able to perform a one of search", async () => {
|
||||
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||
builder.addOneOf("property", ["word", "word2"])
|
||||
const resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(2)
|
||||
})
|
||||
|
||||
it("should return all rows when doing a one of search against falsey value", async () => {
|
||||
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||
builder.addOneOf("property", null)
|
||||
let resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(3)
|
||||
|
||||
builder.addOneOf("property", undefined)
|
||||
resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(3)
|
||||
|
||||
builder.addOneOf("property", "")
|
||||
resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(3)
|
||||
|
||||
builder.addOneOf("property", [])
|
||||
resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(0)
|
||||
})
|
||||
|
||||
it("should be able to perform a contains search", async () => {
|
||||
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||
builder.addContains("property", ["word"])
|
||||
const resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(1)
|
||||
})
|
||||
|
||||
it("should be able to perform a not contains search", async () => {
|
||||
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||
builder.addNotContains("property", ["word2"])
|
||||
const resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(2)
|
||||
})
|
||||
|
||||
it("should be able to perform an or not contains search", async () => {
|
||||
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||
builder.addNotContains("array", ["1"])
|
||||
builder.addNotContains("array", ["2"])
|
||||
builder.setAllOr()
|
||||
const resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(2)
|
||||
})
|
||||
|
||||
describe("empty filters behaviour", () => {
|
||||
it("should return all rows by default", async () => {
|
||||
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||
builder.addEqual("property", "")
|
||||
builder.addEqual("number", null)
|
||||
builder.addString("property", "")
|
||||
builder.addFuzzy("property", "")
|
||||
builder.addNotEqual("number", undefined)
|
||||
builder.addOneOf("number", null)
|
||||
builder.addContains("array", undefined)
|
||||
builder.addNotContains("array", null)
|
||||
builder.addContainsAny("array", null)
|
||||
|
||||
const resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(3)
|
||||
})
|
||||
|
||||
it("should return all rows when onEmptyFilter is ALL", async () => {
|
||||
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||
builder.setOnEmptyFilter(EmptyFilterOption.RETURN_ALL)
|
||||
builder.setAllOr()
|
||||
builder.addEqual("property", "")
|
||||
builder.addEqual("number", null)
|
||||
builder.addString("property", "")
|
||||
builder.addFuzzy("property", "")
|
||||
builder.addNotEqual("number", undefined)
|
||||
builder.addOneOf("number", null)
|
||||
builder.addContains("array", undefined)
|
||||
builder.addNotContains("array", null)
|
||||
builder.addContainsAny("array", null)
|
||||
|
||||
const resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(3)
|
||||
})
|
||||
|
||||
it("should return no rows when onEmptyFilter is NONE", async () => {
|
||||
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||
builder.setOnEmptyFilter(EmptyFilterOption.RETURN_NONE)
|
||||
builder.addEqual("property", "")
|
||||
builder.addEqual("number", null)
|
||||
builder.addString("property", "")
|
||||
builder.addFuzzy("property", "")
|
||||
builder.addNotEqual("number", undefined)
|
||||
builder.addOneOf("number", null)
|
||||
builder.addContains("array", undefined)
|
||||
builder.addNotContains("array", null)
|
||||
builder.addContainsAny("array", null)
|
||||
|
||||
const resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(0)
|
||||
})
|
||||
|
||||
it("should return all matching rows when onEmptyFilter is NONE, but a filter value is provided", async () => {
|
||||
const builder = new QueryBuilder(dbName, INDEX_NAME)
|
||||
builder.setOnEmptyFilter(EmptyFilterOption.RETURN_NONE)
|
||||
builder.addEqual("property", "")
|
||||
builder.addEqual("number", 1)
|
||||
builder.addString("property", "")
|
||||
builder.addFuzzy("property", "")
|
||||
builder.addNotEqual("number", undefined)
|
||||
builder.addOneOf("number", null)
|
||||
builder.addContains("array", undefined)
|
||||
builder.addNotContains("array", null)
|
||||
builder.addContainsAny("array", null)
|
||||
|
||||
const resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe("skip", () => {
|
||||
const skipDbName = `db-${newid()}`
|
||||
let docs: {
|
||||
_id: string
|
||||
property: string
|
||||
array: string[]
|
||||
}[]
|
||||
|
||||
beforeAll(async () => {
|
||||
const db = getDB(skipDbName)
|
||||
|
||||
docs = Array(QueryBuilder.maxLimit * 2.5)
|
||||
.fill(0)
|
||||
.map((_, i) => ({
|
||||
_id: rowId(i.toString().padStart(3, "0")),
|
||||
tableId: TABLE_ID,
|
||||
property: `value_${i.toString().padStart(3, "0")}`,
|
||||
array: [],
|
||||
}))
|
||||
await db.bulkDocs(docs)
|
||||
|
||||
await db.put({
|
||||
_id: "_design/database",
|
||||
indexes: {
|
||||
[INDEX_NAME]: {
|
||||
index: index,
|
||||
analyzer: "standard",
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it("should be able to apply skip", async () => {
|
||||
const builder = new QueryBuilder(skipDbName, INDEX_NAME)
|
||||
const firstResponse = await builder.run()
|
||||
builder.setSkip(40)
|
||||
const secondResponse = await builder.run()
|
||||
|
||||
// Return the default limit
|
||||
expect(firstResponse.rows.length).toBe(50)
|
||||
expect(secondResponse.rows.length).toBe(50)
|
||||
|
||||
// Should have the expected overlap
|
||||
expect(firstResponse.rows.slice(40)).toEqual(
|
||||
secondResponse.rows.slice(0, 10)
|
||||
)
|
||||
})
|
||||
|
||||
it("should handle limits", async () => {
|
||||
const builder = new QueryBuilder(skipDbName, INDEX_NAME)
|
||||
builder.setLimit(10)
|
||||
builder.setSkip(50)
|
||||
builder.setSort("_id")
|
||||
|
||||
const resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(10)
|
||||
expect(resp.rows).toEqual(
|
||||
docs.slice(50, 60).map(expect.objectContaining)
|
||||
)
|
||||
})
|
||||
|
||||
it("should be able to skip searching through multiple responses", async () => {
|
||||
const builder = new QueryBuilder(skipDbName, INDEX_NAME)
|
||||
// Skipping 2 max limits plus a little bit more
|
||||
const skip = QueryBuilder.maxLimit * 2 + 37
|
||||
builder.setSkip(skip)
|
||||
builder.setSort("_id")
|
||||
const resp = await builder.run()
|
||||
|
||||
expect(resp.rows.length).toBe(50)
|
||||
expect(resp.rows).toEqual(
|
||||
docs.slice(skip, skip + resp.rows.length).map(expect.objectContaining)
|
||||
)
|
||||
})
|
||||
|
||||
it("should not return results if skipping all docs", async () => {
|
||||
const builder = new QueryBuilder(skipDbName, INDEX_NAME)
|
||||
// Skipping 2 max limits plus a little bit more
|
||||
const skip = docs.length + 1
|
||||
builder.setSkip(skip)
|
||||
|
||||
const resp = await builder.run()
|
||||
|
||||
expect(resp.rows.length).toBe(0)
|
||||
})
|
||||
|
||||
it("skip should respect with filters", async () => {
|
||||
const builder = new QueryBuilder(skipDbName, INDEX_NAME)
|
||||
builder.setLimit(10)
|
||||
builder.setSkip(50)
|
||||
builder.addString("property", "value_1")
|
||||
builder.setSort("property")
|
||||
|
||||
const resp = await builder.run()
|
||||
expect(resp.rows.length).toBe(10)
|
||||
expect(resp.rows).toEqual(
|
||||
docs.slice(150, 160).map(expect.objectContaining)
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe("paginated search", () => {
|
||||
it("should be able to perform a paginated search", async () => {
|
||||
const page = await paginatedSearch(
|
||||
dbName,
|
||||
INDEX_NAME,
|
||||
{
|
||||
string: {
|
||||
property: "wo",
|
||||
},
|
||||
},
|
||||
{
|
||||
tableId: TABLE_ID,
|
||||
limit: 1,
|
||||
sort: "property",
|
||||
sortType: SortType.STRING,
|
||||
sortOrder: SortOrder.DESCENDING,
|
||||
}
|
||||
)
|
||||
expect(page.rows.length).toBe(1)
|
||||
expect(page.hasNextPage).toBe(true)
|
||||
expect(page.bookmark).toBeDefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe("full search", () => {
|
||||
it("should be able to perform a full search", async () => {
|
||||
const page = await fullSearch(
|
||||
dbName,
|
||||
INDEX_NAME,
|
||||
{
|
||||
string: {
|
||||
property: "wo",
|
||||
},
|
||||
},
|
||||
{
|
||||
tableId: TABLE_ID,
|
||||
query: {},
|
||||
}
|
||||
)
|
||||
expect(page.rows.length).toBe(3)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -5,30 +5,11 @@ import {
|
|||
SEPARATOR,
|
||||
BBReferenceFieldSubType,
|
||||
SearchFilters,
|
||||
SearchIndex,
|
||||
SearchResponse,
|
||||
Row,
|
||||
RowSearchParams,
|
||||
} from "@budibase/types"
|
||||
import { db as dbCore, context } from "@budibase/backend-core"
|
||||
import { db as dbCore } from "@budibase/backend-core"
|
||||
import { utils, dataFilters } from "@budibase/shared-core"
|
||||
|
||||
export async function paginatedSearch(
|
||||
query: SearchFilters,
|
||||
params: RowSearchParams
|
||||
): Promise<SearchResponse<Row>> {
|
||||
const appId = context.getAppId()
|
||||
return dbCore.paginatedSearch(appId!, SearchIndex.ROWS, query, params)
|
||||
}
|
||||
|
||||
export async function fullSearch(
|
||||
query: SearchFilters,
|
||||
params: RowSearchParams
|
||||
): Promise<{ rows: Row[] }> {
|
||||
const appId = context.getAppId()
|
||||
return dbCore.fullSearch(appId!, SearchIndex.ROWS, query, params)
|
||||
}
|
||||
|
||||
function findColumnInQueries(
|
||||
column: string,
|
||||
filters: SearchFilters,
|
||||
|
|
|
|||
Loading…
Reference in New Issue