MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Test deep pagination

This commit is contained in:
adrinr 2023-03-16 13:37:17 +01:00
parent 30c66748af
commit 9a2eaaad42
2 changed files with 25 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
packages/backend-core/src/db/lucene.ts
View File

@ -58,6 +58,8 @@ export class QueryBuilder<T> {
#noEscaping = false #noEscaping = false
#skip?: number #skip?: number
static readonly maxLimit = 200
constructor(dbName: string, index: string, base?: SearchFilters) { constructor(dbName: string, index: string, base?: SearchFilters) {
this.#dbName = dbName this.#dbName = dbName
this.#index = index this.#index = index
@ -459,7 +461,7 @@ export class QueryBuilder<T> {
buildSearchBody() { buildSearchBody() {
let body: any = { let body: any = {
q: this.buildSearchQuery(), q: this.buildSearchQuery(),
limit: Math.min(this.#limit, 200), limit: Math.min(this.#limit, QueryBuilder.maxLimit),
include_docs: this.#includeDocs, include_docs: this.#includeDocs,
} }
if (this.#bookmark) { if (this.#bookmark) {
@ -492,14 +494,13 @@ export class QueryBuilder<T> {
const prevLimit = this.#limit const prevLimit = this.#limit
this.excludeDocs() this.excludeDocs()
const maxPageSize = 1000
let skipRemaining = skip let skipRemaining = skip
do { do {
const toSkip = Math.min(maxPageSize, skipRemaining) const toSkip = Math.min(QueryBuilder.maxLimit, skipRemaining)
this.setLimit(toSkip) this.setLimit(toSkip)
const { bookmark } = await this.#execute() const { bookmark, rows } = await this.#execute()
this.setBookmark(bookmark) this.setBookmark(bookmark)
skipRemaining -= toSkip skipRemaining -= rows.length
} while (skipRemaining > 0) } while (skipRemaining > 0)
this.#includeDocs = prevIncludeDocs this.#includeDocs = prevIncludeDocs
@ -596,8 +597,8 @@ async function recursiveSearch<T>(
if (rows.length >= params.limit) { if (rows.length >= params.limit) {
return rows return rows
} }
let pageSize = 200 let pageSize = QueryBuilder.maxLimit
if (rows.length > params.limit - 200) { if (rows.length > params.limit - QueryBuilder.maxLimit) {
pageSize = params.limit - rows.length pageSize = params.limit - rows.length
} }
const page = await new QueryBuilder<T>(dbName, index, query) const page = await new QueryBuilder<T>(dbName, index, query)
@ -612,7 +613,7 @@ async function recursiveSearch<T>(
if (!page.rows.length) { if (!page.rows.length) {
return rows return rows
} }
if (page.rows.length < 200) { if (page.rows.length < QueryBuilder.maxLimit) {
return [...rows, ...page.rows] return [...rows, ...page.rows]
} }
const newParams = { const newParams = {
@ -650,7 +651,7 @@ export async function paginatedSearch<T>(
if (limit == null || isNaN(limit) || limit < 0) { if (limit == null || isNaN(limit) || limit < 0) {
limit = 50 limit = 50
} }
limit = Math.min(limit, 200) limit = Math.min(limit, QueryBuilder.maxLimit)
const search = new QueryBuilder<T>(dbName, index, query) const search = new QueryBuilder<T>(dbName, index, query)
if (params.version) { if (params.version) {
search.setVersion(params.version) search.setVersion(params.version)

16
packages/backend-core/src/db/tests/lucene.spec.ts
View File

@ -148,7 +148,7 @@ describe("lucene", () => {
beforeAll(async () => { beforeAll(async () => {
const db = getDB(skipDbName) const db = getDB(skipDbName)
docs = Array(1500) docs = Array(QueryBuilder.maxLimit * 2.5)
.fill(0) .fill(0)
.map((_, i) => ({ .map((_, i) => ({
_id: i.toString().padStart(4, "0"), _id: i.toString().padStart(4, "0"),
@ -196,6 +196,20 @@ describe("lucene", () => {
docs.slice(50, 60).map(expect.objectContaining) docs.slice(50, 60).map(expect.objectContaining)
) )
}) })
it("should be able to skip searching through multiple responses", async () => {
const builder = new QueryBuilder(skipDbName, INDEX_NAME)
// Skipping 2 max limits plus a little bit more
const skip = QueryBuilder.maxLimit * 2 + 37
builder.setSkip(skip)
builder.setSort("_id")
const resp = await builder.run()
expect(resp.rows.length).toBe(50)
expect(resp.rows).toEqual(
docs.slice(skip, skip + resp.rows.length).map(expect.objectContaining)
)
})
}) })
}) })