From e09440f077a07858c42c6c2f7e930be7aaca623c Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Thu, 13 May 2021 12:26:18 +0100 Subject: [PATCH] Allow multipage searches and implement optional pagination to data providers --- packages/client/src/api/tables.js | 22 +- packages/server/src/api/controllers/row.js | 40 ---- .../src/api/controllers/search/index.js | 47 ++-- .../src/api/controllers/search/utils.js | 206 ++++++++++++------ packages/server/src/api/routes/row.js | 6 - packages/standard-components/manifest.json | 52 +---- .../src/DataProvider.svelte | 91 +++++--- .../standard-components/src/Search.svelte | 195 ----------------- 8 files changed, 241 insertions(+), 418 deletions(-) delete mode 100644 packages/standard-components/src/Search.svelte diff --git a/packages/client/src/api/tables.js b/packages/client/src/api/tables.js index 693ce8f013..a75a2d368b 100644 --- a/packages/client/src/api/tables.js +++ b/packages/client/src/api/tables.js @@ -17,24 +17,6 @@ export const fetchTableData = async tableId => { return await enrichRows(rows, tableId) } -/** - * Perform a mango query against an internal table - * @param {String} tableId - id of the table to search - * @param {Object} search - Mango Compliant search object - * @param {Object} pagination - the pagination controls - */ -export const searchTableData = async ({ tableId, search, pagination }) => { - const output = await API.post({ - url: `/api/${tableId}/rows/search`, - body: { - query: search, - pagination, - }, - }) - output.rows = await enrichRows(output.rows, tableId) - return output -} - /** * Searches a table using Lucene. */ @@ -47,6 +29,7 @@ export const searchTable = async ({ sort, sortOrder, sortType, + paginate, }) => { if (!tableId || (!query && !raw)) { return @@ -61,10 +44,11 @@ export const searchTable = async ({ sort, sortOrder, sortType, + paginate, }, }) return { + ...res, rows: await enrichRows(res?.rows, tableId), - bookmark: res.bookmark, } } diff --git a/packages/server/src/api/controllers/row.js b/packages/server/src/api/controllers/row.js index 6d0cc08548..1d73abea47 100644 --- a/packages/server/src/api/controllers/row.js +++ b/packages/server/src/api/controllers/row.js @@ -16,7 +16,6 @@ const { const { FieldTypes } = require("../../constants") const { isEqual } = require("lodash") const { cloneDeep } = require("lodash/fp") -const { QueryBuilder, search } = require("./search/utils") const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}` @@ -248,45 +247,6 @@ exports.fetchView = async function (ctx) { } } -exports.search = async function (ctx) { - const appId = ctx.appId - const db = new CouchDB(appId) - const { - query, - pagination: { pageSize = 10, bookmark }, - } = ctx.request.body - const tableId = ctx.params.tableId - - const queryBuilder = new QueryBuilder(appId) - .setLimit(pageSize) - .addTable(tableId) - if (bookmark) { - queryBuilder.setBookmark(bookmark) - } - - let searchString - if (ctx.query && ctx.query.raw && ctx.query.raw !== "") { - searchString = queryBuilder.complete(query["RAW"]) - } else { - // make all strings a starts with operation rather than pure equality - for (const [key, queryVal] of Object.entries(query)) { - if (typeof queryVal === "string") { - queryBuilder.addString(key, queryVal) - } else { - queryBuilder.addEqual(key, queryVal) - } - } - searchString = queryBuilder.complete() - } - - const response = await search(searchString) - const table = await db.get(tableId) - ctx.body = { - rows: await outputProcessing(appId, table, response.rows), - bookmark: response.bookmark, - } -} - exports.fetchTableRows = async function (ctx) { const appId = ctx.appId const db = new CouchDB(appId) diff --git a/packages/server/src/api/controllers/search/index.js b/packages/server/src/api/controllers/search/index.js index 31c81fec9a..b8411e9d3a 100644 --- a/packages/server/src/api/controllers/search/index.js +++ b/packages/server/src/api/controllers/search/index.js @@ -1,4 +1,4 @@ -const { QueryBuilder, buildSearchUrl, search } = require("./utils") +const { fullSearch, paginatedSearch } = require("./utils") const CouchDB = require("../../../db") const { outputProcessing } = require("../../../utilities/rowProcessor") @@ -8,38 +8,45 @@ exports.rowSearch = async ctx => { const { bookmark, query, - raw, limit, sort, sortOrder, sortType, + paginate, } = ctx.request.body const db = new CouchDB(appId) - let url - if (query) { - url = new QueryBuilder( + let response + const start = Date.now() + if (paginate) { + response = await paginatedSearch( appId, query, - bookmark, - limit, + tableId, sort, sortOrder, - sortType + sortType, + limit, + bookmark ) - .addTable(tableId) - .complete() - } else if (raw) { - url = buildSearchUrl({ + } else { + response = await fullSearch( appId, - query: raw, - bookmark, - }) + query, + tableId, + sort, + sortOrder, + sortType, + limit + ) } - const response = await search(url) - const table = await db.get(tableId) - ctx.body = { - rows: await outputProcessing(appId, table, response.rows), - bookmark: response.bookmark, + const end = Date.now() + console.log("Time: " + (end - start) / 1000 + " ms") + + if (response.rows && response.rows.length) { + const table = await db.get(tableId) + response.rows = await outputProcessing(appId, table, response.rows) } + + ctx.body = response } diff --git a/packages/server/src/api/controllers/search/utils.js b/packages/server/src/api/controllers/search/utils.js index 3310f89bf7..6f2f29628a 100644 --- a/packages/server/src/api/controllers/search/utils.js +++ b/packages/server/src/api/controllers/search/utils.js @@ -3,51 +3,12 @@ const { checkSlashesInUrl } = require("../../../utilities") const env = require("../../../environment") const fetch = require("node-fetch") -/** - * Given a set of inputs this will generate the URL which is to be sent to the search proxy in CouchDB. - * @param {string} appId The ID of the app which we will be searching within. - * @param {string} query The lucene query string which is to be used for searching. - * @param {string|null} bookmark If there were more than the limit specified can send the bookmark that was - * returned with query for next set of search results. - * @param {number} limit The number of entries to return per query. - * @param {string} sort The column to sort by. - * @param {string} sortOrder The order to sort by. "ascending" or "descending". - * @param {string} sortType The type of sort to perform. "string" or "number". - * @param {boolean} excludeDocs By default full rows are returned, if required this can be disabled. - * @return {string} The URL which a GET can be performed on to receive results. - */ -function buildSearchUrl({ - appId, - query, - bookmark, - sort, - sortOrder, - sortType, - excludeDocs, - limit = 50, -}) { - let url = `${env.COUCH_DB_URL}/${appId}/_design/database/_search` - url += `/${SearchIndexes.ROWS}?q=${query}` - url += `&limit=${Math.min(limit, 200)}` - if (!excludeDocs) { - url += "&include_docs=true" - } - if (sort) { - const orderChar = sortOrder === "descending" ? "-" : "" - url += `&sort="${orderChar}${sort.replace(/ /, "_")}<${sortType}>"` - } - if (bookmark) { - url += `&bookmark=${bookmark}` - } - return checkSlashesInUrl(url) -} - const luceneEscape = value => { return `${value}`.replace(/[ #+\-&|!(){}\[\]^"~*?:\\]/g, "\\$&") } class QueryBuilder { - constructor(appId, base, bookmark, limit, sort, sortOrder, sortType) { + constructor(appId, base) { this.appId = appId this.query = { string: {}, @@ -59,11 +20,14 @@ class QueryBuilder { notEmpty: {}, ...base, } - this.bookmark = bookmark - this.limit = limit || 50 - this.sort = sort - this.sortOrder = sortOrder || "ascending" - this.sortType = sortType || "string" + this.limit = 50 + this.sortOrder = "ascending" + this.sortType = "string" + } + + setTable(tableId) { + this.query.equal.tableId = tableId + return this } setLimit(limit) { @@ -71,6 +35,21 @@ class QueryBuilder { return this } + setSort(sort) { + this.sort = sort + return this + } + + setSortOrder(sortOrder) { + this.sortOrder = sortOrder + return this + } + + setSortType(sortType) { + this.sortType = sortType + return this + } + setBookmark(bookmark) { this.bookmark = bookmark return this @@ -114,12 +93,7 @@ class QueryBuilder { return this } - addTable(tableId) { - this.query.equal.tableId = tableId - return this - } - - complete(rawQuery = null) { + buildSearchURL(excludeDocs = false) { let output = "*:*" function build(structure, queryFn) { for (let [key, value] of Object.entries(structure)) { @@ -171,22 +145,28 @@ class QueryBuilder { if (this.query.notEmpty) { build(this.query.notEmpty, key => `${key}:["" TO *]`) } - if (rawQuery) { - output = output.length === 0 ? rawQuery : `&${rawQuery}` + + let url = `${env.COUCH_DB_URL}/${this.appId}/_design/database/_search` + url += `/${SearchIndexes.ROWS}?q=${output}` + url += `&limit=${Math.min(this.limit, 200)}` + if (!excludeDocs) { + url += "&include_docs=true" } - return buildSearchUrl({ - appId: this.appId, - query: output, - bookmark: this.bookmark, - limit: this.limit, - sort: this.sort, - sortOrder: this.sortOrder, - sortType: this.sortType, - }) + if (this.sort) { + const orderChar = this.sortOrder === "descending" ? "-" : "" + url += `&sort="${orderChar}${this.sort.replace(/ /, "_")}<${ + this.sortType + }>"` + } + if (this.bookmark) { + url += `&bookmark=${this.bookmark}` + } + console.log(url) + return checkSlashesInUrl(url) } } -exports.search = async query => { +const runQuery = async query => { const response = await fetch(query, { method: "GET", }) @@ -203,5 +183,101 @@ exports.search = async query => { return output } -exports.QueryBuilder = QueryBuilder -exports.buildSearchUrl = buildSearchUrl +const recursiveSearch = async ( + appId, + query, + tableId, + sort, + sortOrder, + sortType, + limit, + bookmark, + rows +) => { + if (rows.length >= limit) { + return rows + } + const pageSize = rows.length > limit - 200 ? limit - rows.length : 200 + const url = new QueryBuilder(appId, query) + .setTable(tableId) + .setBookmark(bookmark) + .setLimit(pageSize) + .setSort(sort) + .setSortOrder(sortOrder) + .setSortType(sortType) + .buildSearchURL() + const page = await runQuery(url) + if (!page.rows.length) { + return rows + } + if (page.rows.length < 200) { + return [...rows, ...page.rows] + } + return await recursiveSearch( + appId, + query, + tableId, + sort, + sortOrder, + sortType, + limit, + page.bookmark, + [...rows, ...page.rows] + ) +} + +exports.paginatedSearch = async ( + appId, + query, + tableId, + sort, + sortOrder, + sortType, + limit, + bookmark +) => { + if (limit == null || isNaN(limit) || limit < 0) { + limit = 50 + } + const builder = new QueryBuilder(appId, query) + .setTable(tableId) + .setSort(sort) + .setSortOrder(sortOrder) + .setSortType(sortType) + .setBookmark(bookmark) + .setLimit(limit) + const searchUrl = builder.buildSearchURL() + const nextUrl = builder.setLimit(1).buildSearchURL() + const searchResults = await runQuery(searchUrl) + const nextResults = await runQuery(nextUrl) + return { + ...searchResults, + hasNextPage: nextResults.rows && nextResults.rows.length > 0, + } +} + +exports.fullSearch = async ( + appId, + query, + tableId, + sort, + sortOrder, + sortType, + limit +) => { + if (limit == null || isNaN(limit) || limit < 0) { + limit = 1000 + } + const rows = await recursiveSearch( + appId, + query, + tableId, + sort, + sortOrder, + sortType, + Math.min(limit, 1000), + null, + [] + ) + return { rows } +} diff --git a/packages/server/src/api/routes/row.js b/packages/server/src/api/routes/row.js index be14910f3e..494ea61608 100644 --- a/packages/server/src/api/routes/row.js +++ b/packages/server/src/api/routes/row.js @@ -39,12 +39,6 @@ router usage, rowController.save ) - .post( - "/api/:tableId/rows/search", - paramResource("tableId"), - authorized(PermissionTypes.TABLE, PermissionLevels.READ), - rowController.search - ) .patch( "/api/:tableId/rows/:rowId", paramSubResource("tableId", "rowId"), diff --git a/packages/standard-components/manifest.json b/packages/standard-components/manifest.json index 6a6b00732b..c19deb9d76 100644 --- a/packages/standard-components/manifest.json +++ b/packages/standard-components/manifest.json @@ -65,41 +65,6 @@ "type": "schema" } }, - "search": { - "name": "Search", - "description": "A searchable list of items.", - "icon": "Search", - "styleable": true, - "hasChildren": true, - "settings": [ - { - "type": "table", - "label": "Table", - "key": "table" - }, - { - "type": "multifield", - "label": "Columns", - "key": "columns", - "dependsOn": "table" - }, - { - "type": "number", - "label": "Rows/Page", - "defaultValue": 25, - "key": "pageSize" - }, - { - "type": "text", - "label": "Empty Text", - "key": "noRowsMessage", - "defaultValue": "No rows found." - } - ], - "context": { - "type": "schema" - } - }, "stackedlist": { "name": "Stacked List", "icon": "TaskList", @@ -1446,7 +1411,14 @@ { "type": "number", "label": "Limit", - "key": "limit" + "key": "limit", + "defaultValue": 50 + }, + { + "type": "boolean", + "label": "Paginate", + "key": "paginate", + "defaultValue": true } ], "context": { @@ -1464,14 +1436,6 @@ "label": "Schema", "key": "schema" }, - { - "label": "Loading", - "key": "loading" - }, - { - "label": "Loaded", - "key": "loaded" - }, { "label": "Page Number", "key": "pageNumber" diff --git a/packages/standard-components/src/DataProvider.svelte b/packages/standard-components/src/DataProvider.svelte index 418d80f7a9..80ddffedee 100644 --- a/packages/standard-components/src/DataProvider.svelte +++ b/packages/standard-components/src/DataProvider.svelte @@ -1,11 +1,13 @@ -
+
- + {#if !loaded && loading} +
+ +
+ {:else} + + {#if paginate} + + {/if} + {/if}
+ + diff --git a/packages/standard-components/src/Search.svelte b/packages/standard-components/src/Search.svelte deleted file mode 100644 index 29ca3b011b..0000000000 --- a/packages/standard-components/src/Search.svelte +++ /dev/null @@ -1,195 +0,0 @@ - - - -
-
- {#if schema} - {#each columns as field} -
- - {#if schema[field].type === "options"} - - {:else if schema[field].type === "datetime"} - - {:else if schema[field].type === "boolean"} - - {:else if schema[field].type === "number"} - - {:else if schema[field].type === "string"} - - {/if} -
- {/each} - {/if} -
- - -
-
- {#if loaded} - {#if rows.length > 0} - {#if $component.children === 0 && $builderStore.inBuilder} -

Add some components to display.

- {:else} - {#each rows as row} - - - - {/each} - {/if} - {:else if noRowsMessage} -

{noRowsMessage}

- {/if} - {/if} - -
- - -