From 97f8d3494775a46882909a7dc3d25234e7541c58 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 26 Mar 2021 13:46:20 +0000 Subject: [PATCH] Some final changes to search system so that the new indexing system is used instead of mango to achieve exactly the same result. --- packages/server/src/api/controllers/row.js | 24 +++++--- .../src/api/controllers/search/index.js | 18 ++++++ .../{search.js => search/utils.js} | 58 +++++++++---------- packages/server/src/db/linkedRows/index.js | 2 +- .../standard-components/src/Search.svelte | 13 +++-- 5 files changed, 71 insertions(+), 44 deletions(-) create mode 100644 packages/server/src/api/controllers/search/index.js rename packages/server/src/api/controllers/{search.js => search/utils.js} (62%) diff --git a/packages/server/src/api/controllers/row.js b/packages/server/src/api/controllers/row.js index 1f5c410aaf..537fc5c850 100644 --- a/packages/server/src/api/controllers/row.js +++ b/packages/server/src/api/controllers/row.js @@ -17,7 +17,7 @@ const { const { FieldTypes } = require("../../constants") const { isEqual } = require("lodash") const { cloneDeep } = require("lodash/fp") -const searchController = require("./search") +const { QueryBuilder, search } = require("./search/utils") const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}` @@ -264,23 +264,29 @@ exports.search = async function(ctx) { } = ctx.request.body const tableId = ctx.params.tableId - const queryBuilder = new searchController.QueryBuilder(appId) + const queryBuilder = new QueryBuilder(appId) .setLimit(pageSize) .addTable(tableId) if (bookmark) { queryBuilder.setBookmark(bookmark) } - // make all strings a starts with operation rather than pure equality - for (const [key, queryVal] of Object.entries(query)) { - if (typeof queryVal === "string") { - queryBuilder.addString(key, queryVal) - } else { - queryBuilder.addEqual(key, queryVal) + let searchString + if (ctx.query.raw && ctx.query.raw !== "") { + searchString = queryBuilder.complete(query["RAW"]) + } else { + // make all strings a starts with operation rather than pure equality + for (const [key, queryVal] of Object.entries(query)) { + if (typeof queryVal === "string") { + queryBuilder.addString(key, queryVal) + } else { + queryBuilder.addEqual(key, queryVal) + } } + searchString = queryBuilder.complete() } - const response = await searchController.search(queryBuilder.complete()) + const response = await search(searchString) // delete passwords from users if (tableId === ViewNames.USERS) { diff --git a/packages/server/src/api/controllers/search/index.js b/packages/server/src/api/controllers/search/index.js new file mode 100644 index 0000000000..1810f07198 --- /dev/null +++ b/packages/server/src/api/controllers/search/index.js @@ -0,0 +1,18 @@ +const { QueryBuilder, buildSearchUrl, search } = require("./utils") + +exports.rowSearch = async ctx => { + // this can't be done through pouch, have to reach for trusty node-fetch + const appId = ctx.user.appId + const bookmark = ctx.params.bookmark + let url + if (ctx.params.query) { + url = new QueryBuilder(appId, ctx.params.query, bookmark).complete() + } else if (ctx.params.raw) { + url = buildSearchUrl({ + appId, + query: ctx.params.raw, + bookmark, + }) + } + ctx.body = await search(url) +} diff --git a/packages/server/src/api/controllers/search.js b/packages/server/src/api/controllers/search/utils.js similarity index 62% rename from packages/server/src/api/controllers/search.js rename to packages/server/src/api/controllers/search/utils.js index 0c60ebe5e6..1032755759 100644 --- a/packages/server/src/api/controllers/search.js +++ b/packages/server/src/api/controllers/search/utils.js @@ -1,23 +1,25 @@ +const { SearchIndexes } = require("../../../db/utils") +const { checkSlashesInUrl } = require("../../../utilities") +const env = require("../../../environment") const fetch = require("node-fetch") -const { SearchIndexes } = require("../../db/utils") -const { checkSlashesInUrl } = require("../../utilities") -const env = require("../../environment") -function buildSearchUrl( - appId, - query, - bookmark = null, - limit = 50, - includeDocs = true -) { +/** + * Given a set of inputs this will generate the URL which is to be sent to the search proxy in CouchDB. + * @param {string} appId The ID of the app which we will be searching within. + * @param {string} query The lucene query string which is to be used for searching. + * @param {string|null} bookmark If there were more than the limit specified can send the bookmark that was + * returned with query for next set of search results. + * @param {number} limit The number of entries to return per query. + * @param {boolean} excludeDocs By default full rows are returned, if required this can be disabled. + * @return {string} The URL which a GET can be performed on to receive results. + */ +function buildSearchUrl({ appId, query, bookmark, excludeDocs, limit = 50 }) { let url = `${env.COUCH_DB_URL}/${appId}/_design/database/_search` url += `/${SearchIndexes.ROWS}?q=${query}` - if (includeDocs) { + url += `&limit=${limit}` + if (!excludeDocs) { url += "&include_docs=true" } - if (limit) { - url += `&limit=${limit}` - } if (bookmark) { url += `&bookmark=${bookmark}` } @@ -77,7 +79,7 @@ class QueryBuilder { return this } - complete() { + complete(rawQuery = null) { let output = "" function build(structure, queryFn) { for (let [key, value] of Object.entries(structure)) { @@ -101,12 +103,18 @@ class QueryBuilder { if (this.query.fuzzy) { build(this.query.fuzzy, (key, value) => `${key}:${value}~`) } - return buildSearchUrl(this.appId, output, this.bookmark, this.limit) + if (rawQuery) { + output = output.length === 0 ? rawQuery : `&${rawQuery}` + } + return buildSearchUrl({ + appId: this.appId, + query: output, + bookmark: this.bookmark, + limit: this.limit, + }) } } -exports.QueryBuilder = QueryBuilder - exports.search = async query => { const response = await fetch(query, { method: "GET", @@ -124,15 +132,5 @@ exports.search = async query => { return output } -exports.rowSearch = async ctx => { - // this can't be done through pouch, have to reach for trusty node-fetch - const appId = ctx.user.appId - const bookmark = ctx.params.bookmark - let url - if (ctx.params.query) { - url = new QueryBuilder(appId, ctx.params.query, bookmark).complete() - } else if (ctx.params.raw) { - url = buildSearchUrl(appId, ctx.params.raw, bookmark) - } - ctx.body = await exports.search(url) -} +exports.QueryBuilder = QueryBuilder +exports.buildSearchUrl = buildSearchUrl diff --git a/packages/server/src/db/linkedRows/index.js b/packages/server/src/db/linkedRows/index.js index 052386ba86..8de2093fb2 100644 --- a/packages/server/src/db/linkedRows/index.js +++ b/packages/server/src/db/linkedRows/index.js @@ -27,7 +27,7 @@ const EventType = { } exports.EventType = EventType -// re-export utils here for ease of use +// re-export search here for ease of use exports.IncludeDocs = IncludeDocs exports.getLinkDocuments = getLinkDocuments exports.createLinkView = createLinkView diff --git a/packages/standard-components/src/Search.svelte b/packages/standard-components/src/Search.svelte index fc09070a87..a36f13b999 100644 --- a/packages/standard-components/src/Search.svelte +++ b/packages/standard-components/src/Search.svelte @@ -48,7 +48,6 @@ if (table) { const tableDef = await API.fetchTableDefinition(table) schema = tableDef.schema - lastBookmark = mark const output = await API.searchTableData({ tableId: table, search: parsedSearch, @@ -70,7 +69,13 @@ function previousPage() { nextBookmark = bookmark - bookmark = lastBookmark + if (lastBookmark !== bookmark) { + bookmark = lastBookmark + } else { + // special case for going back to beginning + bookmark = null + lastBookmark = null + } } @@ -135,10 +140,10 @@ {/if} {/if}