From dfa2881f1c60caa0540e1e69522291814e225201 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 25 Mar 2021 18:04:44 +0000 Subject: [PATCH 1/8] Initialising CouchDB link to lucene and app design DBs for it. --- hosting/couch/Dockerfile | 3 ++ hosting/couch/lucene-proxy.ini | 2 + hosting/docker-compose.dev.yaml | 31 +++++++++-- hosting/hosting.properties | 1 + hosting/lucene/Dockerfile | 51 +++++++++++++++++++ hosting/lucene/run-lucene.sh | 15 ++++++ .../server/src/api/controllers/application.js | 8 ++- packages/server/src/db/views/staticViews.js | 36 +++++++++++++ 8 files changed, 141 insertions(+), 6 deletions(-) create mode 100644 hosting/couch/Dockerfile create mode 100644 hosting/couch/lucene-proxy.ini create mode 100644 hosting/lucene/Dockerfile create mode 100755 hosting/lucene/run-lucene.sh diff --git a/hosting/couch/Dockerfile b/hosting/couch/Dockerfile new file mode 100644 index 0000000000..0695a686be --- /dev/null +++ b/hosting/couch/Dockerfile @@ -0,0 +1,3 @@ +FROM apache/couchdb + +COPY lucene-proxy.ini /usr/local/etc/couchdb/local.d/ diff --git a/hosting/couch/lucene-proxy.ini b/hosting/couch/lucene-proxy.ini new file mode 100644 index 0000000000..3ac6d90542 --- /dev/null +++ b/hosting/couch/lucene-proxy.ini @@ -0,0 +1,2 @@ +[httpd_global_handlers] +_fti = {couch_httpd_proxy, handle_proxy_req, <<"http://couchdb-lucene:5985">>} diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index 39fcb7ec83..2cbba5771f 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -30,26 +30,44 @@ services: - ./envoy.dev.yaml:/etc/envoy/envoy.yaml ports: - "${MAIN_PORT}:10000" - #- "9901:9901" depends_on: - minio-service - couchdb-service + couchdb-lucene: + container_name: budi-couchdb-lucene-dev + build: + context: lucene + dockerfile: Dockerfile + ports: + - "${COUCH_LUCENE_PORT}:5985" + volumes: + - couchdb_lucene:/opt/couchdb-lucene + networks: + dbs: + aliases: + - couchdb-lucene + couchdb-service: container_name: budi-couchdb-dev restart: always - image: apache/couchdb:3.0 + build: + context: couch + dockerfile: Dockerfile environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} ports: - "${COUCH_DB_PORT}:5984" - #- "4369:4369" - #- "9100:9100" volumes: - couchdb_data:/opt/couchdb/data +# networks: +# dbs: +# aliases: +# - couchdb couch-init: + container_name: budi-couchdb-init-dev image: curlimages/curl environment: PUT_CALL: "curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984" @@ -66,6 +84,9 @@ services: volumes: - redis_data:/data +networks: + dbs: + driver: bridge volumes: couchdb_data: @@ -74,3 +95,5 @@ volumes: driver: local redis_data: driver: local + couchdb_lucene: + driver: local diff --git a/hosting/hosting.properties b/hosting/hosting.properties index 138e66d629..5b80a844dd 100644 --- a/hosting/hosting.properties +++ b/hosting/hosting.properties @@ -18,5 +18,6 @@ APP_PORT=4002 WORKER_PORT=4003 MINIO_PORT=4004 COUCH_DB_PORT=4005 +COUCH_LUCENE_PORT=4006 REDIS_PORT=6379 BUDIBASE_ENVIRONMENT=PRODUCTION diff --git a/hosting/lucene/Dockerfile b/hosting/lucene/Dockerfile new file mode 100644 index 0000000000..135237370b --- /dev/null +++ b/hosting/lucene/Dockerfile @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +FROM openjdk:8 + +RUN groupadd -r couchdb && useradd -d /opt/couchdb-lucene -g couchdb couchdb + +# grab gosu for easy step-down from root and tini for signal handling +RUN gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \ + && curl -o /usr/local/bin/gosu -fSL "https://github.com/tianon/gosu/releases/download/1.7/gosu-$(dpkg --print-architecture)" \ + && curl -o /usr/local/bin/gosu.asc -fSL "https://github.com/tianon/gosu/releases/download/1.7/gosu-$(dpkg --print-architecture).asc" \ + && gpg --verify /usr/local/bin/gosu.asc \ + && rm /usr/local/bin/gosu.asc \ + && chmod +x /usr/local/bin/gosu + +ENV COUCHDB_LUCENE_VERSION 2.1.0 + +RUN apt-get update \ + && apt-get install -y maven \ + && cd /usr/src \ + && curl -L https://github.com/rnewson/couchdb-lucene/archive/v$COUCHDB_LUCENE_VERSION.tar.gz | tar -xz \ + && cd couchdb-lucene-$COUCHDB_LUCENE_VERSION \ + && mvn + +RUN cd /usr/src/couchdb-lucene-$COUCHDB_LUCENE_VERSION/target \ + && unzip couchdb-lucene-$COUCHDB_LUCENE_VERSION-dist.zip \ + && mv couchdb-lucene-$COUCHDB_LUCENE_VERSION /opt/couchdb-lucene \ + && rm -rf /usr/src/couchdb-lucene-* + +RUN apt-get remove --auto-remove -y maven \ + && rm -rf /var/lib/apt/lists/* \ + && sed -e 's/^host=localhost$/host=0.0.0.0/' -i /opt/couchdb-lucene/conf/couchdb-lucene.ini \ + && sed -e 's/localhost:5984/couchdb:5984/' -i /opt/couchdb-lucene/conf/couchdb-lucene.ini \ + && chown -R couchdb:couchdb /opt/couchdb-lucene + +COPY ./run-lucene.sh /opt/couchdb-lucene/run-lucene.sh +RUN chmod +x /opt/couchdb-lucene/run-lucene.sh + +WORKDIR /opt/couchdb-lucene +EXPOSE 5985 +VOLUME ["/opt/couchdb-lucene/indexes"] +CMD ["./run-lucene.sh"] diff --git a/hosting/lucene/run-lucene.sh b/hosting/lucene/run-lucene.sh new file mode 100755 index 0000000000..945bb5655c --- /dev/null +++ b/hosting/lucene/run-lucene.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +chown -R couchdb:couchdb /opt/couchdb-lucene +exec gosu couchdb ./bin/run diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index c5d48038ce..59cdb58904 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -2,8 +2,11 @@ const CouchDB = require("../../db") const env = require("../../environment") const setBuilderToken = require("../../utilities/builder/setBuilderToken") const packageJson = require("../../../package.json") -const { createLinkView } = require("../../db/linkedRows") -const { createRoutingView } = require("../../utilities/routing") +const { + createLinkView, + createRoutingView, + createFulltextSearchIndex, +} = require("../../db/views/staticViews") const { getTemplateStream, createApp, @@ -92,6 +95,7 @@ async function createInstance(template) { // add view for linked rows await createLinkView(appId) await createRoutingView(appId) + await createFulltextSearchIndex(appId) // replicate the template data to the instance DB // this is currently very hard to test, downloading and importing template files diff --git a/packages/server/src/db/views/staticViews.js b/packages/server/src/db/views/staticViews.js index a0d0fbb239..50e3a4a1a9 100644 --- a/packages/server/src/db/views/staticViews.js +++ b/packages/server/src/db/views/staticViews.js @@ -72,3 +72,39 @@ exports.createRoutingView = async appId => { } await db.put(designDoc) } + +exports.createFulltextSearchIndex = async appId => { + const db = new CouchDB(appId) + const designDoc = await db.get("_design/database") + designDoc.fulltext = { + everything: { + index: function(doc) { + let ret = new Document() + + function idx(obj) { + for (let key of Object.keys(obj)) { + switch (typeof obj[key]) { + case "object": + idx(obj[key]) + break + case "function": + break + default: + ret.add(obj[key]) + break + } + } + } + + idx(doc) + if (doc._attachments) { + for (let i in Object.keys(doc._attachments)) { + ret.attachment("default", i) + } + } + return ret + }.toString(), + }, + } + await db.put(designDoc) +} From 03ef14129784362192933dec056c0e35461271a9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 25 Mar 2021 19:12:17 +0000 Subject: [PATCH 2/8] Creating CouchDB 3.0 indexes. --- hosting/docker-compose.dev.yaml | 32 ++----------------- hosting/hosting.properties | 1 - packages/server/src/db/views/staticViews.js | 34 ++++++++------------- 3 files changed, 16 insertions(+), 51 deletions(-) diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index 2cbba5771f..8ee1753e11 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -34,37 +34,17 @@ services: - minio-service - couchdb-service - couchdb-lucene: - container_name: budi-couchdb-lucene-dev - build: - context: lucene - dockerfile: Dockerfile - ports: - - "${COUCH_LUCENE_PORT}:5985" - volumes: - - couchdb_lucene:/opt/couchdb-lucene - networks: - dbs: - aliases: - - couchdb-lucene - couchdb-service: container_name: budi-couchdb-dev restart: always - build: - context: couch - dockerfile: Dockerfile + image: ibmcom/couchdb3 environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} ports: - "${COUCH_DB_PORT}:5984" volumes: - - couchdb_data:/opt/couchdb/data -# networks: -# dbs: -# aliases: -# - couchdb + - couchdb3_data:/opt/couchdb/data couch-init: container_name: budi-couchdb-init-dev @@ -84,16 +64,10 @@ services: volumes: - redis_data:/data -networks: - dbs: - driver: bridge - volumes: - couchdb_data: + couchdb3_data: driver: local minio_data: driver: local redis_data: driver: local - couchdb_lucene: - driver: local diff --git a/hosting/hosting.properties b/hosting/hosting.properties index 5b80a844dd..138e66d629 100644 --- a/hosting/hosting.properties +++ b/hosting/hosting.properties @@ -18,6 +18,5 @@ APP_PORT=4002 WORKER_PORT=4003 MINIO_PORT=4004 COUCH_DB_PORT=4005 -COUCH_LUCENE_PORT=4006 REDIS_PORT=6379 BUDIBASE_ENVIRONMENT=PRODUCTION diff --git a/packages/server/src/db/views/staticViews.js b/packages/server/src/db/views/staticViews.js index 50e3a4a1a9..ae6496bb4a 100644 --- a/packages/server/src/db/views/staticViews.js +++ b/packages/server/src/db/views/staticViews.js @@ -76,33 +76,25 @@ exports.createRoutingView = async appId => { exports.createFulltextSearchIndex = async appId => { const db = new CouchDB(appId) const designDoc = await db.get("_design/database") - designDoc.fulltext = { - everything: { + designDoc.indexes = { + rows: { index: function(doc) { - let ret = new Document() - - function idx(obj) { + // eslint-disable-next-line no-undef + index("id", doc._id) + function idx(obj, prev = "") { for (let key of Object.keys(obj)) { - switch (typeof obj[key]) { - case "object": - idx(obj[key]) - break - case "function": - break - default: - ret.add(obj[key]) - break + let prevKey = prev !== "" ? `${prev}.${key}` : key + if (typeof obj[key] !== "object") { + // eslint-disable-next-line no-undef + index(prevKey, obj[key], { store: true }) + } else { + idx(obj[key], prevKey) } } } - - idx(doc) - if (doc._attachments) { - for (let i in Object.keys(doc._attachments)) { - ret.attachment("default", i) - } + if (doc._id.startsWith("ro_")) { + idx(doc) } - return ret }.toString(), }, } From a5fd8d0e335e50cb4289d9cc6ee78ba8a352f962 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 25 Mar 2021 23:42:50 +0000 Subject: [PATCH 3/8] Rewriting search to use the new couchdb 3.0 search functionality. --- packages/client/src/api/tables.js | 6 +- .../server/src/api/controllers/application.js | 4 +- packages/server/src/api/controllers/row.js | 40 ++--- packages/server/src/api/controllers/search.js | 138 ++++++++++++++++++ .../src/api/controllers/static/index.js | 6 +- packages/server/src/api/routes/search.js | 8 + packages/server/src/db/utils.js | 5 + packages/server/src/db/views/staticViews.js | 58 +++++--- packages/server/src/utilities/index.js | 4 + .../standard-components/src/Search.svelte | 32 ++-- 10 files changed, 240 insertions(+), 61 deletions(-) create mode 100644 packages/server/src/api/controllers/search.js create mode 100644 packages/server/src/api/routes/search.js diff --git a/packages/client/src/api/tables.js b/packages/client/src/api/tables.js index ce06019b54..248e1516c2 100644 --- a/packages/client/src/api/tables.js +++ b/packages/client/src/api/tables.js @@ -21,14 +21,16 @@ export const fetchTableData = async tableId => { * Perform a mango query against an internal table * @param {String} tableId - id of the table to search * @param {Object} search - Mango Compliant search object + * @param {Object} pagination - the pagination controls */ export const searchTableData = async ({ tableId, search, pagination }) => { - const rows = await API.post({ + const output = await API.post({ url: `/api/${tableId}/rows/search`, body: { query: search, pagination, }, }) - return await enrichRows(rows, tableId) + output.rows = await enrichRows(output.rows, tableId) + return output } diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index 59cdb58904..00678b85a0 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -5,7 +5,7 @@ const packageJson = require("../../../package.json") const { createLinkView, createRoutingView, - createFulltextSearchIndex, + createAllSearchIndex, } = require("../../db/views/staticViews") const { getTemplateStream, @@ -95,7 +95,7 @@ async function createInstance(template) { // add view for linked rows await createLinkView(appId) await createRoutingView(appId) - await createFulltextSearchIndex(appId) + await createAllSearchIndex(appId) // replicate the template data to the instance DB // this is currently very hard to test, downloading and importing template files diff --git a/packages/server/src/api/controllers/row.js b/packages/server/src/api/controllers/row.js index bf985fe55d..1f5c410aaf 100644 --- a/packages/server/src/api/controllers/row.js +++ b/packages/server/src/api/controllers/row.js @@ -17,6 +17,7 @@ const { const { FieldTypes } = require("../../constants") const { isEqual } = require("lodash") const { cloneDeep } = require("lodash/fp") +const searchController = require("./search") const TABLE_VIEW_BEGINS_WITH = `all${SEPARATOR}${DocumentTypes.TABLE}${SEPARATOR}` @@ -259,39 +260,40 @@ exports.search = async function(ctx) { const db = new CouchDB(appId) const { query, - pagination: { pageSize = 10, page }, + pagination: { pageSize = 10, bookmark }, } = ctx.request.body + const tableId = ctx.params.tableId + + const queryBuilder = new searchController.QueryBuilder(appId) + .setLimit(pageSize) + .addTable(tableId) + if (bookmark) { + queryBuilder.setBookmark(bookmark) + } // make all strings a starts with operation rather than pure equality for (const [key, queryVal] of Object.entries(query)) { if (typeof queryVal === "string") { - query[key] = { - $gt: queryVal, - $lt: `${queryVal}\uffff`, - } + queryBuilder.addString(key, queryVal) + } else { + queryBuilder.addEqual(key, queryVal) } } - // pure equality for table - query.tableId = ctx.params.tableId - const response = await db.find({ - selector: query, - limit: pageSize, - skip: pageSize * page, - }) - - const rows = response.docs + const response = await searchController.search(queryBuilder.complete()) // delete passwords from users - if (query.tableId === ViewNames.USERS) { - for (let row of rows) { + if (tableId === ViewNames.USERS) { + for (let row of response.rows) { delete row.password } } - const table = await db.get(ctx.params.tableId) - - ctx.body = await outputProcessing(appId, table, rows) + const table = await db.get(tableId) + ctx.body = { + rows: await outputProcessing(appId, table, response.rows), + bookmark: response.bookmark, + } } exports.fetchTableRows = async function(ctx) { diff --git a/packages/server/src/api/controllers/search.js b/packages/server/src/api/controllers/search.js new file mode 100644 index 0000000000..0c60ebe5e6 --- /dev/null +++ b/packages/server/src/api/controllers/search.js @@ -0,0 +1,138 @@ +const fetch = require("node-fetch") +const { SearchIndexes } = require("../../db/utils") +const { checkSlashesInUrl } = require("../../utilities") +const env = require("../../environment") + +function buildSearchUrl( + appId, + query, + bookmark = null, + limit = 50, + includeDocs = true +) { + let url = `${env.COUCH_DB_URL}/${appId}/_design/database/_search` + url += `/${SearchIndexes.ROWS}?q=${query}` + if (includeDocs) { + url += "&include_docs=true" + } + if (limit) { + url += `&limit=${limit}` + } + if (bookmark) { + url += `&bookmark=${bookmark}` + } + return checkSlashesInUrl(url) +} + +class QueryBuilder { + constructor(appId, base) { + this.appId = appId + this.query = { + string: {}, + fuzzy: {}, + range: {}, + equal: {}, + meta: {}, + ...base, + } + this.limit = 50 + this.bookmark = null + } + + setLimit(limit) { + this.limit = limit + return this + } + + setBookmark(bookmark) { + this.bookmark = bookmark + return this + } + + addString(key, partial) { + this.query.string[key] = partial + return this + } + + addFuzzy(key, fuzzy) { + this.query.fuzzy[key] = fuzzy + return this + } + + addRange(key, low, high) { + this.query.range = { + low, + high, + } + return this + } + + addEqual(key, value) { + this.query.equal[key] = value + return this + } + + addTable(tableId) { + this.query.equal.tableId = tableId + return this + } + + complete() { + let output = "" + function build(structure, queryFn) { + for (let [key, value] of Object.entries(structure)) { + if (output.length !== 0) { + output += " AND " + } + output += queryFn(key, value) + } + } + + if (this.query.string) { + build(this.query.string, (key, value) => `${key}:${value}*`) + } + if (this.query.number) { + build(this.query.number, (key, value) => + value.length == null + ? `${key}:${value}` + : `${key}:[${value[0]} TO ${value[1]}]` + ) + } + if (this.query.fuzzy) { + build(this.query.fuzzy, (key, value) => `${key}:${value}~`) + } + return buildSearchUrl(this.appId, output, this.bookmark, this.limit) + } +} + +exports.QueryBuilder = QueryBuilder + +exports.search = async query => { + const response = await fetch(query, { + method: "GET", + }) + const json = await response.json() + let output = { + rows: [], + } + if (json.rows != null && json.rows.length > 0) { + output.rows = json.rows.map(row => row.doc) + } + if (json.bookmark) { + output.bookmark = json.bookmark + } + return output +} + +exports.rowSearch = async ctx => { + // this can't be done through pouch, have to reach for trusty node-fetch + const appId = ctx.user.appId + const bookmark = ctx.params.bookmark + let url + if (ctx.params.query) { + url = new QueryBuilder(appId, ctx.params.query, bookmark).complete() + } else if (ctx.params.raw) { + url = buildSearchUrl(appId, ctx.params.raw, bookmark) + } + ctx.body = await exports.search(url) +} diff --git a/packages/server/src/api/controllers/static/index.js b/packages/server/src/api/controllers/static/index.js index c866db3561..7caf6d0f7f 100644 --- a/packages/server/src/api/controllers/static/index.js +++ b/packages/server/src/api/controllers/static/index.js @@ -2,6 +2,7 @@ require("svelte/register") const send = require("koa-send") const { resolve, join } = require("../../../utilities/centralPath") +const { checkSlashesInUrl } = require("../../../utilities") const fetch = require("node-fetch") const uuid = require("uuid") const { prepareUpload } = require("../deploy/utils") @@ -28,10 +29,7 @@ function objectStoreUrl() { function internalObjectStoreUrl() { if (env.SELF_HOSTED) { - return (env.MINIO_URL + OBJ_STORE_DIRECTORY).replace( - /(https?:\/\/)|(\/)+/g, - "$1$2" - ) + return checkSlashesInUrl(env.MINIO_URL + OBJ_STORE_DIRECTORY) } else { return BB_CDN } diff --git a/packages/server/src/api/routes/search.js b/packages/server/src/api/routes/search.js new file mode 100644 index 0000000000..8858a72d6e --- /dev/null +++ b/packages/server/src/api/routes/search.js @@ -0,0 +1,8 @@ +const Router = require("@koa/router") +const controller = require("../controllers/search") + +const router = Router() + +router.get("/api/search/rows", controller.rowSearch) + +module.exports = router diff --git a/packages/server/src/db/utils.js b/packages/server/src/db/utils.js index e480d4f554..4c31f0398e 100644 --- a/packages/server/src/db/utils.js +++ b/packages/server/src/db/utils.js @@ -37,11 +37,16 @@ const ViewNames = { USERS: "ta_users", } +const SearchIndexes = { + ROWS: "rows", +} + exports.StaticDatabases = StaticDatabases exports.ViewNames = ViewNames exports.DocumentTypes = DocumentTypes exports.SEPARATOR = SEPARATOR exports.UNICODE_MAX = UNICODE_MAX +exports.SearchIndexes = SearchIndexes exports.getQueryIndex = viewName => { return `database/${viewName}` diff --git a/packages/server/src/db/views/staticViews.js b/packages/server/src/db/views/staticViews.js index ae6496bb4a..305d042217 100644 --- a/packages/server/src/db/views/staticViews.js +++ b/packages/server/src/db/views/staticViews.js @@ -1,5 +1,10 @@ const CouchDB = require("../index") -const { DocumentTypes, SEPARATOR, ViewNames } = require("../utils") +const { + DocumentTypes, + SEPARATOR, + ViewNames, + SearchIndexes, +} = require("../utils") const SCREEN_PREFIX = DocumentTypes.SCREEN + SEPARATOR /************************************************** @@ -73,30 +78,41 @@ exports.createRoutingView = async appId => { await db.put(designDoc) } -exports.createFulltextSearchIndex = async appId => { +async function searchIndex(appId, indexName, fnString) { const db = new CouchDB(appId) const designDoc = await db.get("_design/database") designDoc.indexes = { - rows: { - index: function(doc) { - // eslint-disable-next-line no-undef - index("id", doc._id) - function idx(obj, prev = "") { - for (let key of Object.keys(obj)) { - let prevKey = prev !== "" ? `${prev}.${key}` : key - if (typeof obj[key] !== "object") { - // eslint-disable-next-line no-undef - index(prevKey, obj[key], { store: true }) - } else { - idx(obj[key], prevKey) - } - } - } - if (doc._id.startsWith("ro_")) { - idx(doc) - } - }.toString(), + [indexName]: { + index: fnString, }, } await db.put(designDoc) } + +exports.createAllSearchIndex = async appId => { + await searchIndex( + appId, + SearchIndexes.ROWS, + function(doc) { + function idx(input, prev) { + for (let key of Object.keys(input)) { + const idxKey = prev != null ? `${prev}.${key}` : key + if (key === "_id" || key === "_rev") { + continue + } + if (typeof input[key] !== "object") { + // eslint-disable-next-line no-undef + index(idxKey, input[key], { store: true }) + } else { + idx(input[key], idxKey) + } + } + } + if (doc._id.startsWith("ro_")) { + // eslint-disable-next-line no-undef + index("default", doc._id) + idx(doc) + } + }.toString() + ) +} diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js index 7d6794b1b3..ad92987434 100644 --- a/packages/server/src/utilities/index.js +++ b/packages/server/src/utilities/index.js @@ -106,3 +106,7 @@ exports.getAllApps = async () => { .map(({ value }) => value) } } + +exports.checkSlashesInUrl = url => { + return url.replace(/(https?:\/\/)|(\/)+/g, "$1$2") +} diff --git a/packages/standard-components/src/Search.svelte b/packages/standard-components/src/Search.svelte index 509205f8f1..fc09070a87 100644 --- a/packages/standard-components/src/Search.svelte +++ b/packages/standard-components/src/Search.svelte @@ -25,10 +25,11 @@ let tableDefinition let schema - // pagination - let page = 0 + let nextBookmark = null + let bookmark = null + let lastBookmark = null - $: fetchData(table, page) + $: fetchData(table, bookmark) // omit empty strings $: parsedSearch = Object.keys(search).reduce( (acc, next) => @@ -38,33 +39,38 @@ $: actions = [ { type: ActionTypes.RefreshDatasource, - callback: () => fetchData(table, page), + callback: () => fetchData(table, bookmark), metadata: { datasource: { type: "table", tableId: table } }, }, ] - async function fetchData(table, page) { + async function fetchData(table, mark) { if (table) { const tableDef = await API.fetchTableDefinition(table) schema = tableDef.schema - rows = await API.searchTableData({ + lastBookmark = mark + const output = await API.searchTableData({ tableId: table, search: parsedSearch, pagination: { pageSize, - page, + bookmark: mark, }, }) + rows = output.rows + nextBookmark = output.bookmark } loaded = true } function nextPage() { - page += 1 + lastBookmark = bookmark + bookmark = nextBookmark } function previousPage() { - page -= 1 + nextBookmark = bookmark + bookmark = lastBookmark } @@ -99,15 +105,15 @@ secondary on:click={() => { search = {} - page = 0 + bookmark = null }}> Reset @@ -129,7 +135,7 @@ {/if} {/if} From 3916b9a29a1a903cd6df30c0ca0aaae221e45465 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 26 Mar 2021 14:11:24 +0000 Subject: [PATCH 6/8] Some minor updates to make search test cases pass. --- packages/server/__mocks__/node-fetch.js | 11 +++++++++++ packages/server/src/api/controllers/row.js | 2 +- .../server/src/api/controllers/search/utils.js | 13 +++++++------ packages/server/src/api/routes/tests/row.spec.js | 15 ++++++--------- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/packages/server/__mocks__/node-fetch.js b/packages/server/__mocks__/node-fetch.js index 3cc412b1c6..d023802582 100644 --- a/packages/server/__mocks__/node-fetch.js +++ b/packages/server/__mocks__/node-fetch.js @@ -30,6 +30,17 @@ module.exports = async (url, opts) => { }, 404 ) + } else if (url.includes("_search")) { + return json({ + rows: [ + { + doc: { + _id: "test", + }, + }, + ], + bookmark: "test", + }) } return fetch(url, opts) } diff --git a/packages/server/src/api/controllers/row.js b/packages/server/src/api/controllers/row.js index 537fc5c850..7540cc1894 100644 --- a/packages/server/src/api/controllers/row.js +++ b/packages/server/src/api/controllers/row.js @@ -272,7 +272,7 @@ exports.search = async function(ctx) { } let searchString - if (ctx.query.raw && ctx.query.raw !== "") { + if (ctx.query && ctx.query.raw && ctx.query.raw !== "") { searchString = queryBuilder.complete(query["RAW"]) } else { // make all strings a starts with operation rather than pure equality diff --git a/packages/server/src/api/controllers/search/utils.js b/packages/server/src/api/controllers/search/utils.js index 1032755759..cb58dc2634 100644 --- a/packages/server/src/api/controllers/search/utils.js +++ b/packages/server/src/api/controllers/search/utils.js @@ -34,7 +34,6 @@ class QueryBuilder { fuzzy: {}, range: {}, equal: {}, - meta: {}, ...base, } this.limit = 50 @@ -93,16 +92,18 @@ class QueryBuilder { if (this.query.string) { build(this.query.string, (key, value) => `${key}:${value}*`) } - if (this.query.number) { - build(this.query.number, (key, value) => - value.length == null - ? `${key}:${value}` - : `${key}:[${value[0]} TO ${value[1]}]` + if (this.query.range) { + build( + this.query.range, + (key, value) => `${key}:[${value[0]} TO ${value[1]}]` ) } if (this.query.fuzzy) { build(this.query.fuzzy, (key, value) => `${key}:${value}~`) } + if (this.query.equal) { + build(this.query.equal, (key, value) => `${key}:${value}`) + } if (rawQuery) { output = output.length === 0 ? rawQuery : `&${rawQuery}` } diff --git a/packages/server/src/api/routes/tests/row.spec.js b/packages/server/src/api/routes/tests/row.spec.js index c79f648c51..6a1c309c39 100644 --- a/packages/server/src/api/routes/tests/row.spec.js +++ b/packages/server/src/api/routes/tests/row.spec.js @@ -2,6 +2,9 @@ const { outputProcessing } = require("../../../utilities/rowProcessor") const setup = require("./utilities") const { basicRow } = setup.structures +// mock the fetch for the search system +jest.mock("node-fetch") + describe("/rows", () => { let request = setup.getRequest() let config = setup.getConfig() @@ -303,25 +306,19 @@ describe("/rows", () => { describe("search", () => { it("should run a search on the table", async () => { - const row = await config.createRow() - // add another row that shouldn't be found - await config.createRow({ - ...basicRow(), - name: "Other Contact", - }) const res = await request .post(`/api/${table._id}/rows/search`) .send({ query: { name: "Test", }, - pagination: { pageSize: 25, page: 0 } + pagination: { pageSize: 25 } }) .set(config.defaultHeaders()) .expect('Content-Type', /json/) .expect(200) - expect(res.body.length).toEqual(1) - expect(res.body[0]._id).toEqual(row._id) + expect(res.body.rows.length).toEqual(1) + expect(res.body.bookmark).toBeDefined() }) }) From 1b4a4deef5d1671eeb779007bb2b7709be45695f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 26 Mar 2021 14:26:07 +0000 Subject: [PATCH 7/8] Removing unnecessary files and updating prod compose file. --- hosting/couch/Dockerfile | 3 -- hosting/couch/lucene-proxy.ini | 2 -- hosting/docker-compose.yaml | 9 ++---- hosting/lucene/Dockerfile | 51 ---------------------------------- hosting/lucene/run-lucene.sh | 15 ---------- 5 files changed, 3 insertions(+), 77 deletions(-) delete mode 100644 hosting/couch/Dockerfile delete mode 100644 hosting/couch/lucene-proxy.ini delete mode 100644 hosting/lucene/Dockerfile delete mode 100755 hosting/lucene/run-lucene.sh diff --git a/hosting/couch/Dockerfile b/hosting/couch/Dockerfile deleted file mode 100644 index 0695a686be..0000000000 --- a/hosting/couch/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM apache/couchdb - -COPY lucene-proxy.ini /usr/local/etc/couchdb/local.d/ diff --git a/hosting/couch/lucene-proxy.ini b/hosting/couch/lucene-proxy.ini deleted file mode 100644 index 3ac6d90542..0000000000 --- a/hosting/couch/lucene-proxy.ini +++ /dev/null @@ -1,2 +0,0 @@ -[httpd_global_handlers] -_fti = {couch_httpd_proxy, handle_proxy_req, <<"http://couchdb-lucene:5985">>} diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index 86269837c2..8de5e9fcdd 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -71,7 +71,6 @@ services: - ./envoy.yaml:/etc/envoy/envoy.yaml ports: - "${MAIN_PORT}:10000" - #- "9901:9901" depends_on: - minio-service - worker-service @@ -80,16 +79,14 @@ services: couchdb-service: restart: always - image: apache/couchdb:3.0 + image: ibmcom/couchdb3 environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} ports: - "${COUCH_DB_PORT}:5984" - #- "4369:4369" - #- "9100:9100" volumes: - - couchdb_data:/opt/couchdb/data + - couchdb3_data:/opt/couchdb/data couch-init: image: curlimages/curl @@ -108,7 +105,7 @@ services: - redis_data:/data volumes: - couchdb_data: + couchdb3_data: driver: local minio_data: driver: local diff --git a/hosting/lucene/Dockerfile b/hosting/lucene/Dockerfile deleted file mode 100644 index 135237370b..0000000000 --- a/hosting/lucene/Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -FROM openjdk:8 - -RUN groupadd -r couchdb && useradd -d /opt/couchdb-lucene -g couchdb couchdb - -# grab gosu for easy step-down from root and tini for signal handling -RUN gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \ - && curl -o /usr/local/bin/gosu -fSL "https://github.com/tianon/gosu/releases/download/1.7/gosu-$(dpkg --print-architecture)" \ - && curl -o /usr/local/bin/gosu.asc -fSL "https://github.com/tianon/gosu/releases/download/1.7/gosu-$(dpkg --print-architecture).asc" \ - && gpg --verify /usr/local/bin/gosu.asc \ - && rm /usr/local/bin/gosu.asc \ - && chmod +x /usr/local/bin/gosu - -ENV COUCHDB_LUCENE_VERSION 2.1.0 - -RUN apt-get update \ - && apt-get install -y maven \ - && cd /usr/src \ - && curl -L https://github.com/rnewson/couchdb-lucene/archive/v$COUCHDB_LUCENE_VERSION.tar.gz | tar -xz \ - && cd couchdb-lucene-$COUCHDB_LUCENE_VERSION \ - && mvn - -RUN cd /usr/src/couchdb-lucene-$COUCHDB_LUCENE_VERSION/target \ - && unzip couchdb-lucene-$COUCHDB_LUCENE_VERSION-dist.zip \ - && mv couchdb-lucene-$COUCHDB_LUCENE_VERSION /opt/couchdb-lucene \ - && rm -rf /usr/src/couchdb-lucene-* - -RUN apt-get remove --auto-remove -y maven \ - && rm -rf /var/lib/apt/lists/* \ - && sed -e 's/^host=localhost$/host=0.0.0.0/' -i /opt/couchdb-lucene/conf/couchdb-lucene.ini \ - && sed -e 's/localhost:5984/couchdb:5984/' -i /opt/couchdb-lucene/conf/couchdb-lucene.ini \ - && chown -R couchdb:couchdb /opt/couchdb-lucene - -COPY ./run-lucene.sh /opt/couchdb-lucene/run-lucene.sh -RUN chmod +x /opt/couchdb-lucene/run-lucene.sh - -WORKDIR /opt/couchdb-lucene -EXPOSE 5985 -VOLUME ["/opt/couchdb-lucene/indexes"] -CMD ["./run-lucene.sh"] diff --git a/hosting/lucene/run-lucene.sh b/hosting/lucene/run-lucene.sh deleted file mode 100755 index 945bb5655c..0000000000 --- a/hosting/lucene/run-lucene.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -chown -R couchdb:couchdb /opt/couchdb-lucene -exec gosu couchdb ./bin/run From 357cd8cfbd4abcb365022a532c87bf1f73084fa3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 26 Mar 2021 14:46:29 +0000 Subject: [PATCH 8/8] Quick change after reviewing. --- packages/server/src/api/controllers/search/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/search/utils.js b/packages/server/src/api/controllers/search/utils.js index cb58dc2634..d3ffb26be7 100644 --- a/packages/server/src/api/controllers/search/utils.js +++ b/packages/server/src/api/controllers/search/utils.js @@ -95,7 +95,7 @@ class QueryBuilder { if (this.query.range) { build( this.query.range, - (key, value) => `${key}:[${value[0]} TO ${value[1]}]` + (key, value) => `${key}:[${value.low} TO ${value.high}]` ) } if (this.query.fuzzy) {