From dfa2881f1c60caa0540e1e69522291814e225201 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 25 Mar 2021 18:04:44 +0000 Subject: [PATCH] Initialising CouchDB link to lucene and app design DBs for it. --- hosting/couch/Dockerfile | 3 ++ hosting/couch/lucene-proxy.ini | 2 + hosting/docker-compose.dev.yaml | 31 +++++++++-- hosting/hosting.properties | 1 + hosting/lucene/Dockerfile | 51 +++++++++++++++++++ hosting/lucene/run-lucene.sh | 15 ++++++ .../server/src/api/controllers/application.js | 8 ++- packages/server/src/db/views/staticViews.js | 36 +++++++++++++ 8 files changed, 141 insertions(+), 6 deletions(-) create mode 100644 hosting/couch/Dockerfile create mode 100644 hosting/couch/lucene-proxy.ini create mode 100644 hosting/lucene/Dockerfile create mode 100755 hosting/lucene/run-lucene.sh diff --git a/hosting/couch/Dockerfile b/hosting/couch/Dockerfile new file mode 100644 index 0000000000..0695a686be --- /dev/null +++ b/hosting/couch/Dockerfile @@ -0,0 +1,3 @@ +FROM apache/couchdb + +COPY lucene-proxy.ini /usr/local/etc/couchdb/local.d/ diff --git a/hosting/couch/lucene-proxy.ini b/hosting/couch/lucene-proxy.ini new file mode 100644 index 0000000000..3ac6d90542 --- /dev/null +++ b/hosting/couch/lucene-proxy.ini @@ -0,0 +1,2 @@ +[httpd_global_handlers] +_fti = {couch_httpd_proxy, handle_proxy_req, <<"http://couchdb-lucene:5985">>} diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index 39fcb7ec83..2cbba5771f 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -30,26 +30,44 @@ services: - ./envoy.dev.yaml:/etc/envoy/envoy.yaml ports: - "${MAIN_PORT}:10000" - #- "9901:9901" depends_on: - minio-service - couchdb-service + couchdb-lucene: + container_name: budi-couchdb-lucene-dev + build: + context: lucene + dockerfile: Dockerfile + ports: + - "${COUCH_LUCENE_PORT}:5985" + volumes: + - couchdb_lucene:/opt/couchdb-lucene + networks: + dbs: + aliases: + - couchdb-lucene + couchdb-service: container_name: budi-couchdb-dev restart: always - image: apache/couchdb:3.0 + build: + context: couch + dockerfile: Dockerfile environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} ports: - "${COUCH_DB_PORT}:5984" - #- "4369:4369" - #- "9100:9100" volumes: - couchdb_data:/opt/couchdb/data +# networks: +# dbs: +# aliases: +# - couchdb couch-init: + container_name: budi-couchdb-init-dev image: curlimages/curl environment: PUT_CALL: "curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984" @@ -66,6 +84,9 @@ services: volumes: - redis_data:/data +networks: + dbs: + driver: bridge volumes: couchdb_data: @@ -74,3 +95,5 @@ volumes: driver: local redis_data: driver: local + couchdb_lucene: + driver: local diff --git a/hosting/hosting.properties b/hosting/hosting.properties index 138e66d629..5b80a844dd 100644 --- a/hosting/hosting.properties +++ b/hosting/hosting.properties @@ -18,5 +18,6 @@ APP_PORT=4002 WORKER_PORT=4003 MINIO_PORT=4004 COUCH_DB_PORT=4005 +COUCH_LUCENE_PORT=4006 REDIS_PORT=6379 BUDIBASE_ENVIRONMENT=PRODUCTION diff --git a/hosting/lucene/Dockerfile b/hosting/lucene/Dockerfile new file mode 100644 index 0000000000..135237370b --- /dev/null +++ b/hosting/lucene/Dockerfile @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +FROM openjdk:8 + +RUN groupadd -r couchdb && useradd -d /opt/couchdb-lucene -g couchdb couchdb + +# grab gosu for easy step-down from root and tini for signal handling +RUN gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \ + && curl -o /usr/local/bin/gosu -fSL "https://github.com/tianon/gosu/releases/download/1.7/gosu-$(dpkg --print-architecture)" \ + && curl -o /usr/local/bin/gosu.asc -fSL "https://github.com/tianon/gosu/releases/download/1.7/gosu-$(dpkg --print-architecture).asc" \ + && gpg --verify /usr/local/bin/gosu.asc \ + && rm /usr/local/bin/gosu.asc \ + && chmod +x /usr/local/bin/gosu + +ENV COUCHDB_LUCENE_VERSION 2.1.0 + +RUN apt-get update \ + && apt-get install -y maven \ + && cd /usr/src \ + && curl -L https://github.com/rnewson/couchdb-lucene/archive/v$COUCHDB_LUCENE_VERSION.tar.gz | tar -xz \ + && cd couchdb-lucene-$COUCHDB_LUCENE_VERSION \ + && mvn + +RUN cd /usr/src/couchdb-lucene-$COUCHDB_LUCENE_VERSION/target \ + && unzip couchdb-lucene-$COUCHDB_LUCENE_VERSION-dist.zip \ + && mv couchdb-lucene-$COUCHDB_LUCENE_VERSION /opt/couchdb-lucene \ + && rm -rf /usr/src/couchdb-lucene-* + +RUN apt-get remove --auto-remove -y maven \ + && rm -rf /var/lib/apt/lists/* \ + && sed -e 's/^host=localhost$/host=0.0.0.0/' -i /opt/couchdb-lucene/conf/couchdb-lucene.ini \ + && sed -e 's/localhost:5984/couchdb:5984/' -i /opt/couchdb-lucene/conf/couchdb-lucene.ini \ + && chown -R couchdb:couchdb /opt/couchdb-lucene + +COPY ./run-lucene.sh /opt/couchdb-lucene/run-lucene.sh +RUN chmod +x /opt/couchdb-lucene/run-lucene.sh + +WORKDIR /opt/couchdb-lucene +EXPOSE 5985 +VOLUME ["/opt/couchdb-lucene/indexes"] +CMD ["./run-lucene.sh"] diff --git a/hosting/lucene/run-lucene.sh b/hosting/lucene/run-lucene.sh new file mode 100755 index 0000000000..945bb5655c --- /dev/null +++ b/hosting/lucene/run-lucene.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +chown -R couchdb:couchdb /opt/couchdb-lucene +exec gosu couchdb ./bin/run diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index c5d48038ce..59cdb58904 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -2,8 +2,11 @@ const CouchDB = require("../../db") const env = require("../../environment") const setBuilderToken = require("../../utilities/builder/setBuilderToken") const packageJson = require("../../../package.json") -const { createLinkView } = require("../../db/linkedRows") -const { createRoutingView } = require("../../utilities/routing") +const { + createLinkView, + createRoutingView, + createFulltextSearchIndex, +} = require("../../db/views/staticViews") const { getTemplateStream, createApp, @@ -92,6 +95,7 @@ async function createInstance(template) { // add view for linked rows await createLinkView(appId) await createRoutingView(appId) + await createFulltextSearchIndex(appId) // replicate the template data to the instance DB // this is currently very hard to test, downloading and importing template files diff --git a/packages/server/src/db/views/staticViews.js b/packages/server/src/db/views/staticViews.js index a0d0fbb239..50e3a4a1a9 100644 --- a/packages/server/src/db/views/staticViews.js +++ b/packages/server/src/db/views/staticViews.js @@ -72,3 +72,39 @@ exports.createRoutingView = async appId => { } await db.put(designDoc) } + +exports.createFulltextSearchIndex = async appId => { + const db = new CouchDB(appId) + const designDoc = await db.get("_design/database") + designDoc.fulltext = { + everything: { + index: function(doc) { + let ret = new Document() + + function idx(obj) { + for (let key of Object.keys(obj)) { + switch (typeof obj[key]) { + case "object": + idx(obj[key]) + break + case "function": + break + default: + ret.add(obj[key]) + break + } + } + } + + idx(doc) + if (doc._attachments) { + for (let i in Object.keys(doc._attachments)) { + ret.attachment("default", i) + } + } + return ret + }.toString(), + }, + } + await db.put(designDoc) +}