From 4b7ffbbce7a1891df12ab869672320b88289a945 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 1 Apr 2022 01:00:52 +0100 Subject: [PATCH 1/5] Single docker container configuration. --- .dockerignore | 9 +++ hosting/single/Dockerfile | 92 +++++++++++++++++++++++ hosting/single/clouseau | 12 +++ hosting/single/clouseau.ini | 13 ++++ hosting/single/log4j.properties | 4 + hosting/single/nginx.conf | 116 +++++++++++++++++++++++++++++ hosting/single/runner.sh | 16 ++++ hosting/single/vm.args | 32 ++++++++ package.json | 1 + packages/server/src/app.ts | 2 +- packages/server/src/environment.js | 2 +- packages/worker/src/environment.js | 2 +- 12 files changed, 298 insertions(+), 3 deletions(-) create mode 100644 .dockerignore create mode 100644 hosting/single/Dockerfile create mode 100644 hosting/single/clouseau create mode 100644 hosting/single/clouseau.ini create mode 100644 hosting/single/log4j.properties create mode 100644 hosting/single/nginx.conf create mode 100644 hosting/single/runner.sh create mode 100644 hosting/single/vm.args diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..92bd33894e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +packages/server/node_modules +packages/builder +packages/frontend-core +packages/backend-core +packages/worker/node_modules +packages/cli +packages/client +packages/bbui +packages/string-templates diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile new file mode 100644 index 0000000000..e1a5acacc2 --- /dev/null +++ b/hosting/single/Dockerfile @@ -0,0 +1,92 @@ +FROM couchdb + +ENV COUCHDB_PASSWORD=budibase +ENV COUCHDB_USER=budibase +ENV COUCH_DB_URL=http://budibase:budibase@localhost:5984 +ENV BUDIBASE_ENVIRONMENT=PRODUCTION +ENV MINIO_URL=http://localhost:10000/ +ENV REDIS_URL=localhost:6379 +ENV WORKER_URL=http://localhost:4002 +ENV INTERNAL_API_KEY=budibase +ENV JWT_SECRET=testsecret +ENV MINIO_ACCESS_KEY=budibase +ENV MINIO_SECRET_KEY=budibase +ENV SELF_HOSTED=1 +ENV CLUSTER_PORT=10000 +ENV REDIS_PASSWORD=budibase +ENV ARCHITECTURE=amd +ENV SERVER_PORT=4001 +ENV WORKER_PORT=4002 + +RUN apt-get update +RUN apt-get install software-properties-common wget nginx -y +RUN apt-add-repository 'deb http://security.debian.org/debian-security stretch/updates main' +RUN apt-get update + +# install java +RUN apt-get install openjdk-8-jdk -y + +# setup nodejs +WORKDIR /nodejs +RUN curl -sL https://deb.nodesource.com/setup_16.x -o /tmp/nodesource_setup.sh +RUN bash /tmp/nodesource_setup.sh +RUN apt-get install nodejs +RUN npm install --global yarn +RUN npm install --global pm2 + +# setup redis +RUN apt install redis-server -y + +# setup server +WORKDIR /app +ADD packages/server . +RUN ls -al +RUN yarn +RUN yarn build +# Install client for oracle datasource +RUN apt-get install unzip libaio1 +RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh + +# setup worker +WORKDIR /worker +ADD packages/worker . +RUN yarn +RUN yarn build + +# setup clouseau +WORKDIR / +RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clouseau-2.21.0-dist.zip +RUN unzip clouseau-2.21.0-dist.zip +RUN mv clouseau-2.21.0 /opt/clouseau +RUN rm clouseau-2.21.0-dist.zip + +WORKDIR /opt/clouseau +RUN mkdir ./bin +ADD hosting/single/clouseau ./bin/ +ADD hosting/single/log4j.properties . +ADD hosting/single/clouseau.ini . +RUN chmod +x ./bin/clouseau + +# setup CouchDB +WORKDIR /opt/couchdb +ADD hosting/single/vm.args ./etc/ + +# setup minio +WORKDIR /minio +RUN wget https://dl.min.io/server/minio/release/linux-${ARCHITECTURE}64/minio +RUN chmod +x minio + +WORKDIR / +ADD hosting/single/runner.sh . +RUN chmod +x ./runner.sh +ADD hosting/single/nginx.conf /etc/nginx +RUN mkdir /etc/nginx/logs +RUN useradd www +RUN touch /etc/nginx/logs/error.log +RUN touch /etc/nginx/logs/nginx.pid + +EXPOSE 10000 + +# must set this just before running +ENV NODE_ENV=production +CMD ["./runner.sh"] diff --git a/hosting/single/clouseau b/hosting/single/clouseau new file mode 100644 index 0000000000..1095ea24cb --- /dev/null +++ b/hosting/single/clouseau @@ -0,0 +1,12 @@ +#!/bin/sh +/usr/bin/java -server \ + -Xmx2G \ + -Dsun.net.inetaddr.ttl=30 \ + -Dsun.net.inetaddr.negative.ttl=30 \ + -Dlog4j.configuration=file:/opt/clouseau/log4j.properties \ + -XX:OnOutOfMemoryError="kill -9 %p" \ + -XX:+UseConcMarkSweepGC \ + -XX:+CMSParallelRemarkEnabled \ + -classpath '/opt/clouseau/*' \ + com.cloudant.clouseau.Main \ + /opt/clouseau/clouseau.ini \ No newline at end of file diff --git a/hosting/single/clouseau.ini b/hosting/single/clouseau.ini new file mode 100644 index 0000000000..f086cf0398 --- /dev/null +++ b/hosting/single/clouseau.ini @@ -0,0 +1,13 @@ +[clouseau] + +; the name of the Erlang node created by the service, leave this unchanged +name=clouseau@127.0.0.1 + +; set this to the same distributed Erlang cookie used by the CouchDB nodes +cookie=monster + +; the path where you would like to store the search index files +dir=/opt/couchdb/data/search + +; the number of search indexes that can be open simultaneously +max_indexes_open=500 diff --git a/hosting/single/log4j.properties b/hosting/single/log4j.properties new file mode 100644 index 0000000000..9d4d9311bc --- /dev/null +++ b/hosting/single/log4j.properties @@ -0,0 +1,4 @@ +log4j.rootLogger=debug, CONSOLE +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} %c [%p] %m%n \ No newline at end of file diff --git a/hosting/single/nginx.conf b/hosting/single/nginx.conf new file mode 100644 index 0000000000..86938ced4e --- /dev/null +++ b/hosting/single/nginx.conf @@ -0,0 +1,116 @@ +user www www; +error_log /etc/nginx/logs/error.log; +pid /etc/nginx/logs/nginx.pid; +worker_processes auto; +worker_rlimit_nofile 8192; + +events { + worker_connections 1024; +} + +http { + limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=20r/s; + proxy_set_header Host $host; + charset utf-8; + sendfile on; + tcp_nopush on; + tcp_nodelay on; + server_tokens off; + types_hash_max_size 2048; + + # buffering + client_header_buffer_size 1k; + client_max_body_size 20M; + ignore_invalid_headers off; + proxy_buffering off; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + map $http_upgrade $connection_upgrade { + default "upgrade"; + } + + server { + listen 10000 default_server; + listen [::]:10000 default_server; + server_name _; + client_max_body_size 1000m; + ignore_invalid_headers off; + proxy_buffering off; + # port_in_redirect off; + + location /app { + proxy_pass http://127.0.0.1:4001; + } + + location = / { + proxy_pass http://127.0.0.1:4001; + } + + location ~ ^/(builder|app_) { + proxy_http_version 1.1; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_pass http://127.0.0.1:4001; + } + + location ~ ^/api/(system|admin|global)/ { + proxy_pass http://127.0.0.1:4002; + } + + location /worker/ { + proxy_pass http://127.0.0.1:4002; + rewrite ^/worker/(.*)$ /$1 break; + } + + location /api/ { + # calls to the API are rate limited with bursting + limit_req zone=ratelimit burst=20 nodelay; + + # 120s timeout on API requests + proxy_read_timeout 120s; + proxy_connect_timeout 120s; + proxy_send_timeout 120s; + + proxy_http_version 1.1; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_pass http://127.0.0.1:4001; + } + + location /db/ { + proxy_pass http://127.0.0.1:5984; + rewrite ^/db/(.*)$ /$1 break; + } + + location / { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + proxy_pass http://127.0.0.1:9000; + } + + client_header_timeout 60; + client_body_timeout 60; + keepalive_timeout 60; + + # gzip + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml; + } +} diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh new file mode 100644 index 0000000000..259fb138fa --- /dev/null +++ b/hosting/single/runner.sh @@ -0,0 +1,16 @@ +redis-server --requirepass $REDIS_PASSWORD & +/opt/clouseau/bin/clouseau & +/minio/minio server /data & +/docker-entrypoint.sh /opt/couchdb/bin/couchdb & +/etc/init.d/nginx restart +pushd app +pm2 start --name app "yarn run:docker" +popd +pushd worker +pm2 start --name worker "yarn run:docker" +popd +sleep 10 +URL=http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984 +curl -X PUT ${URL}/_users +curl -X PUT ${URL}/_replicator +sleep infinity \ No newline at end of file diff --git a/hosting/single/vm.args b/hosting/single/vm.args new file mode 100644 index 0000000000..e9e4416863 --- /dev/null +++ b/hosting/single/vm.args @@ -0,0 +1,32 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +# erlang cookie for clouseau security +-name couchdb@127.0.0.1 +-setcookie monster + +# Ensure that the Erlang VM listens on a known port +-kernel inet_dist_listen_min 9100 +-kernel inet_dist_listen_max 9100 + +# Tell kernel and SASL not to log anything +-kernel error_logger silent +-sasl sasl_error_logger false + +# Use kernel poll functionality if supported by emulator ++K true + +# Start a pool of asynchronous IO threads ++A 16 + +# Comment this line out to enable the interactive Erlang shell on startup ++Bd -noinput diff --git a/package.json b/package.json index 88aa681ad3..07db944ecd 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "build:docker:develop": "node scripts/pinVersions && lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild", "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -", + "build:docker:single": "docker build -f hosting/single/Dockerfile -t budibase:latest .", "build:docs": "lerna run build:docs", "release:helm": "node scripts/releaseHelmChart", "env:multi:enable": "lerna run env:multi:enable", diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 0c0ef68ad9..5ee4e8772f 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -76,7 +76,7 @@ server.on("close", async () => { await redis.shutdown() }) -module.exports = server.listen(env.PORT || 0, async () => { +module.exports = server.listen(env.PORT || 4001, async () => { console.log(`Budibase running on ${JSON.stringify(server.address())}`) env._set("PORT", server.address().port) eventEmitter.emitPort(env.PORT) diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js index bdbbbde79f..ac3a539a12 100644 --- a/packages/server/src/environment.js +++ b/packages/server/src/environment.js @@ -28,7 +28,7 @@ let inThread = false module.exports = { // important - PORT: process.env.PORT, + PORT: process.env.PORT || process.env.APP_PORT, JWT_SECRET: process.env.JWT_SECRET, COUCH_DB_URL: process.env.COUCH_DB_URL, MINIO_URL: process.env.MINIO_URL, diff --git a/packages/worker/src/environment.js b/packages/worker/src/environment.js index 91f06ea46d..c965863a54 100644 --- a/packages/worker/src/environment.js +++ b/packages/worker/src/environment.js @@ -19,7 +19,7 @@ if (!LOADED && isDev() && !isTest()) { module.exports = { NODE_ENV: process.env.NODE_ENV, SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED), - PORT: process.env.PORT, + PORT: process.env.PORT || process.env.WORKER_PORT, CLUSTER_PORT: process.env.CLUSTER_PORT, MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, From a1f4a237f0da52cc9e66d972faea95ce457caf4b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 1 Apr 2022 12:37:01 +0100 Subject: [PATCH 2/5] Minor changes to get BB client and manifest included correctly. --- hosting/docker-compose.yaml | 1 - hosting/single/Dockerfile | 2 +- package.json | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index f3055b19fa..f9d9eaf1c5 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -117,7 +117,6 @@ services: labels: - "com.centurylinklabs.watchtower.enable=false" - volumes: couchdb3_data: driver: local diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index e1a5acacc2..62f819e23c 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -4,7 +4,7 @@ ENV COUCHDB_PASSWORD=budibase ENV COUCHDB_USER=budibase ENV COUCH_DB_URL=http://budibase:budibase@localhost:5984 ENV BUDIBASE_ENVIRONMENT=PRODUCTION -ENV MINIO_URL=http://localhost:10000/ +ENV MINIO_URL=http://localhost:9000 ENV REDIS_URL=localhost:6379 ENV WORKER_URL=http://localhost:4002 ENV INTERNAL_API_KEY=budibase diff --git a/package.json b/package.json index 07db944ecd..f7efcef8ea 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "build:docker:develop": "node scripts/pinVersions && lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild", "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -", - "build:docker:single": "docker build -f hosting/single/Dockerfile -t budibase:latest .", + "build:docker:single": "lerna run build && lerna run predocker && docker build -f hosting/single/Dockerfile -t budibase:latest .", "build:docs": "lerna run build:docs", "release:helm": "node scripts/releaseHelmChart", "env:multi:enable": "lerna run env:multi:enable", From 28e47a5fa851a7d3daa7aabc601ee48634d118ae Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 1 Apr 2022 12:50:26 +0100 Subject: [PATCH 3/5] Adding volumes. --- hosting/single/Dockerfile | 2 ++ hosting/single/runner.sh | 2 +- package.json | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 62f819e23c..eb959c5e21 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -86,6 +86,8 @@ RUN touch /etc/nginx/logs/error.log RUN touch /etc/nginx/logs/nginx.pid EXPOSE 10000 +VOLUME /opt/couchdb/data +VOLUME /minio # must set this just before running ENV NODE_ENV=production diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index 259fb138fa..fab8431796 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -1,6 +1,6 @@ redis-server --requirepass $REDIS_PASSWORD & /opt/clouseau/bin/clouseau & -/minio/minio server /data & +/minio/minio server /minio & /docker-entrypoint.sh /opt/couchdb/bin/couchdb & /etc/init.d/nginx restart pushd app diff --git a/package.json b/package.json index f7efcef8ea..0a52900ca3 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,8 @@ "build:docker:develop": "node scripts/pinVersions && lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild", "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -", - "build:docker:single": "lerna run build && lerna run predocker && docker build -f hosting/single/Dockerfile -t budibase:latest .", + "build:docker:single:image": "docker build -f hosting/single/Dockerfile -t budibase:latest .", + "build:docker:single": "lerna run build && lerna run predocker && npm run build:docker:single:image", "build:docs": "lerna run build:docs", "release:helm": "node scripts/releaseHelmChart", "env:multi:enable": "lerna run env:multi:enable", From 05af8cafc2f05a8849c1ea94ef63059015a15121 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 1 Apr 2022 13:47:38 +0100 Subject: [PATCH 4/5] Fixing test case. --- hosting/single/Dockerfile | 2 +- packages/server/src/app.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index eb959c5e21..208416eb8c 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -15,7 +15,7 @@ ENV SELF_HOSTED=1 ENV CLUSTER_PORT=10000 ENV REDIS_PASSWORD=budibase ENV ARCHITECTURE=amd -ENV SERVER_PORT=4001 +ENV APP_PORT=4001 ENV WORKER_PORT=4002 RUN apt-get update diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 5ee4e8772f..0c0ef68ad9 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -76,7 +76,7 @@ server.on("close", async () => { await redis.shutdown() }) -module.exports = server.listen(env.PORT || 4001, async () => { +module.exports = server.listen(env.PORT || 0, async () => { console.log(`Budibase running on ${JSON.stringify(server.address())}`) env._set("PORT", server.address().port) eventEmitter.emitPort(env.PORT) From 6dd50eb25addae4aec01af28230299c9d4952ab4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 5 Apr 2022 11:33:55 +0100 Subject: [PATCH 5/5] Review comments. --- hosting/single/Dockerfile | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 208416eb8c..2123d237b5 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -23,6 +23,13 @@ RUN apt-get install software-properties-common wget nginx -y RUN apt-add-repository 'deb http://security.debian.org/debian-security stretch/updates main' RUN apt-get update +# setup nginx +ADD hosting/single/nginx.conf /etc/nginx +RUN mkdir /etc/nginx/logs +RUN useradd www +RUN touch /etc/nginx/logs/error.log +RUN touch /etc/nginx/logs/nginx.pid + # install java RUN apt-get install openjdk-8-jdk -y @@ -76,14 +83,10 @@ WORKDIR /minio RUN wget https://dl.min.io/server/minio/release/linux-${ARCHITECTURE}64/minio RUN chmod +x minio +# setup runner file WORKDIR / ADD hosting/single/runner.sh . RUN chmod +x ./runner.sh -ADD hosting/single/nginx.conf /etc/nginx -RUN mkdir /etc/nginx/logs -RUN useradd www -RUN touch /etc/nginx/logs/error.log -RUN touch /etc/nginx/logs/nginx.pid EXPOSE 10000 VOLUME /opt/couchdb/data