diff --git a/hosting/couchdb/Dockerfile b/hosting/couchdb/Dockerfile index f95059e461..ca72153e78 100644 --- a/hosting/couchdb/Dockerfile +++ b/hosting/couchdb/Dockerfile @@ -100,7 +100,6 @@ FROM base as runner ENV COUCHDB_USER admin ENV COUCHDB_PASSWORD admin EXPOSE 5984 -EXPOSE 4984 RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \ wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - && \ @@ -126,10 +125,7 @@ ADD clouseau/log4j.properties clouseau/clouseau.ini ./ WORKDIR /opt/couchdb ADD couch/vm.args couch/local.ini ./etc/ -WORKDIR /opt/sqs -ADD sqs/sqs sqs/better_sqlite3.node ./ - WORKDIR / ADD runner.sh ./bbcouch-runner.sh -RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau /opt/sqs/sqs -CMD ["./bbcouch-runner.sh"] +RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau +CMD ["./bbcouch-runner.sh"] \ No newline at end of file diff --git a/hosting/couchdb/Dockerfile.v2 b/hosting/couchdb/Dockerfile.v2 new file mode 100644 index 0000000000..912344a903 --- /dev/null +++ b/hosting/couchdb/Dockerfile.v2 @@ -0,0 +1,135 @@ +# Modified from https://github.com/apache/couchdb-docker/blob/main/3.3.3/Dockerfile +# +# Everything in this `base` image is adapted from the official `couchdb` image's +# Dockerfile. Only modifications related to upgrading from Debian bullseye to +# bookworm have been included. The `runner` image contains Budibase's +# customisations to the image, e.g. adding Clouseau. +FROM node:20-slim AS base + +# Add CouchDB user account to make sure the IDs are assigned consistently +RUN groupadd -g 5984 -r couchdb && useradd -u 5984 -d /opt/couchdb -g couchdb couchdb + +# be sure GPG and apt-transport-https are available and functional +RUN set -ex; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + dirmngr \ + gnupg \ + ; \ + rm -rf /var/lib/apt/lists/* + +# grab tini for signal handling and zombie reaping +# see https://github.com/apache/couchdb-docker/pull/28#discussion_r141112407 +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends tini; \ + rm -rf /var/lib/apt/lists/*; \ + tini --version + +# http://docs.couchdb.org/en/latest/install/unix.html#installing-the-apache-couchdb-packages +ENV GPG_COUCH_KEY \ +# gpg: rsa8192 205-01-19 The Apache Software Foundation (Package repository signing key) + 390EF70BB1EA12B2773962950EE62FB37A00258D +RUN set -eux; \ + apt-get update; \ + apt-get install -y curl; \ + export GNUPGHOME="$(mktemp -d)"; \ + curl -fL -o keys.asc https://couchdb.apache.org/repo/keys.asc; \ + gpg --batch --import keys.asc; \ + gpg --batch --export "${GPG_COUCH_KEY}" > /usr/share/keyrings/couchdb-archive-keyring.gpg; \ + command -v gpgconf && gpgconf --kill all || :; \ + rm -rf "$GNUPGHOME"; \ + apt-key list; \ + apt purge -y --autoremove curl; \ + rm -rf /var/lib/apt/lists/* + +ENV COUCHDB_VERSION 3.3.3 + +RUN . /etc/os-release; \ + echo "deb [signed-by=/usr/share/keyrings/couchdb-archive-keyring.gpg] https://apache.jfrog.io/artifactory/couchdb-deb/ ${VERSION_CODENAME} main" | \ + tee /etc/apt/sources.list.d/couchdb.list >/dev/null + +# https://github.com/apache/couchdb-pkg/blob/master/debian/README.Debian +RUN set -eux; \ + apt-get update; \ + \ + echo "couchdb couchdb/mode select none" | debconf-set-selections; \ +# we DO want recommends this time + DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \ + couchdb="$COUCHDB_VERSION"~bookworm \ + ; \ +# Undo symlinks to /var/log and /var/lib + rmdir /var/lib/couchdb /var/log/couchdb; \ + rm /opt/couchdb/data /opt/couchdb/var/log; \ + mkdir -p /opt/couchdb/data /opt/couchdb/var/log; \ + chown couchdb:couchdb /opt/couchdb/data /opt/couchdb/var/log; \ + chmod 777 /opt/couchdb/data /opt/couchdb/var/log; \ +# Remove file that sets logging to a file + rm /opt/couchdb/etc/default.d/10-filelog.ini; \ +# Check we own everything in /opt/couchdb. Matches the command in dockerfile_entrypoint.sh + find /opt/couchdb \! \( -user couchdb -group couchdb \) -exec chown -f couchdb:couchdb '{}' +; \ +# Setup directories and permissions for config. Technically these could be 555 and 444 respectively +# but we keep them as 755 and 644 for consistency with CouchDB defaults and the dockerfile_entrypoint.sh. + find /opt/couchdb/etc -type d ! -perm 0755 -exec chmod -f 0755 '{}' +; \ + find /opt/couchdb/etc -type f ! -perm 0644 -exec chmod -f 0644 '{}' +; \ +# only local.d needs to be writable for the docker_entrypoint.sh + chmod -f 0777 /opt/couchdb/etc/local.d; \ +# apt clean-up + rm -rf /var/lib/apt/lists/*; + +# Add configuration +COPY --chown=couchdb:couchdb couch/10-docker-default.ini /opt/couchdb/etc/default.d/ +# COPY --chown=couchdb:couchdb vm.args /opt/couchdb/etc/ + +COPY docker-entrypoint.sh /usr/local/bin +RUN ln -s usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh # backwards compat +ENTRYPOINT ["tini", "--", "/docker-entrypoint.sh"] + +VOLUME /opt/couchdb/data + +# 5984: Main CouchDB endpoint +# 4369: Erlang portmap daemon (epmd) +# 9100: CouchDB cluster communication port +EXPOSE 5984 4369 9100 +CMD ["/opt/couchdb/bin/couchdb"] + +FROM base as runner + +ENV COUCHDB_USER admin +ENV COUCHDB_PASSWORD admin +EXPOSE 5984 +EXPOSE 4984 + +RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \ + wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - && \ + apt-add-repository 'deb http://security.debian.org/debian-security bookworm-security/updates main' && \ + apt-add-repository 'deb http://archive.debian.org/debian stretch-backports main' && \ + apt-add-repository 'deb https://packages.adoptium.net/artifactory/deb bookworm main' && \ + apt-get update && apt-get install -y --no-install-recommends temurin-8-jdk && \ + rm -rf /var/lib/apt/lists/ + +# setup clouseau +WORKDIR / +RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clouseau-2.21.0-dist.zip && \ + unzip clouseau-2.21.0-dist.zip && \ + mv clouseau-2.21.0 /opt/clouseau && \ + rm clouseau-2.21.0-dist.zip + +WORKDIR /opt/clouseau +RUN mkdir ./bin +ADD clouseau/clouseau ./bin/ +ADD clouseau/log4j.properties clouseau/clouseau.ini ./ + +# setup CouchDB +WORKDIR /opt/couchdb +ADD couch/vm.args couch/local.ini ./etc/ + +WORKDIR /opt/sqs +ADD sqs/sqs sqs/better_sqlite3.node ./ + +WORKDIR / +ADD runner.v2.sh ./bbcouch-runner.sh +RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau /opt/sqs/sqs +CMD ["./bbcouch-runner.sh"] diff --git a/hosting/couchdb/runner.sh b/hosting/couchdb/runner.sh index 7ee24327a1..aaadee6b43 100644 --- a/hosting/couchdb/runner.sh +++ b/hosting/couchdb/runner.sh @@ -72,10 +72,7 @@ sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/clouseau/clouse # Start CouchDB. /docker-entrypoint.sh /opt/couchdb/bin/couchdb & -# Start SQS. -/opt/sqs/sqs --server "http://localhost:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 & - -# Wait for CouchDB to start up. +# Wati for CouchDB to start up. while [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; do echo 'Waiting for CouchDB to start...'; sleep 5; @@ -85,4 +82,4 @@ done # function correctly, so we create them here. curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_users curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_replicator -sleep infinity +sleep infinity \ No newline at end of file diff --git a/hosting/couchdb/runner.v2.sh b/hosting/couchdb/runner.v2.sh new file mode 100644 index 0000000000..7ee24327a1 --- /dev/null +++ b/hosting/couchdb/runner.v2.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +DATA_DIR=${DATA_DIR:-/data} +COUCHDB_ERLANG_COOKIE=${COUCHDB_ERLANG_COOKIE:-B9CFC32C-3458-4A86-8448-B3C753991CA7} + +mkdir -p ${DATA_DIR} +mkdir -p ${DATA_DIR}/couch/{dbs,views} +mkdir -p ${DATA_DIR}/search +chown -R couchdb:couchdb ${DATA_DIR}/couch + +echo ${TARGETBUILD} > /buildtarget.txt +if [[ "${TARGETBUILD}" = "aas" ]]; then + # Azure AppService uses /home for persistent data & SSH on port 2222 + DATA_DIR="${DATA_DIR:-/home}" + WEBSITES_ENABLE_APP_SERVICE_STORAGE=true + mkdir -p $DATA_DIR/{search,minio,couch} + mkdir -p $DATA_DIR/couch/{dbs,views} + chown -R couchdb:couchdb $DATA_DIR/couch/ + apt update + apt-get install -y openssh-server + echo "root:Docker!" | chpasswd + mkdir -p /tmp + chmod +x /tmp/ssh_setup.sh \ + && (sleep 1;/tmp/ssh_setup.sh 2>&1 > /dev/null) + cp /etc/sshd_config /etc/ssh/sshd_config + /etc/init.d/ssh restart + sed -i "s#DATA_DIR#/home#g" /opt/clouseau/clouseau.ini + sed -i "s#DATA_DIR#/home#g" /opt/couchdb/etc/local.ini +elif [[ "${TARGETBUILD}" = "single" ]]; then + # In the single image build, the Dockerfile specifies /data as a volume + # mount, so we use that for all persistent data. + sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini + sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini +elif [[ "${TARGETBUILD}" = "docker-compose" ]]; then + # We remove the database_dir and view_index_dir settings from the local.ini + # in docker-compose because it will default to /opt/couchdb/data which is what + # our docker-compose was using prior to us switching to using our own CouchDB + # image. + sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini + sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini + sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini +elif [[ -n $KUBERNETES_SERVICE_HOST ]]; then + # In Kubernetes the directory /opt/couchdb/data has a persistent volume + # mount for storing database data. + sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini + + # We remove the database_dir and view_index_dir settings from the local.ini + # in Kubernetes because it will default to /opt/couchdb/data which is what + # our Helm chart was using prior to us switching to using our own CouchDB + # image. + sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini + sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini + + # We remove the -name setting from the vm.args file in Kubernetes because + # it will default to the pod FQDN, which is what's required for clustering + # to work. + sed -i "s/^-name .*$//g" /opt/couchdb/etc/vm.args +else + # For all other builds, we use /data for persistent data. + sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini + sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini +fi + +sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/couchdb/etc/vm.args +sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/clouseau/clouseau.ini + +# Start Clouseau. Budibase won't function correctly without Clouseau running, it +# powers the search API endpoints which are used to do all sorts, including +# populating app grids. +/opt/clouseau/bin/clouseau > /dev/stdout 2>&1 & + +# Start CouchDB. +/docker-entrypoint.sh /opt/couchdb/bin/couchdb & + +# Start SQS. +/opt/sqs/sqs --server "http://localhost:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 & + +# Wait for CouchDB to start up. +while [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; do + echo 'Waiting for CouchDB to start...'; + sleep 5; +done + +# CouchDB needs the `_users` and `_replicator` databases to exist before it will +# function correctly, so we create them here. +curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_users +curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_replicator +sleep infinity diff --git a/package.json b/package.json index 6674a9c0e1..d99304423a 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,8 @@ "build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .", "build:docker:single": "./scripts/build-single-image.sh", "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting", - "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:v3.2.1-sqs --push ./hosting/couchdb", + "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.2.1 --push ./hosting/couchdb", + "publish:docker:couch-sqs": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile.v2 -t budibase/couchdb:v3.2.1-sqs --push ./hosting/couchdb", "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting", "release:helm": "node scripts/releaseHelmChart", "env:multi:enable": "lerna run --stream env:multi:enable",