Merge pull request #6636 from Budibase/omnibus-action

Single image pipeline

.github/workflows/deploy-single-image.yml

@@ -0,0 +1,62 @@
+name: Deploy Budibase Single Container Image to DockerHub
+on:
+  push:
+    branches:
+      - "omnibus-action"
+      - "develop"
+      - "master"
+      - "main"
+env:
+  BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
+  BRANCH: ${{ github.event.pull_request.head.ref }}
+  CI: true
+  PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+  REGISTRY_URL: registry.hub.docker.com
+jobs:
+  build:
+    name: "build"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [14.x]
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v2
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Setup Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Install Pro
+        run: yarn install:pro $BRANCH $BASE_BRANCH
+      - name: Run Yarn
+        run: yarn
+      - name: Run Yarn Bootstrap
+        run: yarn bootstrap
+      - name: Runt Yarn Lint
+        run: yarn lint
+      - name: Run Yarn Build
+        run: yarn build
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_API_KEY }}
+      - name: Get the latest release version
+        id: version
+        run: |
+          release_version=$(cat lerna.json | jq -r '.version')
+          echo $release_version
+          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
+      - name: Tag and release Budibase service docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64,linux/arm64
+          tags: budibase/budibase,budibase/budibase:v${{ env.RELEASE_VERSION }}
+          file: ./hosting/single/Dockerfile

.yarnrc

@@ -0,0 +1 @@
+network-timeout 100000

hosting/single/Dockerfile

@@ -34,27 +34,32 @@ ENV \
   ARCHITECTURE=amd \
   BUDIBASE_ENVIRONMENT=PRODUCTION \
   CLUSTER_PORT=80 \
-  COUCHDB_PASSWORD=budibase \
-  COUCHDB_USER=budibase \
-  COUCH_DB_URL=http://budibase:budibase@localhost:5984 \
   # CUSTOM_DOMAIN=budi001.custom.com \
   DEPLOYMENT_ENVIRONMENT=docker \
-  INTERNAL_API_KEY=budibase \
-  JWT_SECRET=testsecret \
-  MINIO_ACCESS_KEY=budibase \
-  MINIO_SECRET_KEY=budibase \
   MINIO_URL=http://localhost:9000 \
   POSTHOG_TOKEN=phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS \
-  REDIS_PASSWORD=budibase \
   REDIS_URL=localhost:6379 \
   SELF_HOSTED=1 \
   TARGETBUILD=$TARGETBUILD \
   WORKER_PORT=4002 \
-  WORKER_URL=http://localhost:4002
+  WORKER_URL=http://localhost:4002 \
+  APPS_URL=http://localhost:4001
+
+# These secret env variables are generated by the runner at startup
+# their values can be overriden by the user, they will be written
+# to the .env file in the /data directory for use later on
+#  REDIS_PASSWORD=budibase \
+#  COUCHDB_PASSWORD=budibase \
+#  COUCHDB_USER=budibase \
+#  COUCH_DB_URL=http://budibase:budibase@localhost:5984 \
+#  INTERNAL_API_KEY=budibase \
+#  JWT_SECRET=testsecret \
+#  MINIO_ACCESS_KEY=budibase \
+#  MINIO_SECRET_KEY=budibase \
 
 # install base dependencies
 RUN apt-get update && \
-  apt-get install -y software-properties-common wget nginx && \
+  apt-get install -y software-properties-common wget nginx uuid-runtime && \
   apt-add-repository 'deb http://security.debian.org/debian-security stretch/updates main' && \
   apt-get update
 
@@ -66,8 +71,8 @@ RUN curl -sL https://deb.nodesource.com/setup_16.x -o /tmp/nodesource_setup.sh &
   npm install --global yarn pm2
 
 # setup nginx
-ADD hosting/single/nginx.conf /etc/nginx
+ADD hosting/single/nginx/nginx.conf /etc/nginx
-ADD hosting/single/nginx-default-site.conf /etc/nginx/sites-enabled/default
+ADD hosting/single/nginx/nginx-default-site.conf /etc/nginx/sites-enabled/default
 RUN mkdir -p /var/log/nginx && \
   touch /var/log/nginx/error.log && \
   touch /var/run/nginx.pid
@@ -86,13 +91,13 @@ RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clou
 
 WORKDIR /opt/clouseau
 RUN mkdir ./bin
-ADD hosting/single/clouseau ./bin/
+ADD hosting/single/clouseau/clouseau ./bin/
-ADD hosting/single/log4j.properties hosting/single/clouseau.ini ./
+ADD hosting/single/clouseau/log4j.properties hosting/single/clouseau/clouseau.ini ./
 RUN chmod +x ./bin/clouseau
 
 # setup CouchDB
 WORKDIR /opt/couchdb
-ADD hosting/single/vm.args ./etc/
+ADD hosting/single/couch/vm.args hosting/single/couch/local.ini ./etc/
 
 # setup minio
 WORKDIR /minio

hosting/single/clouseau/clouseau.ini

@@ -7,7 +7,7 @@ name=clouseau@127.0.0.1
 cookie=monster
 
 ; the path where you would like to store the search index files
-dir=/opt/couchdb/data/search
+dir=/data/search
 
 ; the number of search indexes that can be open simultaneously
 max_indexes_open=500

hosting/single/couch/local.ini

@@ -0,0 +1,5 @@
+; CouchDB Configuration Settings
+
+[couchdb]
+database_dir = /data/couch/dbs
+view_index_dir = /data/couch/views

hosting/single/nginx/nginx-default-site.conf

@@ -88,7 +88,4 @@ server {
     gzip_proxied any;
     gzip_comp_level 6;
     gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml;
-
-
-
 }

hosting/single/runner.sh

@@ -1,6 +1,34 @@
+#!/bin/bash
+declare -a ENV_VARS=("COUCHDB_USER" "COUCHDB_PASSWORD" "MINIO_ACCESS_KEY" "MINIO_SECRET_KEY" "INTERNAL_API_KEY" "JWT_SECRET" "REDIS_PASSWORD")
+if [ -f "/data/.env" ]; then
+    export $(cat /data/.env | xargs)
+fi
+# first randomise any unset environment variables
+for ENV_VAR in "${ENV_VARS[@]}"
+do
+    temp=$(eval "echo \$$ENV_VAR")
+    if [[ -z "${temp}" ]]; then
+        eval "export $ENV_VAR=$(uuidgen | sed -e 's/-//g')"
+    fi
+done
+if [[ -z "${COUCH_DB_URL}" ]]; then
+    export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@localhost:5984
+fi
+if [ ! -f "/data/.env" ]; then
+    touch /data/.env
+    for ENV_VAR in "${ENV_VARS[@]}"
+    do
+        temp=$(eval "echo \$$ENV_VAR")
+        echo "$ENV_VAR=$temp" >> /data/.env
+    done
+fi
+
+# make these directories in runner, incase of mount
+mkdir -p /data/couch/dbs /data/couch/views
+chown couchdb:couchdb /data/couch /data/couch/dbs /data/couch/views
 redis-server --requirepass $REDIS_PASSWORD &
 /opt/clouseau/bin/clouseau &
-/minio/minio server /minio &
+/minio/minio server /data/minio &
 /docker-entrypoint.sh /opt/couchdb/bin/couchdb &
 /etc/init.d/nginx restart
 if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then

hosting/single/test.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
-id=$(docker run -t -d -p 80:80 budibase:latest)
+id=$(docker run -t -d -p 8080:80 budibase:latest)
 docker exec -it $id bash
 docker kill $id

packages/server/.yarnrc

@@ -0,0 +1 @@
+network-timeout 100000

packages/worker/.yarnrc

@@ -0,0 +1 @@
+network-timeout 100000

packages/worker/src/api/controllers/global/users.ts

@@ -60,11 +60,13 @@ export const adminUser = async (ctx: any) => {
       tenantId,
     }
     try {
+      // always bust checklist beforehand, if an error occurs but can proceed, don't get
+      // stuck in a cycle
+      await cache.bustCache(cache.CacheKeys.CHECKLIST)
       const finalUser = await users.save(user, {
         hashPassword,
         requirePassword,
       })
-      await cache.bustCache(cache.CacheKeys.CHECKLIST)
 
       // events
       let account: CloudAccount | undefined