Merge branch 'develop' of github.com:Budibase/budibase into feature/automation-logs

hosting/letsencrypt/certificate-request.sh

@@ -10,15 +10,14 @@ certbot certonly --webroot --webroot-path="/var/www/html" \
 
 if (($? != 0)); then
     echo "ERROR: certbot request failed for $CUSTOM_DOMAIN use http on port 80 - exiting"
-    nginx -s stop
     exit 1
 else
     cp /app/letsencrypt/options-ssl-nginx.conf /etc/letsencrypt/options-ssl-nginx.conf
     cp /app/letsencrypt/ssl-dhparams.pem /etc/letsencrypt/ssl-dhparams.pem
     cp /app/letsencrypt/nginx-ssl.conf /etc/nginx/sites-available/nginx-ssl.conf
-    sed -i 's/CUSTOM_DOMAIN/$CUSTOM_DOMAIN/g' /etc/nginx/sites-available/nginx-ssl.conf
+    sed -i "s/CUSTOM_DOMAIN/$CUSTOM_DOMAIN/g" /etc/nginx/sites-available/nginx-ssl.conf
     ln -s /etc/nginx/sites-available/nginx-ssl.conf /etc/nginx/sites-enabled/nginx-ssl.conf
 
     echo "INFO: restart nginx after certbot request"
-    nginx -s reload
+    /etc/init.d/nginx restart
 fi

hosting/letsencrypt/nginx-ssl.conf

@@ -6,6 +6,7 @@ server {
     ssl_certificate_key /etc/letsencrypt/live/CUSTOM_DOMAIN/privkey.pem;
     include /etc/letsencrypt/options-ssl-nginx.conf;
     ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
+
     client_max_body_size 1000m;
     ignore_invalid_headers off;
     proxy_buffering off;
@@ -91,4 +92,5 @@ server {
     gzip_proxied any;
     gzip_comp_level 6;
     gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml;
+
 }

hosting/scripts/build-target-paths.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+echo ${TARGETBUILD} > /buildtarget.txt
+if [[ "${TARGETBUILD}" = "aas" ]]; then
+    # Azure AppService uses /home for persisent data & SSH on port 2222
+    mkdir -p /home/budibase/{minio,couchdb}
+    mkdir -p  /home/budibase/couchdb/data
+    chown -R couchdb:couchdb /home/budibase/couchdb/
+    apt update
+    apt-get install -y openssh-server
+    sed -i 's#dir=/opt/couchdb/data/search#dir=/home/budibase/couchdb/data/search#' /opt/clouseau/clouseau.ini
+    sed -i 's#/minio/minio server /minio &#/minio/minio server /home/budibase/minio &#' /runner.sh
+    sed -i 's#database_dir = ./data#database_dir = /home/budibase/couchdb/data#' /opt/couchdb/etc/default.ini
+    sed -i 's#view_index_dir = ./data#view_index_dir = /home/budibase/couchdb/data#' /opt/couchdb/etc/default.ini
+    sed -i "s/#Port 22/Port 2222/" /etc/ssh/sshd_config
+    /etc/init.d/ssh restart
+fi

hosting/scripts/healthcheck.sh

@@ -25,6 +25,13 @@ if [[ $(redis-cli -a $REDIS_PASSWORD --no-auth-warning  ping) != 'PONG' ]]; then
     healthy=false
 fi
 # mino, clouseau, 
+nginx -t -q
+NGINX_STATUS=$?
+
+if [[ $NGINX_STATUS -gt 0 ]]; then
+    echo 'ERROR: Nginx config problem';
+    healthy=false
+fi
 
 if [ $healthy == true ]; then
   exit 0

hosting/single/Dockerfile

@@ -19,8 +19,12 @@ ADD packages/worker .
 RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh
 
 FROM couchdb:3.2.1
-
+# TARGETARCH can be amd64 or arm e.g. docker build --build-arg TARGETARCH=amd64
 ARG TARGETARCH amd64
+#TARGETBUILD can be set to single (for single docker image) or aas (for azure app service)
+# e.g. docker build --build-arg TARGETBUILD=aas ....
+ARG TARGETBUILD single
+ENV TARGETBUILD $TARGETBUILD
 
 COPY --from=build /app /app
 COPY --from=build /worker /worker
@@ -33,7 +37,7 @@ ENV \
   COUCHDB_PASSWORD=budibase \
   COUCHDB_USER=budibase \
   COUCH_DB_URL=http://budibase:budibase@localhost:5984 \
-  CUSTOM_DOMAIN=budi001.custom.com \
+  # CUSTOM_DOMAIN=budi001.custom.com \
   DEPLOYMENT_ENVIRONMENT=docker \
   INTERNAL_API_KEY=budibase \
   JWT_SECRET=testsecret \
@@ -44,6 +48,7 @@ ENV \
   REDIS_PASSWORD=budibase \
   REDIS_URL=localhost:6379 \
   SELF_HOSTED=1 \
+  TARGETBUILD=$TARGETBUILD \
   WORKER_PORT=4002 \
   WORKER_URL=http://localhost:4002
 
@@ -62,6 +67,7 @@ RUN curl -sL https://deb.nodesource.com/setup_16.x -o /tmp/nodesource_setup.sh &
 
 # setup nginx
 ADD hosting/single/nginx.conf /etc/nginx
+ADD hosting/single/nginx-default-site.conf /etc/nginx/sites-enabled/default
 RUN mkdir -p /var/log/nginx && \
   touch /var/log/nginx/error.log && \
   touch /var/run/nginx.pid
@@ -100,6 +106,12 @@ RUN chmod +x ./runner.sh
 ADD hosting/scripts/healthcheck.sh .
 RUN chmod +x ./healthcheck.sh
 
+ADD hosting/scripts/build-target-paths.sh .
+RUN chmod +x ./build-target-paths.sh
+
+# For Azure App Service install SSH & point data locations to /home
+RUN /build-target-paths.sh
+
 # cleanup cache
 RUN yarn cache clean -f
 

hosting/single/README.md

@@ -4,7 +4,6 @@
 As an alternative to running several docker containers via docker-compose, the files under ./hosting/single can be used to build a docker image containing all of the Budibase components (minio, couch, clouseau etc).
 We call this the 'single image' container as the Dockerfile adds all the components to a single docker image.
 
-
 ## Usage
 
 - Amend Environment Variables
@@ -22,9 +21,9 @@ If you have other arrangements for a proxy in front of the single image containe
 We would suggest building the image with 6GB of RAM and 20GB of free disk space for build artifacts. The resulting image size will use approx 2GB of disk space.
 
 ### Build the Image
-The guidance below is based on building the Budibase single image on Debian 11. If you use another distro or OS you will need to amend the commands to suit. 
+The guidance below is based on building the Budibase single image on Debian 11 and AlmaLinux 8. If you use another distro or OS you will need to amend the commands to suit. 
-Install Node
+#### Install Node
-Budibase requires a recent version of node (14+) than is in the base Debian repos so:
+Budibase requires a more recent version of node (14+) than is available in the base Debian repos so:
 
 ```
 curl -sL https://deb.nodesource.com/setup_16.x | sudo bash -
@@ -35,25 +34,26 @@ Install yarn and lerna:
 ```
 npm install -g yarn jest lerna
 ```
-Install Docker
+#### Install Docker
+
 ```
 apt install -y docker.io
-apt install -y python3-pip
-pip3 install docker-compose
 ```
+
 Check the versions of each installed version. This process was tested with the version numbers below so YMMV using anything else:
 
 - Docker: 20.10.5
-- docker-compose: 1.29.2
 - node: 16.15.1
 - yarn: 1.22.19
 - lerna: 5.1.4
 
+#### Get the Code
 Clone the Budibase repo
 ```
 git clone https://github.com/Budibase/budibase.git
 cd budibase
 ```
+#### Setup Node
 Node setup:
 ```
 node ./hosting/scripts/setup.js
@@ -61,15 +61,20 @@ yarn
 yarn bootstrap
 yarn build
 ```
-
+#### Build Image
-Build the image from the Dockerfile:
+The following yarn command does some prep and then runs the docker build command:
-
 ```
 yarn build:docker:single
 ```
-If the docker build step fails run that step again manually with:
+If the docker build step fails try running that step again manually with:
 ```
-docker build --no-cache -t budibase:latest -f ./hosting/single/Dockerfile .
+docker build --build-arg TARGETARCH=amd --no-cache -t budibase:latest -f ./hosting/single/Dockerfile .
+```
+
+#### Azure App Services
+Azure have some specific requirements for running a container in their App Service. Specifically, installation of SSH to port 2222 and data storage under /home. If you would like to build a budibase container for Azure App Service add the build argument shown below setting it to 'aas'. You can remove the CUSTOM_DOMAIN env variable from the Dockerfile too as Azure terminate SSL before requests reach the container.
+```
+docker build --build-arg TARGETARCH=amd --build-arg TARGETBUILD=aas -t budibase:latest -f ./hosting/single/Dockerfile .
 ```
 
 ### Run the Container
@@ -85,6 +90,9 @@ When the container runs you should be able to access the container over http at
 
 When the Budibase UI appears you will be prompted to create an account to get started.
 
+### Podman
+The single image container builds fine when using podman in place of docker. You may be prompted for the registry to use for the CouchDB image and the HEALTHCHECK parameter is not OCI compliant so is ignored.
+
 ### Check
 There are many things that could go wrong so if your container is not building or running as expected please check the following before opening a support issue.
 Verify the healthcheck status of the container:
@@ -96,7 +104,6 @@ Check the container logs:
 docker logs budibase
 ```
 
-
 ### Support
 This single image build is still a work-in-progress so if you open an issue please provide the following information:
 - The OS and OS version you are building on

hosting/single/nginx-default-site.conf

@@ -0,0 +1,94 @@
+server {
+    listen       80 default_server;
+    listen  [::]:80 default_server;
+    server_name  _;
+
+    client_max_body_size 1000m;
+    ignore_invalid_headers off;
+    proxy_buffering off;
+    # port_in_redirect off;
+
+    location ^~ /.well-known/acme-challenge/ {
+        default_type "text/plain";
+        root         /var/www/html;
+        break;
+    }
+    location = /.well-known/acme-challenge/ {
+        return 404;
+    }
+
+    location /app {
+        proxy_pass      http://127.0.0.1:4001;
+    }
+
+    location = / {
+        proxy_pass      http://127.0.0.1:4001;
+    }
+
+    location ~ ^/(builder|app_) {
+        proxy_http_version  1.1;
+        proxy_set_header    Connection          $connection_upgrade;
+        proxy_set_header    Upgrade             $http_upgrade;
+        proxy_set_header    X-Real-IP           $remote_addr;
+        proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+        proxy_pass      http://127.0.0.1:4001;
+    }
+
+    location ~ ^/api/(system|admin|global)/ {
+        proxy_pass      http://127.0.0.1:4002;
+    }
+
+    location /worker/ {
+        proxy_pass      http://127.0.0.1:4002;
+        rewrite ^/worker/(.*)$ /$1 break;
+    }
+
+    location /api/ {
+        # calls to the API are rate limited with bursting
+        limit_req zone=ratelimit burst=20 nodelay;
+
+        # 120s timeout on API requests
+        proxy_read_timeout 120s;
+        proxy_connect_timeout 120s;
+        proxy_send_timeout 120s;
+
+        proxy_http_version  1.1;
+        proxy_set_header    Connection          $connection_upgrade;
+        proxy_set_header    Upgrade             $http_upgrade;
+        proxy_set_header    X-Real-IP           $remote_addr;
+        proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+
+        proxy_pass      http://127.0.0.1:4001;
+    }
+
+    location /db/ {
+        proxy_pass      http://127.0.0.1:5984;
+        rewrite ^/db/(.*)$ /$1 break;
+    }
+
+    location / {
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+
+        proxy_connect_timeout 300;
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
+        chunked_transfer_encoding off;
+        proxy_pass      http://127.0.0.1:9000;
+    }
+
+    client_header_timeout 60;
+    client_body_timeout   60;
+    keepalive_timeout     60;
+
+    # gzip
+    gzip on;
+    gzip_vary on;
+    gzip_proxied any;
+    gzip_comp_level 6;
+    gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml;
+
+
+
+}

hosting/single/nginx.conf

@@ -32,94 +32,6 @@ http {
     default     "upgrade";
   }
 
-  server {
+  include /etc/nginx/sites-enabled/*;
-    listen       80 default_server;
-    listen  [::]:80 default_server;
-    server_name  _;
-    client_max_body_size 1000m;
-    ignore_invalid_headers off;
-    proxy_buffering off;
-    # port_in_redirect off;
 
-    location ^~ /.well-known/acme-challenge/ {
-        default_type "text/plain";
-        root         /var/www/html;
-        break;
-    }
-    location = /.well-known/acme-challenge/ {
-        return 404;
-    }
-
-    location /app {
-      proxy_pass      http://127.0.0.1:4001;
-    }
-
-    location = / {
-      proxy_pass      http://127.0.0.1:4001;
-    }
-
-    location ~ ^/(builder|app_) {
-      proxy_http_version  1.1;
-      proxy_set_header    Connection          $connection_upgrade;
-      proxy_set_header    Upgrade             $http_upgrade;
-      proxy_set_header    X-Real-IP           $remote_addr;
-      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
-      proxy_pass      http://127.0.0.1:4001;
-    }
-
-    location ~ ^/api/(system|admin|global)/ {
-      proxy_pass      http://127.0.0.1:4002;
-    }
-
-    location /worker/ {
-      proxy_pass      http://127.0.0.1:4002;
-      rewrite ^/worker/(.*)$ /$1 break;
-    }
-
-    location /api/ {
-      # calls to the API are rate limited with bursting
-      limit_req zone=ratelimit burst=20 nodelay;
-
-      # 120s timeout on API requests
-      proxy_read_timeout 120s;
-      proxy_connect_timeout 120s;
-      proxy_send_timeout 120s;
-
-      proxy_http_version  1.1;
-      proxy_set_header    Connection          $connection_upgrade;
-      proxy_set_header    Upgrade             $http_upgrade;
-      proxy_set_header    X-Real-IP           $remote_addr;
-      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
-
-      proxy_pass      http://127.0.0.1:4001;
-    }
-
-    location /db/ {
-      proxy_pass      http://127.0.0.1:5984;
-      rewrite ^/db/(.*)$ /$1 break;
-    }
-
-    location / {
-      proxy_set_header X-Real-IP $remote_addr;
-      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_set_header X-Forwarded-Proto $scheme;
-
-      proxy_connect_timeout 300;
-      proxy_http_version 1.1;
-      proxy_set_header Connection "";
-      chunked_transfer_encoding off;
-      proxy_pass      http://127.0.0.1:9000;
-    }
-
-    client_header_timeout 60;
-    client_body_timeout   60;
-    keepalive_timeout     60;
-
-    # gzip
-    gzip on;
-    gzip_vary on;
-    gzip_proxied any;
-    gzip_comp_level 6;
-    gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml;
-  }
 }

hosting/single/test.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
-id=$(docker run -t -d -p 10000:10000 budibase:latest)
+id=$(docker run -t -d -p 80:80 budibase:latest)
 docker exec -it $id bash
 docker kill $id

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.211",
+  "version": "1.0.212-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.0.211",
+  "version": "1.0.212-alpha.0",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -20,7 +20,7 @@
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
-    "@budibase/types": "^1.0.211",
+    "@budibase/types": "^1.0.212-alpha.0",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.0.1",

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.211",
+  "version": "1.0.212-alpha.0",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "^1.2.1",
-    "@budibase/string-templates": "^1.0.211",
+    "@budibase/string-templates": "^1.0.212-alpha.0",
     "@spectrum-css/actionbutton": "^1.0.1",
     "@spectrum-css/actiongroup": "^1.0.1",
     "@spectrum-css/avatar": "^3.0.2",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.211",
+  "version": "1.0.212-alpha.0",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -69,10 +69,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.211",
+    "@budibase/bbui": "^1.0.212-alpha.0",
-    "@budibase/client": "^1.0.211",
+    "@budibase/client": "^1.0.212-alpha.0",
-    "@budibase/frontend-core": "^1.0.211",
+    "@budibase/frontend-core": "^1.0.212-alpha.0",
-    "@budibase/string-templates": "^1.0.211",
+    "@budibase/string-templates": "^1.0.212-alpha.0",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.211",
+  "version": "1.0.212-alpha.0",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.0.211",
+  "version": "1.0.212-alpha.0",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.211",
+    "@budibase/bbui": "^1.0.212-alpha.0",
-    "@budibase/frontend-core": "^1.0.211",
+    "@budibase/frontend-core": "^1.0.212-alpha.0",
-    "@budibase/string-templates": "^1.0.211",
+    "@budibase/string-templates": "^1.0.212-alpha.0",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "1.0.211",
+  "version": "1.0.212-alpha.0",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "^1.0.211",
+    "@budibase/bbui": "^1.0.212-alpha.0",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.211",
+  "version": "1.0.212-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/types": "^1.0.211",
+    "@budibase/backend-core": "^1.0.212-alpha.0",
-    "@budibase/backend-core": "^1.0.211",
+    "@budibase/client": "^1.0.212-alpha.0",
-    "@budibase/client": "^1.0.211",
+    "@budibase/pro": "1.0.212-alpha.0",
-    "@budibase/pro": "1.0.211",
+    "@budibase/string-templates": "^1.0.212-alpha.0",
-    "@budibase/string-templates": "^1.0.211",
+    "@budibase/types": "^1.0.212-alpha.0",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "1.0.211",
+  "version": "1.0.212-alpha.0",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/types",
-  "version": "1.0.211",
+  "version": "1.0.212-alpha.0",
   "description": "Budibase types",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "1.0.211",
+  "version": "1.0.212-alpha.0",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -34,10 +34,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/types": "^1.0.211",
+    "@budibase/backend-core": "^1.0.212-alpha.0",
-    "@budibase/backend-core": "^1.0.211",
+    "@budibase/pro": "1.0.212-alpha.0",
-    "@budibase/pro": "1.0.211",
+    "@budibase/string-templates": "^1.0.212-alpha.0",
-    "@budibase/string-templates": "^1.0.211",
+    "@budibase/types": "^1.0.212-alpha.0",
     "@koa/router": "8.0.8",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "0.3.2",