diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..85b026dd08 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +scripts/resources/minio filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index e294e50e8d..2e7851b338 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -30,7 +30,7 @@ env: jobs: lint: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout repo uses: actions/checkout@v4 @@ -47,7 +47,7 @@ jobs: - run: yarn lint build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout repo uses: actions/checkout@v4 @@ -76,7 +76,7 @@ jobs: fi helm-lint: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout repo uses: actions/checkout@v4 @@ -88,7 +88,7 @@ jobs: - run: cd charts/budibase && helm lint . test-libraries: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout repo uses: actions/checkout@v4 @@ -122,7 +122,7 @@ jobs: fi test-worker: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout repo uses: actions/checkout@v4 @@ -151,11 +151,22 @@ jobs: yarn test --verbose --reporters=default --reporters=github-actions test-server: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: datasource: - [mssql, mysql, postgres, postgres_legacy, mongodb, mariadb, oracle, sqs, none] + [ + mssql, + mysql, + postgres, + postgres_legacy, + mongodb, + mariadb, + oracle, + sqs, + elasticsearch, + none, + ] steps: - name: Checkout repo uses: actions/checkout@v4 @@ -192,6 +203,8 @@ jobs: docker pull budibase/oracle-database:23.2-slim-faststart elif [ "${{ matrix.datasource }}" == "postgres_legacy" ]; then docker pull postgres:9.5.25 + elif [ "${{ matrix.datasource }}" == "elasticsearch" ]; then + docker pull elasticsearch@${{ steps.dotenv.outputs.ELASTICSEARCH_SHA }} fi docker pull minio/minio & docker pull redis & @@ -240,7 +253,7 @@ jobs: yarn test --filter $FILTER --verbose --reporters=default --reporters=github-actions check-pro-submodule: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') steps: - name: Checkout repo and submodules @@ -299,7 +312,7 @@ jobs: fi check-lockfile: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') steps: - name: Checkout repo diff --git a/.github/workflows/readme-openapi.yml b/.github/workflows/readme-openapi.yml index 9f42f6141b..27a68e69cf 100644 --- a/.github/workflows/readme-openapi.yml +++ b/.github/workflows/readme-openapi.yml @@ -20,7 +20,7 @@ jobs: - run: yarn --frozen-lockfile - name: Install OpenAPI pkg - run: yarn global add openapi + run: yarn global add rdme@8.6.6 - name: update specs - run: cd packages/server && yarn specs && openapi specs/openapi.yaml --key=${{ secrets.README_API_KEY }} --id=6728a74f5918b50036c61841 + run: cd packages/server && yarn specs && rdme openapi specs/openapi.yaml --key=${{ secrets.README_API_KEY }} --id=67c16880add6da002352069a diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 311afbe706..ed2d94aacd 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -108,7 +108,7 @@ You can install them following any of the steps described below: - Installation steps: https://asdf-vm.com/guide/getting-started.html - asdf plugin add nodejs - asdf plugin add python - - npm install -g yarn + - asdf plugin add yarn ### Using NVM and pyenv diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index e4858d4af0..75b1a12d12 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -1,5 +1,5 @@ ARG BASEIMG=budibase/couchdb:v3.3.3-sqs-v2.1.1 -FROM node:20-slim as build +FROM node:20-slim AS build # install node-gyp dependencies RUN apt-get update && apt-get install -y --no-install-recommends g++ make python3 jq @@ -34,13 +34,13 @@ COPY packages/worker/dist packages/worker/dist COPY packages/worker/pm2.config.js packages/worker/pm2.config.js -FROM $BASEIMG as runner +FROM $BASEIMG AS runner ARG TARGETARCH -ENV TARGETARCH $TARGETARCH +ENV TARGETARCH=$TARGETARCH #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) # e.g. docker build --build-arg TARGETBUILD=aas .... ARG TARGETBUILD=single -ENV TARGETBUILD $TARGETBUILD +ENV TARGETBUILD=$TARGETBUILD # install base dependencies RUN apt-get update && \ @@ -67,6 +67,12 @@ RUN mkdir -p /var/log/nginx && \ # setup minio WORKDIR /minio + +# a 2022 version of minio that supports gateway mode +COPY scripts/resources/minio /minio +RUN chmod +x minio + +# handles the installation of minio in non-aas environments COPY scripts/install-minio.sh ./install.sh RUN chmod +x install.sh && ./install.sh @@ -86,7 +92,7 @@ COPY hosting/single/ssh/sshd_config /etc/ COPY hosting/single/ssh/ssh_setup.sh /tmp # setup letsencrypt certificate -RUN apt-get install -y certbot python3-certbot-nginx +RUN apt-get update && apt-get install -y certbot python3-certbot-nginx COPY hosting/letsencrypt /app/letsencrypt RUN chmod +x /app/letsencrypt/certificate-request.sh /app/letsencrypt/certificate-renew.sh diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index e06a197ad5..1a81515d31 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -1,53 +1,61 @@ #!/bin/bash -declare -a ENV_VARS=("COUCHDB_USER" "COUCHDB_PASSWORD" "DATA_DIR" "MINIO_ACCESS_KEY" "MINIO_SECRET_KEY" "INTERNAL_API_KEY" "JWT_SECRET" "REDIS_PASSWORD") -declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONMENT" "CLUSTER_PORT" "DEPLOYMENT_ENVIRONMENT" "MINIO_URL" "NODE_ENV" "POSTHOG_TOKEN" "REDIS_URL" "SELF_HOSTED" "WORKER_PORT" "WORKER_URL" "TENANT_FEATURE_FLAGS" "ACCOUNT_PORTAL_URL") -# Check the env vars set in Dockerfile have come through, AAS seems to drop them -[[ -z "${APP_PORT}" ]] && export APP_PORT=4001 -[[ -z "${ARCHITECTURE}" ]] && export ARCHITECTURE=amd -[[ -z "${BUDIBASE_ENVIRONMENT}" ]] && export BUDIBASE_ENVIRONMENT=PRODUCTION -[[ -z "${CLUSTER_PORT}" ]] && export CLUSTER_PORT=80 -[[ -z "${DEPLOYMENT_ENVIRONMENT}" ]] && export DEPLOYMENT_ENVIRONMENT=docker -[[ -z "${MINIO_URL}" ]] && [[ -z "${USE_S3}" ]] && export MINIO_URL=http://127.0.0.1:9000 -[[ -z "${NODE_ENV}" ]] && export NODE_ENV=production -[[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU -[[ -z "${ACCOUNT_PORTAL_URL}" ]] && export ACCOUNT_PORTAL_URL=https://account.budibase.app -[[ -z "${REDIS_URL}" ]] && export REDIS_URL=127.0.0.1:6379 -[[ -z "${SELF_HOSTED}" ]] && export SELF_HOSTED=1 -[[ -z "${WORKER_PORT}" ]] && export WORKER_PORT=4002 -[[ -z "${WORKER_URL}" ]] && export WORKER_URL=http://127.0.0.1:4002 -[[ -z "${APPS_URL}" ]] && export APPS_URL=http://127.0.0.1:4001 -[[ -z "${SERVER_TOP_LEVEL_PATH}" ]] && export SERVER_TOP_LEVEL_PATH=/app -# export CUSTOM_DOMAIN=budi001.custom.com -# Azure App Service customisations -if [[ "${TARGETBUILD}" = "aas" ]]; then - export DATA_DIR="${DATA_DIR:-/home}" - WEBSITES_ENABLE_APP_SERVICE_STORAGE=true - /etc/init.d/ssh start -else - export DATA_DIR=${DATA_DIR:-/data} +echo "Starting runner.sh..." + +# set defaults for Docker-related variables +export APP_PORT="${APP_PORT:-4001}" +export ARCHITECTURE="${ARCHITECTURE:-amd}" +export BUDIBASE_ENVIRONMENT="${BUDIBASE_ENVIRONMENT:-PRODUCTION}" +export CLUSTER_PORT="${CLUSTER_PORT:-80}" +export DEPLOYMENT_ENVIRONMENT="${DEPLOYMENT_ENVIRONMENT:-docker}" + +# only set MINIO_URL if neither MINIO_URL nor USE_S3 is set +if [[ -z "${MINIO_URL}" && -z "${USE_S3}" ]]; then + export MINIO_URL="http://127.0.0.1:9000" fi -mkdir -p ${DATA_DIR} -# Mount NFS or GCP Filestore if env vars exist for it -if [[ ! -z ${FILESHARE_IP} && ! -z ${FILESHARE_NAME} ]]; then + +export NODE_ENV="${NODE_ENV:-production}" +export POSTHOG_TOKEN="${POSTHOG_TOKEN:-phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU}" +export ACCOUNT_PORTAL_URL="${ACCOUNT_PORTAL_URL:-https://account.budibase.app}" +export REDIS_URL="${REDIS_URL:-127.0.0.1:6379}" +export SELF_HOSTED="${SELF_HOSTED:-1}" +export WORKER_PORT="${WORKER_PORT:-4002}" +export WORKER_URL="${WORKER_URL:-http://127.0.0.1:4002}" +export APPS_URL="${APPS_URL:-http://127.0.0.1:4001}" +export SERVER_TOP_LEVEL_PATH="${SERVER_TOP_LEVEL_PATH:-/app}" + +# set DATA_DIR and ensure the directory exists +if [[ ${TARGETBUILD} == "aas" ]]; then + export DATA_DIR="/home" +else + export DATA_DIR="${DATA_DIR:-/data}" +fi +mkdir -p "${DATA_DIR}" + +# mount NFS or GCP Filestore if FILESHARE_IP and FILESHARE_NAME are set +if [[ -n "${FILESHARE_IP}" && -n "${FILESHARE_NAME}" ]]; then echo "Mounting NFS share" apt update && apt install -y nfs-common nfs-kernel-server echo "Mount file share ${FILESHARE_IP}:/${FILESHARE_NAME} to ${DATA_DIR}" - mount -o nolock ${FILESHARE_IP}:/${FILESHARE_NAME} ${DATA_DIR} + mount -o nolock "${FILESHARE_IP}:/${FILESHARE_NAME}" "${DATA_DIR}" echo "Mounting result: $?" fi -if [ -f "${DATA_DIR}/.env" ]; then - # Read in the .env file and export the variables - for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done +# source environment variables from a .env file if it exists in DATA_DIR +if [[ -f "${DATA_DIR}/.env" ]]; then + set -a # Automatically export all variables loaded from .env + source "${DATA_DIR}/.env" + set +a fi -# randomise any unset environment variables -for ENV_VAR in "${ENV_VARS[@]}" -do - if [[ -z "${!ENV_VAR}" ]]; then - eval "export $ENV_VAR=$(uuidgen | sed -e 's/-//g')" + +# randomize any unset sensitive environment variables using uuidgen +env_vars=(COUCHDB_USER COUCHDB_PASSWORD MINIO_ACCESS_KEY MINIO_SECRET_KEY INTERNAL_API_KEY JWT_SECRET REDIS_PASSWORD) +for var in "${env_vars[@]}"; do + if [[ -z "${!var}" ]]; then + export "$var"="$(uuidgen | tr -d '-')" fi done + if [[ -z "${COUCH_DB_URL}" ]]; then export COUCH_DB_URL=http://$COUCHDB_USER:$COUCHDB_PASSWORD@127.0.0.1:5984 fi @@ -58,17 +66,15 @@ fi if [ ! -f "${DATA_DIR}/.env" ]; then touch ${DATA_DIR}/.env - for ENV_VAR in "${ENV_VARS[@]}" - do + for ENV_VAR in "${ENV_VARS[@]}"; do temp=$(eval "echo \$$ENV_VAR") - echo "$ENV_VAR=$temp" >> ${DATA_DIR}/.env + echo "$ENV_VAR=$temp" >>${DATA_DIR}/.env done - for ENV_VAR in "${DOCKER_VARS[@]}" - do + for ENV_VAR in "${DOCKER_VARS[@]}"; do temp=$(eval "echo \$$ENV_VAR") - echo "$ENV_VAR=$temp" >> ${DATA_DIR}/.env + echo "$ENV_VAR=$temp" >>${DATA_DIR}/.env done - echo "COUCH_DB_URL=${COUCH_DB_URL}" >> ${DATA_DIR}/.env + echo "COUCH_DB_URL=${COUCH_DB_URL}" >>${DATA_DIR}/.env fi # Read in the .env file and export the variables @@ -79,31 +85,44 @@ ln -s ${DATA_DIR}/.env /worker/.env # make these directories in runner, incase of mount mkdir -p ${DATA_DIR}/minio mkdir -p ${DATA_DIR}/redis +mkdir -p ${DATA_DIR}/couch chown -R couchdb:couchdb ${DATA_DIR}/couch REDIS_CONFIG="/etc/redis/redis.conf" sed -i "s#DATA_DIR#${DATA_DIR}#g" "${REDIS_CONFIG}" if [[ -n "${USE_DEFAULT_REDIS_CONFIG}" ]]; then - REDIS_CONFIG="" + REDIS_CONFIG="" fi if [[ -n "${REDIS_PASSWORD}" ]]; then - redis-server "${REDIS_CONFIG}" --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 & + redis-server "${REDIS_CONFIG}" --requirepass $REDIS_PASSWORD >/dev/stdout 2>&1 & else - redis-server "${REDIS_CONFIG}" > /dev/stdout 2>&1 & + redis-server "${REDIS_CONFIG}" >/dev/stdout 2>&1 & fi -/bbcouch-runner.sh & + +echo "Starting callback CouchDB runner..." +./bbcouch-runner.sh & # only start minio if use s3 isn't passed if [[ -z "${USE_S3}" ]]; then - /minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 & + if [[ ${TARGETBUILD} == aas ]]; then + echo "Starting MinIO in Azure Gateway mode" + if [[ -z "${AZURE_STORAGE_ACCOUNT}" || -z "${AZURE_STORAGE_KEY}" || -z "${MINIO_ACCESS_KEY}" || -z "${MINIO_SECRET_KEY}" ]]; then + echo "The following environment variables must be set: AZURE_STORAGE_ACCOUNT, AZURE_STORAGE_KEY, MINIO_ACCESS_KEY, MINIO_SECRET_KEY" + exit 1 + fi + /minio/minio gateway azure --console-address ":9001" >/dev/stdout 2>&1 & + else + echo "Starting MinIO in standalone mode" + /minio/minio server --console-address ":9001" ${DATA_DIR}/minio >/dev/stdout 2>&1 & + fi fi /etc/init.d/nginx restart if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then # Add monthly cron job to renew certbot certificate - echo -n "* * 2 * * root exec /app/letsencrypt/certificate-renew.sh ${CUSTOM_DOMAIN}" >> /etc/cron.d/certificate-renew + echo -n "* * 2 * * root exec /app/letsencrypt/certificate-renew.sh ${CUSTOM_DOMAIN}" >>/etc/cron.d/certificate-renew chmod +x /etc/cron.d/certificate-renew # Request the certbot certificate /app/letsencrypt/certificate-request.sh ${CUSTOM_DOMAIN} diff --git a/lerna.json b/lerna.json index b6eb31f2b0..d20097b6e5 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "3.4.13", + "version": "3.4.22", "npmClient": "yarn", "concurrency": 20, "command": { diff --git a/packages/bbui/src/Badge/Badge.svelte b/packages/bbui/src/Badge/Badge.svelte index e4ec7d4f33..dadbaa3317 100644 --- a/packages/bbui/src/Badge/Badge.svelte +++ b/packages/bbui/src/Badge/Badge.svelte @@ -11,6 +11,7 @@ export let active = false export let inactive = false export let hoverable = false + export let outlineColor = null @@ -29,6 +30,7 @@ class:spectrum-Label--seafoam={seafoam} class:spectrum-Label--active={active} class:spectrum-Label--inactive={inactive} + style={outlineColor ? `border: 2px solid ${outlineColor}` : ""} > diff --git a/packages/bbui/src/ClearButton/ClearButton.svelte b/packages/bbui/src/ClearButton/ClearButton.svelte index 5a2f6bc04c..720a462546 100644 --- a/packages/bbui/src/ClearButton/ClearButton.svelte +++ b/packages/bbui/src/ClearButton/ClearButton.svelte @@ -1,6 +1,6 @@ -