diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 65e6529678..757a9a4983 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -1,63 +1,66 @@ name: Budibase CI -on: - # Trigger the workflow on push or pull request, - # but only for the master branch - push: - branches: - - master - - develop - pull_request: +on: + # Trigger the workflow on push or pull request, + # but only for the master branch + push: branches: - master - develop - workflow_dispatch: + pull_request: + branches: + - master + - develop + workflow_dispatch: env: BRANCH: ${{ github.event.pull_request.head.ref }} BASE_BRANCH: ${{ github.event.pull_request.base.ref}} - PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Use Node.js 14.x - uses: actions/setup-node@v1 - with: - node-version: 14.x - - run: yarn - - run: yarn lint + - uses: actions/checkout@v3 + - name: Use Node.js 14.x + uses: actions/setup-node@v3 + with: + node-version: 14.x + cache: "yarn" + - run: yarn + - run: yarn lint build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - name: Use Node.js 14.x - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: 14.x - - name: Install Pro - run: yarn install:pro $BRANCH $BASE_BRANCH + cache: "yarn" - run: yarn - - run: yarn bootstrap - run: yarn build test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - name: Use Node.js 14.x - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: 14.x - - name: Install Pro - run: yarn install:pro $BRANCH $BASE_BRANCH + cache: "yarn" - run: yarn - - run: yarn bootstrap - - run: yarn build - - run: yarn test + - run: yarn build --scope=@budibase/types --scope=@budibase/shared-core --scope=@budibase/string-templates + - run: yarn test --ignore=@budibase/pro - uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos @@ -67,32 +70,58 @@ jobs: test-pro: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - name: Use Node.js 14.x - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: 14.x - - name: Install Pro - run: yarn install:pro $BRANCH $BASE_BRANCH + cache: "yarn" - run: yarn - - run: yarn bootstrap - - run: yarn test:pro + - run: yarn test --scope=@budibase/pro integration-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - name: Use Node.js 14.x - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: 14.x - - name: Install Pro - run: yarn install:pro $BRANCH $BASE_BRANCH - - run: yarn && yarn bootstrap && yarn build - - run: | + cache: "yarn" + - run: yarn + - run: yarn build + - name: Run tests + run: | cd qa-core yarn setup yarn test:ci env: BB_ADMIN_USER_EMAIL: admin BB_ADMIN_USER_PASSWORD: admin + + check-pro-submodule: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + fetch-depth: 0 + - name: Check submodule + run: | + cd packages/pro + git fetch + if ! git merge-base --is-ancestor $(git log -n 1 --pretty=format:%H) origin/develop; then + echo "Current commit has not been merged to develop" + echo "Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/develop/docs/getting_started.md" + exit 1 + else + echo "All good, the submodule had been merged!" + fi diff --git a/.github/workflows/release-develop.yml b/.github/workflows/release-develop.yml index 46e82e6efc..9b84b70de0 100644 --- a/.github/workflows/release-develop.yml +++ b/.github/workflows/release-develop.yml @@ -1,21 +1,11 @@ name: Budibase Prerelease concurrency: release-prerelease -on: - push: - branches: - - develop - paths: - - '.aws/**' - - '.github/**' - - 'charts/**' - - 'packages/**' - - 'scripts/**' - - 'package.json' - - 'yarn.lock' - - 'package.json' - - 'yarn.lock' - workflow_dispatch: +on: + push: + tags: + - v*-alpha.* + workflow_dispatch: env: # Posthog token used by ui at build time @@ -24,43 +14,60 @@ env: INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }} PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} FEATURE_PREVIEW_URL: https://budirelease.live - + jobs: release-images: - runs-on: ubuntu-latest + runs-on: ubuntu-latest steps: - - name: Fail if branch is not develop - if: github.ref != 'refs/heads/develop' - run: | - echo "Ref is not develop, you must run this job from develop." - exit 1 - uses: actions/checkout@v2 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + fetch-depth: 0 + + - name: Fail if tag is not develop + run: | + if ! git merge-base --is-ancestor ${{ github.sha }} origin/develop; then + echo "Tag is not in develop" + exit 1 + fi + - uses: actions/setup-node@v1 with: node-version: 14.x - - name: Install Pro - run: yarn install:pro develop - - - run: yarn - - run: yarn bootstrap + - run: yarn + - name: Update versions + run: | + version=$(cat lerna.json \ + | grep version \ + | head -1 \ + | awk -F: '{gsub(/"/,"",$2);gsub(/[[:space:]]*/,"",$2); print $2}' \ + | sed 's/[",]//g') + echo "Setting version $version" + yarn lerna exec "yarn version --no-git-tag-version --new-version=$version" + echo "Updating dependencies" + node scripts/syncLocalDependencies.js $version + echo "Syncing yarn workspace" + yarn - run: yarn build - run: yarn build:sdk -# - run: yarn test - name: Publish budibase packages to NPM env: NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | + run: | # setup the username and email. git config --global user.name "Budibase Staging Release Bot" git config --global user.email "<>" + git submodule foreach git commit -a -m 'Release process' + git commit -a -m 'Release process' echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc yarn release:develop - name: Build/release Docker images - run: | + run: | docker login -u $DOCKER_USER -p $DOCKER_PASSWORD yarn build:docker:develop env: @@ -84,7 +91,7 @@ jobs: git config user.name "Budibase Helm Bot" git config user.email "<>" git reset --hard - git pull + git fetch mkdir sync echo "Packaging chart to sync dir" helm package charts/budibase --version 0.0.0-develop --app-version develop --destination sync diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index 5d51b080f0..fdea9f76c2 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -2,57 +2,60 @@ name: Budibase Release concurrency: release on: - push: - branches: - - master - paths: - - '.aws/**' - - '.github/**' - - 'charts/**' - - 'packages/**' - - 'scripts/**' - - 'package.json' - - 'yarn.lock' - - 'package.json' - - 'yarn.lock' - workflow_dispatch: - inputs: - versioning: - type: choice - description: "Versioning type: patch, minor, major" - default: patch - options: - - patch - - minor - - major - required: true + push: + tags: + - "v[0-9]+.[0-9]+.[0-9]+" + # Exclude all pre-releases + - "!v*[0-9]+.[0-9]+.[0-9]+-*" + workflow_dispatch: + inputs: + tags: + description: "Release tag" + required: true + type: boolean env: - # Posthog token used by ui at build time + # Posthog token used by ui at build time POSTHOG_TOKEN: phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} - PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} jobs: release-images: runs-on: ubuntu-latest steps: - - name: Fail if branch is not master - if: github.ref != 'refs/heads/master' - run: | - echo "Ref is not master, you must run this job from master." - exit 1 - uses: actions/checkout@v2 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + fetch-depth: 0 + + - name: Fail if branch is not master + if: github.ref != 'refs/heads/master' + run: | + echo "Ref is not master, you must run this job from master." + // Change to "exit 1" when merged. Left to 0 to not fail all the pipelines and not to cause noise + exit 0 + - uses: actions/setup-node@v1 with: node-version: 14.x - - name: Install Pro - run: yarn install:pro master - - run: yarn - - run: yarn bootstrap + - name: Update versions + run: | + version=$(cat lerna.json \ + | grep version \ + | head -1 \ + | awk -F: '{gsub(/"/,"",$2);gsub(/[[:space:]]*/,"",$2); print $2}' \ + | sed 's/[",]//g') + echo "Setting version $version" + yarn lerna exec "yarn version --no-git-tag-version --new-version=$version" + echo "Updating dependencies" + node scripts/syncLocalDependencies.js $version + echo "Syncing yarn workspace" + yarn - run: yarn lint - run: yarn build - run: yarn build:sdk @@ -65,15 +68,17 @@ jobs: # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default git config --global user.name "Budibase Release Bot" git config --global user.email "<>" + git submodule foreach git commit -a -m 'Release process' + git commit -a -m 'Release process' echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc yarn release - - name: 'Get Previous tag' + - name: "Get Previous tag" id: previoustag uses: "WyriHaximus/github-action-get-previous-tag@v1" - name: Build/release Docker images - run: | + run: | docker login -u $DOCKER_USER -p $DOCKER_PASSWORD yarn build:docker env: @@ -103,7 +108,7 @@ jobs: git config user.name "Budibase Helm Bot" git config user.email "<>" git reset --hard - git pull + git fetch mkdir sync echo "Packaging chart to sync dir" helm package charts/budibase --version 0.0.0-master --app-version v"$RELEASE_VERSION" --destination sync diff --git a/.github/workflows/tag-prerelease.yml b/.github/workflows/tag-prerelease.yml new file mode 100644 index 0000000000..e04ef3b17d --- /dev/null +++ b/.github/workflows/tag-prerelease.yml @@ -0,0 +1,39 @@ +name: Tag prerelease +concurrency: release-prerelease + +on: + push: + branches: + - develop + paths: + - ".aws/**" + - ".github/**" + - "charts/**" + - "packages/**" + - "scripts/**" + - "package.json" + - "yarn.lock" + workflow_dispatch: + +jobs: + tag-prerelease: + runs-on: ubuntu-latest + + steps: + - name: Fail if branch is not develop + if: github.ref != 'refs/heads/develop' + run: | + echo "Ref is not develop, you must run this job from develop." + exit 1 + - uses: actions/checkout@v2 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + + - run: yarn + - name: Tag prerelease + run: | + # setup the username and email. + git config --global user.name "Budibase Staging Release Bot" + git config --global user.email "<>" + ./scripts/versionCommit.sh prerelease diff --git a/.github/workflows/tag-release.yml b/.github/workflows/tag-release.yml new file mode 100644 index 0000000000..55549a7e9d --- /dev/null +++ b/.github/workflows/tag-release.yml @@ -0,0 +1,49 @@ +name: Tag release +concurrency: release-prerelease + +on: + push: + branches: + - master + paths: + - ".aws/**" + - ".github/**" + - "charts/**" + - "packages/**" + - "scripts/**" + - "package.json" + - "yarn.lock" + workflow_dispatch: + inputs: + versioning: + type: choice + description: "Versioning type: patch, minor, major" + default: patch + options: + - patch + - minor + - major + required: true + +jobs: + tag-prerelease: + runs-on: ubuntu-latest + + steps: + - name: Fail if branch is not master + if: github.ref != 'refs/heads/master' + run: | + echo "Ref is not master, you must run this job from master." + exit 1 + - uses: actions/checkout@v2 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + + - run: yarn + - name: Tag prerelease + run: | + # setup the username and email. + git config --global user.name "Budibase Staging Release Bot" + git config --global user.email "<>" + ./scripts/versionCommit.sh ${{ github.event.inputs.versioning }} diff --git a/.gitmodules b/.gitmodules index e69de29bb2..2dd6ea53f2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "packages/pro"] + path = packages/pro + url = git@github.com:Budibase/budibase-pro.git diff --git a/.husky/post-checkout b/.husky/post-checkout new file mode 100755 index 0000000000..506b8bf5af --- /dev/null +++ b/.husky/post-checkout @@ -0,0 +1,4 @@ +# .husky/post-checkout +# ... + +git config submodule.recurse true \ No newline at end of file diff --git a/docs/DEV-SETUP-DEBIAN.md b/docs/DEV-SETUP-DEBIAN.md index cfd7eebf47..a8b1e3dce4 100644 --- a/docs/DEV-SETUP-DEBIAN.md +++ b/docs/DEV-SETUP-DEBIAN.md @@ -1,13 +1,17 @@ ## Dev Environment on Debian 11 ### Install NVM & Node 14 + NVM documentation: https://github.com/nvm-sh/nvm#installing-and-updating Install NVM + ``` curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash ``` + Install Node 14 + ``` nvm install 14 ``` @@ -17,13 +21,16 @@ nvm install 14 ``` npm install -g yarn jest lerna ``` + ### Install Docker and Docker Compose ``` apt install docker.io pip3 install docker-compose ``` + ### Clone the repo + ``` git clone https://github.com/Budibase/budibase.git ``` @@ -44,10 +51,13 @@ This setup process was tested on Debian 11 (bullseye) with version numbers show cd budibase yarn setup ``` + The yarn setup command runs several build steps i.e. + ``` node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev ``` + So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose. The dev version will be available on port 10000 i.e. @@ -55,6 +65,7 @@ The dev version will be available on port 10000 i.e. http://127.0.0.1:10000/builder/admin ### File descriptor issues with Vite and Chrome in Linux + If your dev environment stalls forever, with some network requests stuck in flight, it's likely that Chrome is trying to open more file descriptors than your system allows. To fix this, apply the following tweaks. @@ -62,4 +73,4 @@ Debian based distros: Add `* - nofile 65536` to `/etc/security/limits.conf`. Arch: -Add `DefaultLimitNOFILE=65536` to `/etc/systemd/system.conf`. \ No newline at end of file +Add `DefaultLimitNOFILE=65536` to `/etc/systemd/system.conf`. diff --git a/docs/DEV-SETUP-MACOSX.md b/docs/DEV-SETUP-MACOSX.md index 67eb5506ff..94ed3fc1ee 100644 --- a/docs/DEV-SETUP-MACOSX.md +++ b/docs/DEV-SETUP-MACOSX.md @@ -4,14 +4,14 @@ Install instructions [here](https://brew.sh/) -| **NOTE**: If you are working on a M1 Apple Silicon which is running Z shell, you could need to add -`eval $(/opt/homebrew/bin/brew shellenv)` line to your `.zshrc`. This will make your zsh to find the apps you install +| **NOTE**: If you are working on a M1 Apple Silicon which is running Z shell, you could need to add +`eval $(/opt/homebrew/bin/brew shellenv)` line to your `.zshrc`. This will make your zsh to find the apps you install through brew. - ### Install Node Budibase requires a recent version of node 14: + ``` brew install node npm node -v @@ -22,12 +22,15 @@ node -v ``` npm install -g yarn jest lerna ``` + ### Install Docker and Docker Compose ``` brew install docker docker-compose ``` + ### Clone the repo + ``` git clone https://github.com/Budibase/budibase.git ``` @@ -48,10 +51,13 @@ This setup process was tested on Mac OSX 12 (Monterey) with version numbers show cd budibase yarn setup ``` + The yarn setup command runs several build steps i.e. + ``` node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev ``` + So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose. The dev version will be available on port 10000 i.e. diff --git a/docs/DEV-SETUP-WINDOWS.md b/docs/DEV-SETUP-WINDOWS.md index c5608b7567..176e0700d7 100644 --- a/docs/DEV-SETUP-WINDOWS.md +++ b/docs/DEV-SETUP-WINDOWS.md @@ -1,13 +1,15 @@ ## Dev Environment on Windows 10/11 (WSL2) - ### Install WSL with Ubuntu LTS Enable WSL 2 on Windows 10/11 for docker support. + ``` wsl --set-default-version 2 ``` + Install Ubuntu LTS. + ``` wsl --install Ubuntu ``` @@ -16,6 +18,7 @@ Or follow the instruction here: https://learn.microsoft.com/en-us/windows/wsl/install ### Install Docker in windows + Download the installer from docker and install it. Check this url for more detailed instructions: @@ -24,18 +27,21 @@ https://docs.docker.com/desktop/install/windows-install/ You should follow the next steps from within the Ubuntu terminal. ### Install NVM & Node 14 + NVM documentation: https://github.com/nvm-sh/nvm#installing-and-updating Install NVM + ``` curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash ``` + Install Node 14 + ``` nvm install 14 ``` - ### Install npm requirements ``` @@ -43,6 +49,7 @@ npm install -g yarn jest lerna ``` ### Clone the repo + ``` git clone https://github.com/Budibase/budibase.git ``` @@ -63,10 +70,13 @@ This setup process was tested on Windows 11 with version numbers show below. You cd budibase yarn setup ``` + The yarn setup command runs several build steps i.e. + ``` node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev ``` + So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose. The dev version will be available on port 10000 i.e. @@ -74,8 +84,9 @@ The dev version will be available on port 10000 i.e. http://127.0.0.1:10000/builder/admin ### Working with the code + Here are the instructions to work on the application from within Visual Studio Code (in Windows) through the WSL. All the commands and files are within the Ubuntu system and it should run as if you were working on a Linux machine. https://code.visualstudio.com/docs/remote/wsl -Note you will be able to run the application from within the WSL terminal and you will be able to access the application from the a browser in Windows. \ No newline at end of file +Note you will be able to run the application from within the WSL terminal and you will be able to access the application from the a browser in Windows. diff --git a/hosting/couchdb/Dockerfile b/hosting/couchdb/Dockerfile index 11fab7129f..70b4413859 100644 --- a/hosting/couchdb/Dockerfile +++ b/hosting/couchdb/Dockerfile @@ -5,8 +5,11 @@ ENV COUCHDB_PASSWORD admin EXPOSE 5984 RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \ - apt-add-repository 'deb http://security.debian.org/debian-security stretch/updates main' && \ - apt-get update && apt-get install -y --no-install-recommends openjdk-8-jre && \ + wget -qO - https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public | apt-key add - && \ + apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security/updates main' && \ + apt-add-repository 'deb http://archive.debian.org/debian stretch-backports main' && \ + apt-add-repository --yes https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ && \ + apt-get update && apt-get install -y --no-install-recommends adoptopenjdk-8-hotspot && \ rm -rf /var/lib/apt/lists/ # setup clouseau diff --git a/hosting/proxy/nginx.prod.conf b/hosting/proxy/nginx.prod.conf index 8954106feb..001a08a9a6 100644 --- a/hosting/proxy/nginx.prod.conf +++ b/hosting/proxy/nginx.prod.conf @@ -55,7 +55,7 @@ http { set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com"; set $csp_object "object-src 'none'"; set $csp_base_uri "base-uri 'self'"; - set $csp_connect "connect-src 'self' https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com"; + set $csp_connect "connect-src 'self' https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.amazonaws.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com"; set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com https://js.intercomcdn.com https://fonts.intercomcdn.com"; set $csp_frame "frame-src 'self' https:"; set $csp_img "img-src http: https: data: blob:"; @@ -82,6 +82,12 @@ http { set $couchdb ${COUCHDB_UPSTREAM_URL}; set $watchtower ${WATCHTOWER_UPSTREAM_URL}; + location /health { + access_log off; + add_header 'Content-Type' 'application/json'; + return 200 '{ "status": "OK" }'; + } + location /app { proxy_pass $apps; } @@ -222,9 +228,9 @@ http { rewrite ^/files/signed/(.*)$ /$1 break; } - client_header_timeout 60; - client_body_timeout 60; - keepalive_timeout 60; + client_header_timeout 120; + client_body_timeout 120; + keepalive_timeout 120; # gzip gzip on; diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 2c6c06aa6e..fadcf235e9 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -22,7 +22,7 @@ FROM budibase/couchdb ARG TARGETARCH ENV TARGETARCH $TARGETARCH #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) -# e.g. docker build --build-arg TARGETBUILD=aas .... +# e.g. docker build --build-arg TARGETBUILD=aas .... ARG TARGETBUILD=single ENV TARGETBUILD $TARGETBUILD @@ -32,7 +32,7 @@ COPY --from=build /worker /worker # install base dependencies RUN apt-get update && \ apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server && \ - apt-add-repository 'deb http://security.debian.org/debian-security stretch/updates main' && \ + apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security/updates main' && \ apt-get update # install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx diff --git a/lerna.json b/lerna.json index ee4d8e8598..173e3b7a15 100644 --- a/lerna.json +++ b/lerna.json @@ -1,8 +1,22 @@ { - "version": "2.5.6-alpha.30", + "version": "2.6.19-alpha.4", "npmClient": "yarn", + "packages": [ + "packages/backend-core", + "packages/bbui", + "packages/builder", + "packages/cli", + "packages/client", + "packages/frontend-core", + "packages/sdk", + "packages/server", + "packages/shared-core", + "packages/string-templates", + "packages/types", + "packages/worker", + "packages/pro/packages/pro" + ], "useWorkspaces": true, - "packages": ["packages/*"], "command": { "publish": { "ignoreChanges": [ @@ -17,4 +31,4 @@ "loadEnvFiles": false } } -} +} \ No newline at end of file diff --git a/package.json b/package.json index 29b7c7c723..27f94ada0d 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "eslint": "^7.28.0", "eslint-plugin-cypress": "^2.11.3", "eslint-plugin-svelte3": "^3.2.0", - "husky": "^7.0.1", + "husky": "^8.0.3", "js-yaml": "^4.1.0", "kill-port": "^1.6.1", "lerna": "^6.6.1", @@ -17,22 +17,22 @@ "prettier-plugin-svelte": "^2.3.0", "rimraf": "^3.0.2", "rollup-plugin-replace": "^2.2.0", + "semver": "^7.5.0", "svelte": "^3.38.2", "typescript": "4.7.3" }, "scripts": { - "setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev", - "bootstrap": "lerna link && ./scripts/link-dependencies.sh", + "preinstall": "node scripts/syncProPackage.js", + "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev", + "bootstrap": "./scripts/bootstrap.sh && lerna link && ./scripts/link-dependencies.sh", "build": "lerna run --stream build", - "build:dev": "lerna run --stream prebuild && tsc --build --watch --preserveWatchOutput", + "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap", "backend:build": "./scripts/scopeBackend.sh 'lerna run --stream build'", "build:sdk": "lerna run --stream build:sdk", "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular", - "release": "lerna publish ${RELEASE_VERSION_TYPE:-patch} --yes --force-publish && yarn release:pro", - "release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop --exact && yarn release:pro:develop", - "release:pro": "bash scripts/pro/release.sh", - "release:pro:develop": "bash scripts/pro/release.sh develop", + "release": "lerna publish ${RELEASE_VERSION_TYPE:-patch} --yes --force-publish --no-git-tag-version --no-push --no-git-reset", + "release:develop": "lerna publish from-package --yes --force-publish --dist-tag develop --exact --no-git-tag-version --no-push --no-git-reset", "restore": "yarn run clean && yarn run bootstrap && yarn run build", "nuke": "yarn run nuke:packages && yarn run nuke:docker", "nuke:packages": "yarn run restore", @@ -46,7 +46,6 @@ "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built", "test": "lerna run --stream test --stream", - "test:pro": "bash scripts/pro/test.sh", "lint:eslint": "eslint packages && eslint qa-core", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"", "lint": "yarn run lint:eslint && yarn run lint:prettier", @@ -82,12 +81,25 @@ "mode:account": "yarn mode:cloud && yarn env:account:enable", "security:audit": "node scripts/audit.js", "postinstall": "husky install", - "install:pro": "bash scripts/pro/install.sh", - "dep:clean": "yarn clean && yarn bootstrap" + "dep:clean": "yarn clean -y && yarn bootstrap", + "submodules:load": "git submodule init && git submodule update && yarn && yarn bootstrap", + "submodules:unload": "git submodule deinit --all && yarn && yarn bootstrap" }, "workspaces": { "packages": [ - "packages/*" + "packages/backend-core", + "packages/bbui", + "packages/builder", + "packages/cli", + "packages/client", + "packages/frontend-core", + "packages/sdk", + "packages/server", + "packages/shared-core", + "packages/string-templates", + "packages/types", + "packages/worker", + "packages/pro/packages/pro" ] } } diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 7477406505..6a52cc6c03 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.5.6-alpha.30", + "version": "0.0.1", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -15,8 +15,6 @@ "prebuild": "rimraf dist/", "prepack": "cp package.json dist", "build": "tsc -p tsconfig.build.json", - "build:pro": "../../scripts/pro/build.sh", - "postbuild": "yarn run build:pro", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", "test": "bash scripts/test.sh", "test:watch": "jest --watchAll" @@ -24,7 +22,7 @@ "dependencies": { "@budibase/nano": "10.1.2", "@budibase/pouchdb-replication-stream": "1.2.10", - "@budibase/types": "2.5.6-alpha.30", + "@budibase/types": "0.0.1", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index a3b1ecc08d..e64c116663 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -47,7 +47,7 @@ async function put( type: LockType.TRY_ONCE, name: LockName.PERSIST_WRITETHROUGH, resource: key, - ttl: 1000, + ttl: 15000, }, async () => { const writeDb = async (toWrite: any) => { @@ -71,6 +71,7 @@ async function put( } } ) + if (!lockResponse.executed) { logWarn(`Ignoring redlock conflict in write-through cache`) } diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index aa40f13775..be49b9f261 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -21,7 +21,7 @@ export enum ViewName { AUTOMATION_LOGS = "automation_logs", ACCOUNT_BY_EMAIL = "account_by_email", PLATFORM_USERS_LOWERCASE = "platform_users_lowercase", - USER_BY_GROUP = "by_group_user", + USER_BY_GROUP = "user_by_group", APP_BACKUP_BY_TRIGGER = "by_trigger", } diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 861777b679..61d96bb4b0 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -104,6 +104,22 @@ async function newContext(updates: ContextMap, task: any) { return Context.run(context, task) } +export async function doInAutomationContext(params: { + appId: string + automationId: string + task: any +}): Promise { + const tenantId = getTenantIDFromAppID(params.appId) + return newContext( + { + tenantId, + appId: params.appId, + automationId: params.automationId, + }, + params.task + ) +} + export async function doInContext(appId: string, task: any): Promise { const tenantId = getTenantIDFromAppID(appId) return newContext( @@ -187,6 +203,11 @@ export function getTenantId(): string { return tenantId } +export function getAutomationId(): string | undefined { + const context = Context.get() + return context?.automationId +} + export function getAppId(): string | undefined { const context = Context.get() const foundId = context?.appId diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts index 727dad80bc..d687a93594 100644 --- a/packages/backend-core/src/context/types.ts +++ b/packages/backend-core/src/context/types.ts @@ -7,4 +7,5 @@ export type ContextMap = { identity?: IdentityContext environmentVariables?: Record isScim?: boolean + automationId?: string } diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 94d78e94ff..29ca4123f5 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -12,7 +12,7 @@ import { isDocument, } from "@budibase/types" import { getCouchInfo } from "./connections" -import { directCouchCall } from "./utils" +import { directCouchUrlCall } from "./utils" import { getPouchDB } from "./pouchDB" import { WriteStream, ReadStream } from "fs" import { newid } from "../../docIds/newid" @@ -46,6 +46,8 @@ export class DatabaseImpl implements Database { private readonly instanceNano?: Nano.ServerScope private readonly pouchOpts: DatabaseOpts + private readonly couchInfo = getCouchInfo() + constructor(dbName?: string, opts?: DatabaseOpts, connection?: string) { if (dbName == null) { throw new Error("Database name cannot be undefined.") @@ -53,8 +55,8 @@ export class DatabaseImpl implements Database { this.name = dbName this.pouchOpts = opts || {} if (connection) { - const couchInfo = getCouchInfo(connection) - this.instanceNano = buildNano(couchInfo) + this.couchInfo = getCouchInfo(connection) + this.instanceNano = buildNano(this.couchInfo) } if (!DatabaseImpl.nano) { DatabaseImpl.init() @@ -67,7 +69,11 @@ export class DatabaseImpl implements Database { } async exists() { - let response = await directCouchCall(`/${this.name}`, "HEAD") + const response = await directCouchUrlCall({ + url: `${this.couchInfo.url}/${this.name}`, + method: "HEAD", + cookie: this.couchInfo.cookie, + }) return response.status === 200 } diff --git a/packages/backend-core/src/db/couch/connections.ts b/packages/backend-core/src/db/couch/connections.ts index 06c661f350..4214c7cdc6 100644 --- a/packages/backend-core/src/db/couch/connections.ts +++ b/packages/backend-core/src/db/couch/connections.ts @@ -4,21 +4,21 @@ export const getCouchInfo = (connection?: string) => { const urlInfo = getUrlInfo(connection) let username let password - if (env.COUCH_DB_USERNAME) { - // set from env - username = env.COUCH_DB_USERNAME - } else if (urlInfo.auth.username) { + if (urlInfo.auth?.username) { // set from url username = urlInfo.auth.username + } else if (env.COUCH_DB_USERNAME) { + // set from env + username = env.COUCH_DB_USERNAME } else if (!env.isTest()) { throw new Error("CouchDB username not set") } - if (env.COUCH_DB_PASSWORD) { - // set from env - password = env.COUCH_DB_PASSWORD - } else if (urlInfo.auth.password) { + if (urlInfo.auth?.password) { // set from url password = urlInfo.auth.password + } else if (env.COUCH_DB_PASSWORD) { + // set from env + password = env.COUCH_DB_PASSWORD } else if (!env.isTest()) { throw new Error("CouchDB password not set") } diff --git a/packages/backend-core/src/db/couch/utils.ts b/packages/backend-core/src/db/couch/utils.ts index 426bf92158..51b2a38998 100644 --- a/packages/backend-core/src/db/couch/utils.ts +++ b/packages/backend-core/src/db/couch/utils.ts @@ -9,6 +9,20 @@ export async function directCouchCall( ) { let { url, cookie } = getCouchInfo() const couchUrl = `${url}/${path}` + return await directCouchUrlCall({ url: couchUrl, cookie, method, body }) +} + +export async function directCouchUrlCall({ + url, + cookie, + method, + body, +}: { + url: string + cookie: string + method: string + body?: any +}) { const params: any = { method: method, headers: { @@ -19,7 +33,7 @@ export async function directCouchCall( params.body = JSON.stringify(body) params.headers["Content-Type"] = "application/json" } - return await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params) + return await fetch(checkSlashesInUrl(encodeURI(url)), params) } export async function directCouchQuery( diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index 5d21d4f4e4..4660be81aa 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -434,7 +434,7 @@ export class QueryBuilder { }) } if (this.#query.empty) { - build(this.#query.empty, (key: string) => `!${key}:["" TO *]`) + build(this.#query.empty, (key: string) => `(*:* -${key}:["" TO *])`) } if (this.#query.notEmpty) { build(this.#query.notEmpty, (key: string) => `${key}:["" TO *]`) diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 0d413b8fa9..9163dfeba6 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -69,10 +69,10 @@ function findVersion() { try { const packageJsonFile = findFileInAncestors("package.json", process.cwd()) const content = readFileSync(packageJsonFile!, "utf-8") - const version = JSON.parse(content).version - return version + return JSON.parse(content).version } catch { - throw new Error("Cannot find a valid version in its package.json") + // throwing an error here is confusing/causes backend-core to be hard to import + return undefined } } @@ -95,7 +95,8 @@ const environment = { GOOGLE_CLIENT_SECRET: process.env.GOOGLE_CLIENT_SECRET, SALT_ROUNDS: process.env.SALT_ROUNDS, REDIS_URL: process.env.REDIS_URL || "localhost:6379", - REDIS_PASSWORD: process.env.REDIS_PASSWORD || "budibase", + REDIS_PASSWORD: process.env.REDIS_PASSWORD, + REDIS_CLUSTERED: process.env.REDIS_CLUSTERED, MOCK_REDIS: process.env.MOCK_REDIS, MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, @@ -154,6 +155,7 @@ const environment = { ? process.env.ENABLE_SSO_MAINTENANCE_MODE : false, VERSION: findVersion(), + DISABLE_PINO_LOGGER: process.env.DISABLE_PINO_LOGGER, _set(key: any, value: any) { process.env[key] = value // @ts-ignore diff --git a/packages/backend-core/src/events/publishers/license.ts b/packages/backend-core/src/events/publishers/license.ts index aff3286c87..094cf7fac1 100644 --- a/packages/backend-core/src/events/publishers/license.ts +++ b/packages/backend-core/src/events/publishers/license.ts @@ -3,7 +3,6 @@ import { Event, LicenseActivatedEvent, LicensePlanChangedEvent, - LicenseTierChangedEvent, PlanType, Account, LicensePortalOpenedEvent, @@ -11,22 +10,23 @@ import { LicenseCheckoutOpenedEvent, LicensePaymentFailedEvent, LicensePaymentRecoveredEvent, + PriceDuration, } from "@budibase/types" -async function tierChanged(account: Account, from: number, to: number) { - const properties: LicenseTierChangedEvent = { - accountId: account.accountId, - to, - from, +async function planChanged( + account: Account, + opts: { + from: PlanType + to: PlanType + fromQuantity: number | undefined + toQuantity: number | undefined + fromDuration: PriceDuration | undefined + toDuration: PriceDuration | undefined } - await publishEvent(Event.LICENSE_TIER_CHANGED, properties) -} - -async function planChanged(account: Account, from: PlanType, to: PlanType) { +) { const properties: LicensePlanChangedEvent = { accountId: account.accountId, - to, - from, + ...opts, } await publishEvent(Event.LICENSE_PLAN_CHANGED, properties) } @@ -74,7 +74,6 @@ async function paymentRecovered(account: Account) { } export default { - tierChanged, planChanged, activated, checkoutOpened, diff --git a/packages/backend-core/src/logging/index.ts b/packages/backend-core/src/logging/index.ts index b229f47dea..b87062c478 100644 --- a/packages/backend-core/src/logging/index.ts +++ b/packages/backend-core/src/logging/index.ts @@ -1,5 +1,5 @@ export * as correlation from "./correlation/correlation" -export { logger, disableLogger } from "./pino/logger" +export { logger } from "./pino/logger" export * from "./alerts" // turn off or on context logging i.e. tenantId, appId etc diff --git a/packages/backend-core/src/logging/pino/logger.ts b/packages/backend-core/src/logging/pino/logger.ts index dd4b505d30..cebc78ffc7 100644 --- a/packages/backend-core/src/logging/pino/logger.ts +++ b/packages/backend-core/src/logging/pino/logger.ts @@ -5,184 +5,206 @@ import * as correlation from "../correlation" import { IdentityType } from "@budibase/types" import { LOG_CONTEXT } from "../index" -// CORE LOGGERS - for disabling - -const BUILT_INS = { - log: console.log, - error: console.error, - info: console.info, - warn: console.warn, - trace: console.trace, - debug: console.debug, -} - // LOGGER -const pinoOptions: LoggerOptions = { - level: env.LOG_LEVEL, - formatters: { - level: label => { - return { level: label.toUpperCase() } - }, - bindings: () => { - return {} - }, - }, - timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`, -} - -if (env.isDev()) { - pinoOptions.transport = { - target: "pino-pretty", - options: { - singleLine: true, +let pinoInstance: pino.Logger | undefined +if (!env.DISABLE_PINO_LOGGER) { + const pinoOptions: LoggerOptions = { + level: env.LOG_LEVEL, + formatters: { + level: label => { + return { level: label.toUpperCase() } + }, + bindings: () => { + return {} + }, }, + timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`, } -} -export const logger = pino(pinoOptions) - -export function disableLogger() { - console.log = BUILT_INS.log - console.error = BUILT_INS.error - console.info = BUILT_INS.info - console.warn = BUILT_INS.warn - console.trace = BUILT_INS.trace - console.debug = BUILT_INS.debug -} - -// CONSOLE OVERRIDES - -interface MergingObject { - objects?: any[] - tenantId?: string - appId?: string - identityId?: string - identityType?: IdentityType - correlationId?: string - err?: Error -} - -function isPlainObject(obj: any) { - return typeof obj === "object" && obj !== null && !(obj instanceof Error) -} - -function isError(obj: any) { - return obj instanceof Error -} - -function isMessage(obj: any) { - return typeof obj === "string" -} - -/** - * Backwards compatibility between console logging statements - * and pino logging requirements. - */ -function getLogParams(args: any[]): [MergingObject, string] { - let error = undefined - let objects: any[] = [] - let message = "" - - args.forEach(arg => { - if (isMessage(arg)) { - message = `${message} ${arg}`.trimStart() - } - if (isPlainObject(arg)) { - objects.push(arg) - } - if (isError(arg)) { - error = arg - } - }) - - const identity = getIdentity() - - let contextObject = {} - - if (LOG_CONTEXT) { - contextObject = { - tenantId: getTenantId(), - appId: getAppId(), - identityId: identity?._id, - identityType: identity?.type, - correlationId: correlation.getId(), + if (env.isDev()) { + pinoOptions.transport = { + target: "pino-pretty", + options: { + singleLine: true, + }, } } - const mergingObject = { - objects: objects.length ? objects : undefined, - err: error, - ...contextObject, + pinoInstance = pino(pinoOptions) + + // CONSOLE OVERRIDES + + interface MergingObject { + objects?: any[] + tenantId?: string + appId?: string + automationId?: string + identityId?: string + identityType?: IdentityType + correlationId?: string + err?: Error } - return [mergingObject, message] -} - -console.log = (...arg: any[]) => { - const [obj, msg] = getLogParams(arg) - logger.info(obj, msg) -} -console.info = (...arg: any[]) => { - const [obj, msg] = getLogParams(arg) - logger.info(obj, msg) -} -console.warn = (...arg: any[]) => { - const [obj, msg] = getLogParams(arg) - logger.warn(obj, msg) -} -console.error = (...arg: any[]) => { - const [obj, msg] = getLogParams(arg) - logger.error(obj, msg) -} - -/** - * custom trace impl - this resembles the node trace behaviour rather - * than traditional trace logging - * @param arg - */ -console.trace = (...arg: any[]) => { - const [obj, msg] = getLogParams(arg) - if (!obj.err) { - // to get stack trace - obj.err = new Error() + function isPlainObject(obj: any) { + return typeof obj === "object" && obj !== null && !(obj instanceof Error) } - logger.trace(obj, msg) -} -console.debug = (...arg: any) => { - const [obj, msg] = getLogParams(arg) - logger.debug(obj, msg) -} - -// CONTEXT - -const getTenantId = () => { - let tenantId - try { - tenantId = context.getTenantId() - } catch (e: any) { - // do nothing + function isError(obj: any) { + return obj instanceof Error + } + + function isMessage(obj: any) { + return typeof obj === "string" + } + + /** + * Backwards compatibility between console logging statements + * and pino logging requirements. + */ + function getLogParams(args: any[]): [MergingObject, string] { + let error = undefined + let objects: any[] = [] + let message = "" + + args.forEach(arg => { + if (isMessage(arg)) { + message = `${message} ${arg}`.trimStart() + } + if (isPlainObject(arg)) { + objects.push(arg) + } + if (isError(arg)) { + error = arg + } + }) + + const identity = getIdentity() + + let contextObject = {} + + if (LOG_CONTEXT) { + contextObject = { + tenantId: getTenantId(), + appId: getAppId(), + automationId: getAutomationId(), + identityId: identity?._id, + identityType: identity?.type, + correlationId: correlation.getId(), + } + } + + const mergingObject: any = { + err: error, + ...contextObject, + } + + if (objects.length) { + // init generic data object for params supplied that don't have a + // '_logKey' field. This prints an object using argument index as the key + // e.g. { 0: {}, 1: {} } + const data: any = {} + let dataIndex = 0 + + for (let i = 0; i < objects.length; i++) { + const object = objects[i] + // the object has specified a log key + // use this instead of generic key + const logKey = object._logKey + if (logKey) { + delete object._logKey + mergingObject[logKey] = object + } else { + data[dataIndex] = object + dataIndex++ + } + } + + if (Object.keys(data).length) { + mergingObject.data = data + } + } + + return [mergingObject, message] + } + + console.log = (...arg: any[]) => { + const [obj, msg] = getLogParams(arg) + pinoInstance?.info(obj, msg) + } + console.info = (...arg: any[]) => { + const [obj, msg] = getLogParams(arg) + pinoInstance?.info(obj, msg) + } + console.warn = (...arg: any[]) => { + const [obj, msg] = getLogParams(arg) + pinoInstance?.warn(obj, msg) + } + console.error = (...arg: any[]) => { + const [obj, msg] = getLogParams(arg) + pinoInstance?.error(obj, msg) + } + + /** + * custom trace impl - this resembles the node trace behaviour rather + * than traditional trace logging + * @param arg + */ + console.trace = (...arg: any[]) => { + const [obj, msg] = getLogParams(arg) + if (!obj.err) { + // to get stack trace + obj.err = new Error() + } + pinoInstance?.trace(obj, msg) + } + + console.debug = (...arg: any) => { + const [obj, msg] = getLogParams(arg) + pinoInstance?.debug(obj, msg) + } + + // CONTEXT + + const getTenantId = () => { + let tenantId + try { + tenantId = context.getTenantId() + } catch (e: any) { + // do nothing + } + return tenantId + } + + const getAppId = () => { + let appId + try { + appId = context.getAppId() + } catch (e) { + // do nothing + } + return appId + } + + const getAutomationId = () => { + let appId + try { + appId = context.getAutomationId() + } catch (e) { + // do nothing + } + return appId + } + + const getIdentity = () => { + let identity + try { + identity = context.getIdentity() + } catch (e) { + // do nothing + } + return identity } - return tenantId } -const getAppId = () => { - let appId - try { - appId = context.getAppId() - } catch (e) { - // do nothing - } - return appId -} - -const getIdentity = () => { - let identity - try { - identity = context.getIdentity() - } catch (e) { - // do nothing - } - return identity -} +export const logger = pinoInstance diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index b80aece418..ec1d9d4a90 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -128,6 +128,7 @@ class InMemoryQueue { on() { // do nothing + return this } async waitForCompletion() { diff --git a/packages/backend-core/src/queue/listeners.ts b/packages/backend-core/src/queue/listeners.ts index e1975b5d06..42e3172364 100644 --- a/packages/backend-core/src/queue/listeners.ts +++ b/packages/backend-core/src/queue/listeners.ts @@ -1,5 +1,6 @@ import { Job, JobId, Queue } from "bull" import { JobQueue } from "./constants" +import * as context from "../context" export type StalledFn = (job: Job) => Promise @@ -31,71 +32,164 @@ function handleStalled(queue: Queue, removeStalledCb?: StalledFn) { }) } -function logging(queue: Queue, jobQueue: JobQueue) { - let eventType: string - switch (jobQueue) { - case JobQueue.AUTOMATION: - eventType = "automation-event" - break - case JobQueue.APP_BACKUP: - eventType = "app-backup-event" - break +function getLogParams( + eventType: QueueEventType, + event: BullEvent, + opts: { + job?: Job + jobId?: JobId + error?: Error + } = {}, + extra: any = {} +) { + const message = `[BULL] ${eventType}=${event}` + const err = opts.error + + const bullLog = { + _logKey: "bull", + eventType, + event, + job: opts.job, + jobId: opts.jobId || opts.job?.id, + ...extra, } + + let automationLog + if (opts.job?.data?.automation) { + automationLog = { + _logKey: "automation", + trigger: opts.job + ? opts.job.data.automation.definition.trigger.event + : undefined, + } + } + + return [message, err, bullLog, automationLog] +} + +enum BullEvent { + ERROR = "error", + WAITING = "waiting", + ACTIVE = "active", + STALLED = "stalled", + PROGRESS = "progress", + COMPLETED = "completed", + FAILED = "failed", + PAUSED = "paused", + RESUMED = "resumed", + CLEANED = "cleaned", + DRAINED = "drained", + REMOVED = "removed", +} + +enum QueueEventType { + AUTOMATION_EVENT = "automation-event", + APP_BACKUP_EVENT = "app-backup-event", + AUDIT_LOG_EVENT = "audit-log-event", + SYSTEM_EVENT = "system-event", +} + +const EventTypeMap: { [key in JobQueue]: QueueEventType } = { + [JobQueue.AUTOMATION]: QueueEventType.AUTOMATION_EVENT, + [JobQueue.APP_BACKUP]: QueueEventType.APP_BACKUP_EVENT, + [JobQueue.AUDIT_LOG]: QueueEventType.AUDIT_LOG_EVENT, + [JobQueue.SYSTEM_EVENT_QUEUE]: QueueEventType.SYSTEM_EVENT, +} + +function logging(queue: Queue, jobQueue: JobQueue) { + const eventType = EventTypeMap[jobQueue] + + function doInJobContext(job: Job, task: any) { + // if this is an automation job try to get the app id + const appId = job.data.event?.appId + if (appId) { + return context.doInContext(appId, task) + } else { + task() + } + } + + queue + .on(BullEvent.STALLED, async (job: Job) => { + // A job has been marked as stalled. This is useful for debugging job + // workers that crash or pause the event loop. + await doInJobContext(job, () => { + console.error(...getLogParams(eventType, BullEvent.STALLED, { job })) + }) + }) + .on(BullEvent.ERROR, (error: any) => { + // An error occurred. + console.error(...getLogParams(eventType, BullEvent.ERROR, { error })) + }) + if (process.env.NODE_DEBUG?.includes("bull")) { queue - .on("error", (error: any) => { - // An error occurred. - console.error(`${eventType}=error error=${JSON.stringify(error)}`) - }) - .on("waiting", (jobId: JobId) => { + .on(BullEvent.WAITING, (jobId: JobId) => { // A Job is waiting to be processed as soon as a worker is idling. - console.log(`${eventType}=waiting jobId=${jobId}`) + console.info(...getLogParams(eventType, BullEvent.WAITING, { jobId })) }) - .on("active", (job: Job, jobPromise: any) => { + .on(BullEvent.ACTIVE, async (job: Job, jobPromise: any) => { // A job has started. You can use `jobPromise.cancel()`` to abort it. - console.log(`${eventType}=active jobId=${job.id}`) + await doInJobContext(job, () => { + console.info(...getLogParams(eventType, BullEvent.ACTIVE, { job })) + }) }) - .on("stalled", (job: Job) => { - // A job has been marked as stalled. This is useful for debugging job - // workers that crash or pause the event loop. - console.error( - `${eventType}=stalled jobId=${job.id} job=${JSON.stringify(job)}` - ) + .on(BullEvent.PROGRESS, async (job: Job, progress: any) => { + // A job's progress was updated + await doInJobContext(job, () => { + console.info( + ...getLogParams( + eventType, + BullEvent.PROGRESS, + { job }, + { progress } + ) + ) + }) }) - .on("progress", (job: Job, progress: any) => { - // A job's progress was updated! - console.log( - `${eventType}=progress jobId=${job.id} progress=${progress}` - ) - }) - .on("completed", (job: Job, result) => { + .on(BullEvent.COMPLETED, async (job: Job, result) => { // A job successfully completed with a `result`. - console.log(`${eventType}=completed jobId=${job.id} result=${result}`) + await doInJobContext(job, () => { + console.info( + ...getLogParams(eventType, BullEvent.COMPLETED, { job }, { result }) + ) + }) }) - .on("failed", (job, err: any) => { + .on(BullEvent.FAILED, async (job: Job, error: any) => { // A job failed with reason `err`! - console.log(`${eventType}=failed jobId=${job.id} error=${err}`) + await doInJobContext(job, () => { + console.error( + ...getLogParams(eventType, BullEvent.FAILED, { job, error }) + ) + }) }) - .on("paused", () => { + .on(BullEvent.PAUSED, () => { // The queue has been paused. - console.log(`${eventType}=paused`) + console.info(...getLogParams(eventType, BullEvent.PAUSED)) }) - .on("resumed", (job: Job) => { + .on(BullEvent.RESUMED, () => { // The queue has been resumed. - console.log(`${eventType}=paused jobId=${job.id}`) + console.info(...getLogParams(eventType, BullEvent.RESUMED)) }) - .on("cleaned", (jobs: Job[], type: string) => { + .on(BullEvent.CLEANED, (jobs: Job[], type: string) => { // Old jobs have been cleaned from the queue. `jobs` is an array of cleaned // jobs, and `type` is the type of jobs cleaned. - console.log(`${eventType}=cleaned length=${jobs.length} type=${type}`) + console.info( + ...getLogParams( + eventType, + BullEvent.CLEANED, + {}, + { length: jobs.length, type } + ) + ) }) - .on("drained", () => { + .on(BullEvent.DRAINED, () => { // Emitted every time the queue has processed all the waiting jobs (even if there can be some delayed jobs not yet processed) - console.log(`${eventType}=drained`) + console.info(...getLogParams(eventType, BullEvent.DRAINED)) }) - .on("removed", (job: Job) => { + .on(BullEvent.REMOVED, (job: Job) => { // A job successfully removed. - console.log(`${eventType}=removed jobId=${job.id}`) + console.info(...getLogParams(eventType, BullEvent.REMOVED, { job })) }) } } diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 186865ccda..2d54b51a9f 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -12,7 +12,7 @@ import * as timers from "../timers" const RETRY_PERIOD_MS = 2000 const STARTUP_TIMEOUT_MS = 5000 -const CLUSTERED = false +const CLUSTERED = env.REDIS_CLUSTERED const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT // for testing just generate the client once @@ -81,7 +81,7 @@ function init(selectDb = DEFAULT_SELECT_DB) { if (client) { client.disconnect() } - const { redisProtocolUrl, opts, host, port } = getRedisOptions(CLUSTERED) + const { redisProtocolUrl, opts, host, port } = getRedisOptions() if (CLUSTERED) { client = new Redis.Cluster([{ host, port }], opts) diff --git a/packages/backend-core/src/redis/redlockImpl.ts b/packages/backend-core/src/redis/redlockImpl.ts index 4e9cd569ed..55b891ea84 100644 --- a/packages/backend-core/src/redis/redlockImpl.ts +++ b/packages/backend-core/src/redis/redlockImpl.ts @@ -85,7 +85,7 @@ export const doWithLock = async ( opts: LockOptions, task: () => Promise ): Promise> => { - const redlock = await getClient(opts.type) + const redlock = await getClient(opts.type, opts.customOptions) let lock try { // determine lock name diff --git a/packages/backend-core/src/redis/utils.ts b/packages/backend-core/src/redis/utils.ts index 7606c77b87..2c49ee4941 100644 --- a/packages/backend-core/src/redis/utils.ts +++ b/packages/backend-core/src/redis/utils.ts @@ -57,7 +57,7 @@ export enum SelectableDatabase { UNUSED_14 = 15, } -export function getRedisOptions(clustered = false) { +export function getRedisOptions() { let password = env.REDIS_PASSWORD let url: string[] | string = env.REDIS_URL.split("//") // get rid of the protocol @@ -83,7 +83,7 @@ export function getRedisOptions(clustered = false) { const opts: any = { connectTimeout: CONNECT_TIMEOUT_MS, } - if (clustered) { + if (env.REDIS_CLUSTERED) { opts.redisOptions = {} opts.redisOptions.tls = {} opts.redisOptions.password = password diff --git a/packages/backend-core/src/utils/tests/utils.spec.ts b/packages/backend-core/src/utils/tests/utils.spec.ts index ededa48628..5a0ac4f283 100644 --- a/packages/backend-core/src/utils/tests/utils.spec.ts +++ b/packages/backend-core/src/utils/tests/utils.spec.ts @@ -5,6 +5,7 @@ import * as db from "../../db" import { Header } from "../../constants" import { newid } from "../../utils" import env from "../../environment" +import { BBContext } from "@budibase/types" describe("utils", () => { const config = new DBTestConfiguration() @@ -106,4 +107,85 @@ describe("utils", () => { expect(actual).toBe(undefined) }) }) + + describe("isServingBuilder", () => { + let ctx: BBContext + + const expectResult = (result: boolean) => + expect(utils.isServingBuilder(ctx)).toBe(result) + + beforeEach(() => { + ctx = structures.koa.newContext() + }) + + it("returns true if current path is in builder", async () => { + ctx.path = "/builder/app/app_" + expectResult(true) + }) + + it("returns false if current path doesn't have '/' suffix", async () => { + ctx.path = "/builder/app" + expectResult(false) + + ctx.path = "/xx" + expectResult(false) + }) + }) + + describe("isServingBuilderPreview", () => { + let ctx: BBContext + + const expectResult = (result: boolean) => + expect(utils.isServingBuilderPreview(ctx)).toBe(result) + + beforeEach(() => { + ctx = structures.koa.newContext() + }) + + it("returns true if current path is in builder preview", async () => { + ctx.path = "/app/preview/xx" + expectResult(true) + }) + + it("returns false if current path is not in builder preview", async () => { + ctx.path = "/builder" + expectResult(false) + + ctx.path = "/xx" + expectResult(false) + }) + }) + + describe("isPublicAPIRequest", () => { + let ctx: BBContext + + const expectResult = (result: boolean) => + expect(utils.isPublicApiRequest(ctx)).toBe(result) + + beforeEach(() => { + ctx = structures.koa.newContext() + }) + + it("returns true if current path remains to public API", async () => { + ctx.path = "/api/public/v1/invoices" + expectResult(true) + + ctx.path = "/api/public/v1" + expectResult(true) + + ctx.path = "/api/public/v2" + expectResult(true) + + ctx.path = "/api/public/v21" + expectResult(true) + }) + + it("returns false if current path doesn't remain to public API", async () => { + ctx.path = "/api/public" + expectResult(false) + + ctx.path = "/xx" + expectResult(false) + }) + }) }) diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index 75b098093b..82da95983a 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -1,11 +1,5 @@ -import { getAllApps, queryGlobalView } from "../db" -import { - Header, - MAX_VALID_DATE, - DocumentType, - SEPARATOR, - ViewName, -} from "../constants" +import { getAllApps } from "../db" +import { Header, MAX_VALID_DATE, DocumentType, SEPARATOR } from "../constants" import env from "../environment" import * as tenancy from "../tenancy" import * as context from "../context" @@ -23,7 +17,9 @@ const APP_PREFIX = DocumentType.APP + SEPARATOR const PROD_APP_PREFIX = "/app/" const BUILDER_PREVIEW_PATH = "/app/preview" -const BUILDER_REFERER_PREFIX = "/builder/app/" +const BUILDER_PREFIX = "/builder" +const BUILDER_APP_PREFIX = `${BUILDER_PREFIX}/app/` +const PUBLIC_API_PREFIX = "/api/public/v" function confirmAppId(possibleAppId: string | undefined) { return possibleAppId && possibleAppId.startsWith(APP_PREFIX) @@ -69,6 +65,18 @@ export function isServingApp(ctx: Ctx) { return false } +export function isServingBuilder(ctx: Ctx): boolean { + return ctx.path.startsWith(BUILDER_APP_PREFIX) +} + +export function isServingBuilderPreview(ctx: Ctx): boolean { + return ctx.path.startsWith(BUILDER_PREVIEW_PATH) +} + +export function isPublicApiRequest(ctx: Ctx): boolean { + return ctx.path.startsWith(PUBLIC_API_PREFIX) +} + /** * Given a request tries to find the appId, which can be located in various places * @param {object} ctx The main request body to look through. @@ -110,7 +118,7 @@ export async function getAppIdFromCtx(ctx: Ctx) { // make sure this is performed after prod app url resolution, in case the // referer header is present from a builder redirect const referer = ctx.request.headers.referer - if (!appId && referer?.includes(BUILDER_REFERER_PREFIX)) { + if (!appId && referer?.includes(BUILDER_APP_PREFIX)) { const refererId = parseAppIdFromUrl(ctx.request.headers.referer) appId = confirmAppId(refererId) } diff --git a/packages/backend-core/tests/core/utilities/mocks/events.ts b/packages/backend-core/tests/core/utilities/mocks/events.ts index dacf7dcce8..81de1f8175 100644 --- a/packages/backend-core/tests/core/utilities/mocks/events.ts +++ b/packages/backend-core/tests/core/utilities/mocks/events.ts @@ -123,7 +123,6 @@ beforeAll(async () => { jest.spyOn(events.plugin, "imported") jest.spyOn(events.plugin, "deleted") - jest.spyOn(events.license, "tierChanged") jest.spyOn(events.license, "planChanged") jest.spyOn(events.license, "activated") jest.spyOn(events.license, "checkoutOpened") diff --git a/packages/backend-core/tests/core/utilities/structures/licenses.ts b/packages/backend-core/tests/core/utilities/structures/licenses.ts index 24b120451e..22e73f2871 100644 --- a/packages/backend-core/tests/core/utilities/structures/licenses.ts +++ b/packages/backend-core/tests/core/utilities/structures/licenses.ts @@ -7,16 +7,29 @@ import { PlanType, PriceDuration, PurchasedPlan, + PurchasedPrice, Quotas, Subscription, } from "@budibase/types" +export function price(): PurchasedPrice { + return { + amount: 10000, + amountMonthly: 10000, + currency: "usd", + duration: PriceDuration.MONTHLY, + priceId: "price_123", + dayPasses: undefined, + isPerUser: true, + } +} + export const plan = (type: PlanType = PlanType.FREE): PurchasedPlan => { return { type, usesInvoicing: false, - minUsers: 1, model: PlanModel.PER_USER, + price: type !== PlanType.FREE ? price() : undefined, } } diff --git a/packages/backend-core/tsconfig.json b/packages/backend-core/tsconfig.json index e95fb9ab4d..2b1419b051 100644 --- a/packages/backend-core/tsconfig.json +++ b/packages/backend-core/tsconfig.json @@ -7,11 +7,6 @@ "@budibase/types": ["../types/src"] } }, - "references": [ - { "path": "../types" } - ], - "exclude": [ - "node_modules", - "dist", - ] -} \ No newline at end of file + + "exclude": ["node_modules", "dist"] +} diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 6d4ba0a882..de1fc0db5e 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.5.6-alpha.30", + "version": "0.0.1", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,8 +38,8 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "1.2.1", - "@budibase/shared-core": "2.5.6-alpha.30", - "@budibase/string-templates": "2.5.6-alpha.30", + "@budibase/shared-core": "0.0.1", + "@budibase/string-templates": "0.0.1", "@spectrum-css/accordion": "3.0.24", "@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actiongroup": "1.0.1", @@ -84,7 +84,7 @@ "@spectrum-css/vars": "3.0.1", "dayjs": "^1.10.4", "easymde": "^2.16.1", - "svelte-flatpickr": "^3.3.2", + "svelte-flatpickr": "3.2.3", "svelte-portal": "^1.0.0" }, "resolutions": { diff --git a/packages/bbui/src/Actions/click_outside.js b/packages/bbui/src/Actions/click_outside.js index 5b7a059493..1961dca47c 100644 --- a/packages/bbui/src/Actions/click_outside.js +++ b/packages/bbui/src/Actions/click_outside.js @@ -1,4 +1,8 @@ -const ignoredClasses = [".flatpickr-calendar", ".spectrum-Popover"] +const ignoredClasses = [ + ".flatpickr-calendar", + ".spectrum-Popover", + ".download-js-link", +] let clickHandlers = [] /** @@ -22,8 +26,8 @@ const handleClick = event => { } // Ignore clicks for modals, unless the handler is registered from a modal - const sourceInModal = handler.anchor.closest(".spectrum-Modal") != null - const clickInModal = event.target.closest(".spectrum-Modal") != null + const sourceInModal = handler.anchor.closest(".spectrum-Underlay") != null + const clickInModal = event.target.closest(".spectrum-Underlay") != null if (clickInModal && !sourceInModal) { return } diff --git a/packages/bbui/src/Button/Button.svelte b/packages/bbui/src/Button/Button.svelte index f8a6004f8f..efd5f33bd2 100644 --- a/packages/bbui/src/Button/Button.svelte +++ b/packages/bbui/src/Button/Button.svelte @@ -2,6 +2,7 @@ import "@spectrum-css/button/dist/index-vars.css" import Tooltip from "../Tooltip/Tooltip.svelte" + export let type export let disabled = false export let size = "M" export let cta = false @@ -21,6 +22,7 @@