diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 65e6529678..6c875f2dfe 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -1,98 +1,149 @@ name: Budibase CI -on: - # Trigger the workflow on push or pull request, - # but only for the master branch - push: - branches: - - master - - develop - pull_request: +on: + # Trigger the workflow on push or pull request, + # but only for the master branch + push: branches: - master - develop - workflow_dispatch: + pull_request: + branches: + - master + - develop + workflow_dispatch: env: BRANCH: ${{ github.event.pull_request.head.ref }} BASE_BRANCH: ${{ github.event.pull_request.base.ref}} - PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Use Node.js 14.x - uses: actions/setup-node@v1 - with: - node-version: 14.x - - run: yarn - - run: yarn lint + - uses: actions/checkout@v3 + - name: Use Node.js 14.x + uses: actions/setup-node@v3 + with: + node-version: 14.x + cache: "yarn" + - run: yarn + - run: yarn lint build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - name: Use Node.js 14.x - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: 14.x - - name: Install Pro - run: yarn install:pro $BRANCH $BASE_BRANCH + cache: "yarn" - run: yarn - - run: yarn bootstrap + # Run build all the projects - run: yarn build + # Check the types of the projects built via esbuild + - run: yarn check:types - test: + test-libraries: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - name: Use Node.js 14.x - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: 14.x - - name: Install Pro - run: yarn install:pro $BRANCH $BASE_BRANCH + cache: "yarn" - run: yarn - - run: yarn bootstrap - - run: yarn build - - run: yarn test + - run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro - uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos name: codecov-umbrella verbose: true + test-services: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} + - name: Use Node.js 14.x + uses: actions/setup-node@v3 + with: + node-version: 14.x + cache: "yarn" + - run: yarn + - run: yarn test --scope=@budibase/worker --scope=@budibase/server + - uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN || github.token }} # not required for public repos + name: codecov-umbrella + verbose: true + test-pro: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - name: Use Node.js 14.x - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: 14.x - - name: Install Pro - run: yarn install:pro $BRANCH $BASE_BRANCH + cache: "yarn" - run: yarn - - run: yarn bootstrap - - run: yarn test:pro + - run: yarn test --scope=@budibase/pro integration-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - name: Use Node.js 14.x - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: 14.x - - name: Install Pro - run: yarn install:pro $BRANCH $BASE_BRANCH - - run: yarn && yarn bootstrap && yarn build - - run: | + cache: "yarn" + - run: yarn + - run: yarn build + - name: Run tests + run: | cd qa-core yarn setup yarn test:ci env: BB_ADMIN_USER_EMAIL: admin BB_ADMIN_USER_PASSWORD: admin + + check-pro-submodule: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} + fetch-depth: 0 + - name: Check submodule + run: | + cd packages/pro + git fetch + if ! git merge-base --is-ancestor $(git log -n 1 --pretty=format:%H) origin/develop; then + echo "Current commit has not been merged to develop" + echo "Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/develop/docs/getting_started.md" + exit 1 + else + echo "All good, the submodule had been merged!" + fi diff --git a/.github/workflows/release-develop.yml b/.github/workflows/release-develop.yml index 46e82e6efc..48c51e8457 100644 --- a/.github/workflows/release-develop.yml +++ b/.github/workflows/release-develop.yml @@ -1,21 +1,13 @@ name: Budibase Prerelease -concurrency: release-prerelease +concurrency: + group: release-prerelease + cancel-in-progress: false -on: - push: - branches: - - develop - paths: - - '.aws/**' - - '.github/**' - - 'charts/**' - - 'packages/**' - - 'scripts/**' - - 'package.json' - - 'yarn.lock' - - 'package.json' - - 'yarn.lock' - workflow_dispatch: +on: + push: + tags: + - v*-alpha.* + workflow_dispatch: env: # Posthog token used by ui at build time @@ -24,43 +16,60 @@ env: INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }} PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} FEATURE_PREVIEW_URL: https://budirelease.live - + jobs: release-images: - runs-on: ubuntu-latest + runs-on: ubuntu-latest steps: - - name: Fail if branch is not develop - if: github.ref != 'refs/heads/develop' - run: | - echo "Ref is not develop, you must run this job from develop." - exit 1 - uses: actions/checkout@v2 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + fetch-depth: 0 + + - name: Fail if tag is not develop + run: | + if ! git merge-base --is-ancestor ${{ github.sha }} origin/develop; then + echo "Tag is not in develop" + exit 1 + fi + - uses: actions/setup-node@v1 with: node-version: 14.x - - name: Install Pro - run: yarn install:pro develop - - - run: yarn - - run: yarn bootstrap - - run: yarn build + - run: yarn install --frozen-lockfile + - name: Update versions + run: | + version=$(cat lerna.json \ + | grep version \ + | head -1 \ + | awk -F: '{gsub(/"/,"",$2);gsub(/[[:space:]]*/,"",$2); print $2}' \ + | sed 's/[",]//g') + echo "Setting version $version" + yarn lerna exec "yarn version --no-git-tag-version --new-version=$version" + echo "Updating dependencies" + node scripts/syncLocalDependencies.js $version + echo "Syncing yarn workspace" + yarn + - run: yarn build --configuration=production - run: yarn build:sdk -# - run: yarn test - name: Publish budibase packages to NPM env: NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | + run: | # setup the username and email. git config --global user.name "Budibase Staging Release Bot" git config --global user.email "<>" + git submodule foreach git commit -a -m 'Release process' + git commit -a -m 'Release process' echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc yarn release:develop - name: Build/release Docker images - run: | + run: | docker login -u $DOCKER_USER -p $DOCKER_PASSWORD yarn build:docker:develop env: @@ -84,7 +93,7 @@ jobs: git config user.name "Budibase Helm Bot" git config user.email "<>" git reset --hard - git pull + git fetch mkdir sync echo "Packaging chart to sync dir" helm package charts/budibase --version 0.0.0-develop --app-version develop --destination sync diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index 5d51b080f0..f05d369a34 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -1,60 +1,65 @@ name: Budibase Release -concurrency: release +concurrency: + group: release + cancel-in-progress: false on: - push: - branches: - - master - paths: - - '.aws/**' - - '.github/**' - - 'charts/**' - - 'packages/**' - - 'scripts/**' - - 'package.json' - - 'yarn.lock' - - 'package.json' - - 'yarn.lock' - workflow_dispatch: - inputs: - versioning: - type: choice - description: "Versioning type: patch, minor, major" - default: patch - options: - - patch - - minor - - major - required: true + push: + tags: + - "v[0-9]+.[0-9]+.[0-9]+" + # Exclude all pre-releases + - "!v*[0-9]+.[0-9]+.[0-9]+-*" + workflow_dispatch: + inputs: + tags: + description: "Release tag" + required: true + type: boolean env: - # Posthog token used by ui at build time + # Posthog token used by ui at build time POSTHOG_TOKEN: phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }} - PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} jobs: release-images: runs-on: ubuntu-latest steps: - - name: Fail if branch is not master - if: github.ref != 'refs/heads/master' - run: | - echo "Ref is not master, you must run this job from master." - exit 1 - uses: actions/checkout@v2 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + fetch-depth: 0 + + - name: Fail if branch is not master + if: github.ref != 'refs/heads/master' + run: | + echo "Ref is not master, you must run this job from master." + // Change to "exit 1" when merged. Left to 0 to not fail all the pipelines and not to cause noise + exit 0 + - uses: actions/setup-node@v1 with: node-version: 14.x - - name: Install Pro - run: yarn install:pro master - - - run: yarn - - run: yarn bootstrap + - run: yarn install --frozen-lockfile + - name: Update versions + run: | + version=$(cat lerna.json \ + | grep version \ + | head -1 \ + | awk -F: '{gsub(/"/,"",$2);gsub(/[[:space:]]*/,"",$2); print $2}' \ + | sed 's/[",]//g') + echo "Setting version $version" + yarn lerna exec "yarn version --no-git-tag-version --new-version=$version" + echo "Updating dependencies" + node scripts/syncLocalDependencies.js $version + echo "Syncing yarn workspace" + yarn - run: yarn lint - - run: yarn build + - run: yarn build --configuration=production - run: yarn build:sdk - name: Publish budibase packages to NPM @@ -65,15 +70,17 @@ jobs: # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default git config --global user.name "Budibase Release Bot" git config --global user.email "<>" + git submodule foreach git commit -a -m 'Release process' + git commit -a -m 'Release process' echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc yarn release - - name: 'Get Previous tag' + - name: "Get Previous tag" id: previoustag uses: "WyriHaximus/github-action-get-previous-tag@v1" - name: Build/release Docker images - run: | + run: | docker login -u $DOCKER_USER -p $DOCKER_PASSWORD yarn build:docker env: @@ -103,7 +110,7 @@ jobs: git config user.name "Budibase Helm Bot" git config user.email "<>" git reset --hard - git pull + git fetch mkdir sync echo "Packaging chart to sync dir" helm package charts/budibase --version 0.0.0-master --app-version v"$RELEASE_VERSION" --destination sync diff --git a/.github/workflows/tag-prerelease.yml b/.github/workflows/tag-prerelease.yml new file mode 100644 index 0000000000..83660e409d --- /dev/null +++ b/.github/workflows/tag-prerelease.yml @@ -0,0 +1,41 @@ +name: Tag prerelease +concurrency: + group: tag-prerelease + cancel-in-progress: false + +on: + push: + branches: + - develop + paths: + - ".aws/**" + - ".github/**" + - "charts/**" + - "packages/**" + - "scripts/**" + - "package.json" + - "yarn.lock" + workflow_dispatch: + +jobs: + tag-prerelease: + runs-on: ubuntu-latest + + steps: + - name: Fail if branch is not develop + if: github.ref != 'refs/heads/develop' + run: | + echo "Ref is not develop, you must run this job from develop." + exit 1 + - uses: actions/checkout@v2 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + + - run: yarn + - name: Tag prerelease + run: | + # setup the username and email. + git config --global user.name "Budibase Staging Release Bot" + git config --global user.email "<>" + ./scripts/versionCommit.sh prerelease diff --git a/.github/workflows/tag-release.yml b/.github/workflows/tag-release.yml new file mode 100644 index 0000000000..1dcb16ac56 --- /dev/null +++ b/.github/workflows/tag-release.yml @@ -0,0 +1,51 @@ +name: Tag release +concurrency: + group: tag-release + cancel-in-progress: false + +on: + push: + branches: + - master + paths: + - ".aws/**" + - ".github/**" + - "charts/**" + - "packages/**" + - "scripts/**" + - "package.json" + - "yarn.lock" + workflow_dispatch: + inputs: + versioning: + type: choice + description: "Versioning type: patch, minor, major" + default: patch + options: + - patch + - minor + - major + required: true + +jobs: + tag-prerelease: + runs-on: ubuntu-latest + + steps: + - name: Fail if branch is not master + if: github.ref != 'refs/heads/master' + run: | + echo "Ref is not master, you must run this job from master." + exit 1 + - uses: actions/checkout@v2 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + + - run: yarn + - name: Tag prerelease + run: | + # setup the username and email. + git config --global user.name "Budibase Staging Release Bot" + git config --global user.email "<>" + ./scripts/versionCommit.sh ${{ github.event.inputs.versioning }} diff --git a/.gitmodules b/.gitmodules index e69de29bb2..2dd6ea53f2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "packages/pro"] + path = packages/pro + url = git@github.com:Budibase/budibase-pro.git diff --git a/.husky/post-checkout b/.husky/post-checkout new file mode 100755 index 0000000000..506b8bf5af --- /dev/null +++ b/.husky/post-checkout @@ -0,0 +1,4 @@ +# .husky/post-checkout +# ... + +git config submodule.recurse true \ No newline at end of file diff --git a/.nvmrc b/.nvmrc index 436b34d932..835d07c442 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v14.20.1 \ No newline at end of file +v14.20.1 diff --git a/.python-version b/.python-version index e06d07afe1..30291cba22 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.10.0 \ No newline at end of file +3.10.0 diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 6e667d23a8..ac35929be1 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -144,8 +144,6 @@ The following commands can be executed to manually get Budibase up and running ( `yarn` to install project dependencies -`yarn bootstrap` will install all budibase modules and symlink them together using lerna. - `yarn build` will build all budibase packages. #### 4. Running @@ -243,7 +241,7 @@ An overview of the CI pipelines can be found [here](../.github/workflows/README. Note that only budibase maintainers will be able to access the pro repo. -The `yarn bootstrap` command can be used to replace the NPM supplied dependency with the local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev. +By default, NX will make sure that dependencies are replaced with local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev. ### Troubleshooting diff --git a/docs/DEV-SETUP-DEBIAN.md b/docs/DEV-SETUP-DEBIAN.md index cfd7eebf47..a8b1e3dce4 100644 --- a/docs/DEV-SETUP-DEBIAN.md +++ b/docs/DEV-SETUP-DEBIAN.md @@ -1,13 +1,17 @@ ## Dev Environment on Debian 11 ### Install NVM & Node 14 + NVM documentation: https://github.com/nvm-sh/nvm#installing-and-updating Install NVM + ``` curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash ``` + Install Node 14 + ``` nvm install 14 ``` @@ -17,13 +21,16 @@ nvm install 14 ``` npm install -g yarn jest lerna ``` + ### Install Docker and Docker Compose ``` apt install docker.io pip3 install docker-compose ``` + ### Clone the repo + ``` git clone https://github.com/Budibase/budibase.git ``` @@ -44,10 +51,13 @@ This setup process was tested on Debian 11 (bullseye) with version numbers show cd budibase yarn setup ``` + The yarn setup command runs several build steps i.e. + ``` node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev ``` + So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose. The dev version will be available on port 10000 i.e. @@ -55,6 +65,7 @@ The dev version will be available on port 10000 i.e. http://127.0.0.1:10000/builder/admin ### File descriptor issues with Vite and Chrome in Linux + If your dev environment stalls forever, with some network requests stuck in flight, it's likely that Chrome is trying to open more file descriptors than your system allows. To fix this, apply the following tweaks. @@ -62,4 +73,4 @@ Debian based distros: Add `* - nofile 65536` to `/etc/security/limits.conf`. Arch: -Add `DefaultLimitNOFILE=65536` to `/etc/systemd/system.conf`. \ No newline at end of file +Add `DefaultLimitNOFILE=65536` to `/etc/systemd/system.conf`. diff --git a/docs/DEV-SETUP-MACOSX.md b/docs/DEV-SETUP-MACOSX.md index 67eb5506ff..94ed3fc1ee 100644 --- a/docs/DEV-SETUP-MACOSX.md +++ b/docs/DEV-SETUP-MACOSX.md @@ -4,14 +4,14 @@ Install instructions [here](https://brew.sh/) -| **NOTE**: If you are working on a M1 Apple Silicon which is running Z shell, you could need to add -`eval $(/opt/homebrew/bin/brew shellenv)` line to your `.zshrc`. This will make your zsh to find the apps you install +| **NOTE**: If you are working on a M1 Apple Silicon which is running Z shell, you could need to add +`eval $(/opt/homebrew/bin/brew shellenv)` line to your `.zshrc`. This will make your zsh to find the apps you install through brew. - ### Install Node Budibase requires a recent version of node 14: + ``` brew install node npm node -v @@ -22,12 +22,15 @@ node -v ``` npm install -g yarn jest lerna ``` + ### Install Docker and Docker Compose ``` brew install docker docker-compose ``` + ### Clone the repo + ``` git clone https://github.com/Budibase/budibase.git ``` @@ -48,10 +51,13 @@ This setup process was tested on Mac OSX 12 (Monterey) with version numbers show cd budibase yarn setup ``` + The yarn setup command runs several build steps i.e. + ``` node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev ``` + So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose. The dev version will be available on port 10000 i.e. diff --git a/docs/DEV-SETUP-WINDOWS.md b/docs/DEV-SETUP-WINDOWS.md index c5608b7567..176e0700d7 100644 --- a/docs/DEV-SETUP-WINDOWS.md +++ b/docs/DEV-SETUP-WINDOWS.md @@ -1,13 +1,15 @@ ## Dev Environment on Windows 10/11 (WSL2) - ### Install WSL with Ubuntu LTS Enable WSL 2 on Windows 10/11 for docker support. + ``` wsl --set-default-version 2 ``` + Install Ubuntu LTS. + ``` wsl --install Ubuntu ``` @@ -16,6 +18,7 @@ Or follow the instruction here: https://learn.microsoft.com/en-us/windows/wsl/install ### Install Docker in windows + Download the installer from docker and install it. Check this url for more detailed instructions: @@ -24,18 +27,21 @@ https://docs.docker.com/desktop/install/windows-install/ You should follow the next steps from within the Ubuntu terminal. ### Install NVM & Node 14 + NVM documentation: https://github.com/nvm-sh/nvm#installing-and-updating Install NVM + ``` curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash ``` + Install Node 14 + ``` nvm install 14 ``` - ### Install npm requirements ``` @@ -43,6 +49,7 @@ npm install -g yarn jest lerna ``` ### Clone the repo + ``` git clone https://github.com/Budibase/budibase.git ``` @@ -63,10 +70,13 @@ This setup process was tested on Windows 11 with version numbers show below. You cd budibase yarn setup ``` + The yarn setup command runs several build steps i.e. + ``` node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev ``` + So this command will actually run the application in dev mode. It creates .env files under `./packages/server` and `./packages/worker` and runs docker containers for each service via docker-compose. The dev version will be available on port 10000 i.e. @@ -74,8 +84,9 @@ The dev version will be available on port 10000 i.e. http://127.0.0.1:10000/builder/admin ### Working with the code + Here are the instructions to work on the application from within Visual Studio Code (in Windows) through the WSL. All the commands and files are within the Ubuntu system and it should run as if you were working on a Linux machine. https://code.visualstudio.com/docs/remote/wsl -Note you will be able to run the application from within the WSL terminal and you will be able to access the application from the a browser in Windows. \ No newline at end of file +Note you will be able to run the application from within the WSL terminal and you will be able to access the application from the a browser in Windows. diff --git a/hosting/docker-compose.build.yaml b/hosting/docker-compose.build.yaml new file mode 100644 index 0000000000..391d263098 --- /dev/null +++ b/hosting/docker-compose.build.yaml @@ -0,0 +1,77 @@ +version: "3" + +# optional ports are specified throughout for more advanced use cases. + +services: + app-service: + build: ../packages/server + container_name: build-bbapps + environment: + SELF_HOSTED: 1 + COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984 + WORKER_URL: http://worker-service:4003 + MINIO_URL: http://minio-service:9000 + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + INTERNAL_API_KEY: ${INTERNAL_API_KEY} + BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT} + PORT: 4002 + API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY} + JWT_SECRET: ${JWT_SECRET} + LOG_LEVEL: info + SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131 + ENABLE_ANALYTICS: "true" + REDIS_URL: redis-service:6379 + REDIS_PASSWORD: ${REDIS_PASSWORD} + BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL} + BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD} + PLUGINS_DIR: ${PLUGINS_DIR} + depends_on: + - worker-service + - redis-service + # volumes: + # - /some/path/to/plugins:/plugins + + worker-service: + build: ../packages/worker + container_name: build-bbworker + environment: + SELF_HOSTED: 1 + PORT: 4003 + CLUSTER_PORT: ${MAIN_PORT} + API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY} + JWT_SECRET: ${JWT_SECRET} + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} + MINIO_URL: http://minio-service:9000 + APPS_URL: http://app-service:4002 + COUCH_DB_USERNAME: ${COUCH_DB_USER} + COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD} + COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984 + SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131 + INTERNAL_API_KEY: ${INTERNAL_API_KEY} + REDIS_URL: redis-service:6379 + REDIS_PASSWORD: ${REDIS_PASSWORD} + depends_on: + - redis-service + - minio-service + + proxy-service-docker: + ports: + - "${MAIN_PORT}:10000" + container_name: build-bbproxy + image: budibase/proxy + environment: + - PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10 + - PROXY_RATE_LIMIT_API_PER_SECOND=20 + - APPS_UPSTREAM_URL=http://app-service:4002 + - WORKER_UPSTREAM_URL=http://worker-service:4003 + - MINIO_UPSTREAM_URL=http://minio-service:9000 + - COUCHDB_UPSTREAM_URL=http://couchdb-service:5984 + - WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080 + - RESOLVER=127.0.0.11 + depends_on: + - minio-service + - worker-service + - app-service + - couchdb-service diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index fadcf235e9..64a6b01365 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -1,22 +1,22 @@ -FROM node:14-slim as build +FROM node:16-slim as build # install node-gyp dependencies RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends apt-utils cron g++ make python # add pin script WORKDIR / -ADD scripts/pinVersions.js scripts/cleanup.sh ./ +ADD scripts/cleanup.sh ./ RUN chmod +x /cleanup.sh # build server WORKDIR /app ADD packages/server . -RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh +RUN yarn install --frozen-lockfile --production=true && /cleanup.sh # build worker WORKDIR /worker ADD packages/worker . -RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh +RUN yarn install --frozen-lockfile --production=true && /cleanup.sh FROM budibase/couchdb ARG TARGETARCH @@ -31,9 +31,7 @@ COPY --from=build /worker /worker # install base dependencies RUN apt-get update && \ - apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server && \ - apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security/updates main' && \ - apt-get update + apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server # install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx WORKDIR /nodejs diff --git a/lerna.json b/lerna.json index 39c3634921..9f8cd12e31 100644 --- a/lerna.json +++ b/lerna.json @@ -1,8 +1,22 @@ { - "version": "2.6.23", + "version": "2.6.24-alpha.0", "npmClient": "yarn", + "packages": [ + "packages/backend-core", + "packages/bbui", + "packages/builder", + "packages/cli", + "packages/client", + "packages/frontend-core", + "packages/sdk", + "packages/server", + "packages/shared-core", + "packages/string-templates", + "packages/types", + "packages/worker", + "packages/pro/packages/pro" + ], "useWorkspaces": true, - "packages": ["packages/*"], "command": { "publish": { "ignoreChanges": [ @@ -17,4 +31,4 @@ "loadEnvFiles": false } } -} +} \ No newline at end of file diff --git a/nx.json b/nx.json index efb577a01b..fc0712eed4 100644 --- a/nx.json +++ b/nx.json @@ -6,5 +6,15 @@ "cacheableOperations": ["build", "test"] } } + }, + "targetDefaults": { + "dev:builder": { + "dependsOn": [ + { + "projects": ["@budibase/string-templates"], + "target": "build" + } + ] + } } } diff --git a/package.json b/package.json index 29b7c7c723..3849c65274 100644 --- a/package.json +++ b/package.json @@ -2,37 +2,44 @@ "name": "root", "private": true, "devDependencies": { + "@esbuild-plugins/node-resolve": "^0.2.2", + "@esbuild-plugins/tsconfig-paths": "^0.1.2", + "@nx/js": "16.2.1", "@rollup/plugin-json": "^4.0.2", "@typescript-eslint/parser": "5.45.0", "babel-eslint": "^10.0.3", + "esbuild": "^0.17.18", "eslint": "^7.28.0", "eslint-plugin-cypress": "^2.11.3", "eslint-plugin-svelte3": "^3.2.0", - "husky": "^7.0.1", + "husky": "^8.0.3", "js-yaml": "^4.1.0", "kill-port": "^1.6.1", - "lerna": "^6.6.1", + "lerna": "7.0.0-alpha.0", "madge": "^6.0.0", + "minimist": "^1.2.8", + "nx": "^16.2.1", "prettier": "^2.3.1", "prettier-plugin-svelte": "^2.3.0", "rimraf": "^3.0.2", "rollup-plugin-replace": "^2.2.0", + "semver": "^7.5.0", "svelte": "^3.38.2", "typescript": "4.7.3" }, "scripts": { - "setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev", - "bootstrap": "lerna link && ./scripts/link-dependencies.sh", - "build": "lerna run --stream build", - "build:dev": "lerna run --stream prebuild && tsc --build --watch --preserveWatchOutput", + "preinstall": "node scripts/syncProPackage.js", + "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev", + "bootstrap": "./scripts/link-dependencies.sh && echo '***BOOTSTRAP ONLY REQUIRED FOR USE WITH ACCOUNT PORTAL***'", + "build": "yarn nx run-many -t=build", + "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", + "check:types": "lerna run check:types --skip-nx-cache", "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap", "backend:build": "./scripts/scopeBackend.sh 'lerna run --stream build'", "build:sdk": "lerna run --stream build:sdk", "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular", - "release": "lerna publish ${RELEASE_VERSION_TYPE:-patch} --yes --force-publish && yarn release:pro", - "release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop --exact && yarn release:pro:develop", - "release:pro": "bash scripts/pro/release.sh", - "release:pro:develop": "bash scripts/pro/release.sh develop", + "release": "lerna publish ${RELEASE_VERSION_TYPE:-patch} --yes --force-publish --no-git-tag-version --no-push --no-git-reset", + "release:develop": "lerna publish from-package --yes --force-publish --dist-tag develop --exact --no-git-tag-version --no-push --no-git-reset", "restore": "yarn run clean && yarn run bootstrap && yarn run build", "nuke": "yarn run nuke:packages && yarn run nuke:docker", "nuke:packages": "yarn run restore", @@ -41,12 +48,12 @@ "kill-builder": "kill-port 3000", "kill-server": "kill-port 4001 4002", "kill-all": "yarn run kill-builder && yarn run kill-server", - "dev": "yarn run kill-all && lerna link && lerna run --stream --parallel dev:builder --concurrency 1 --stream", - "dev:noserver": "yarn run kill-builder && lerna link && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --concurrency 1 --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", - "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server", + "dev": "yarn run kill-all && lerna run --stream --parallel dev:builder --stream", + "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", + "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --scope @budibase/worker --scope @budibase/server", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built", + "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", "test": "lerna run --stream test --stream", - "test:pro": "bash scripts/pro/test.sh", "lint:eslint": "eslint packages && eslint qa-core", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"", "lint": "yarn run lint:eslint && yarn run lint:prettier", @@ -54,16 +61,16 @@ "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"", "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint", "build:specs": "lerna run --stream specs", - "build:docker": "lerna run --stream build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", + "build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", "build:docker:pre": "lerna run --stream build && lerna run --stream predocker", "build:docker:proxy": "docker build hosting/proxy -t proxy-service", "build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -", - "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", + "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild", "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -", "build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .", "build:docker:single:image": "docker build -f hosting/single/Dockerfile -t budibase:latest .", - "build:docker:single": "npm run build:docker:pre && npm run build:docker:single:image", + "build:docker:single": "yarn build && lerna run --concurrency 1 predocker && yarn build:docker:single:image", "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting", "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.2.1 --push ./hosting/couchdb", "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting", @@ -82,12 +89,32 @@ "mode:account": "yarn mode:cloud && yarn env:account:enable", "security:audit": "node scripts/audit.js", "postinstall": "husky install", - "install:pro": "bash scripts/pro/install.sh", - "dep:clean": "yarn clean && yarn bootstrap" + "dep:clean": "yarn clean -y && yarn bootstrap", + "submodules:load": "git submodule init && git submodule update && yarn && yarn bootstrap", + "submodules:unload": "git submodule deinit --all && yarn && yarn bootstrap" }, "workspaces": { "packages": [ - "packages/*" + "packages/backend-core", + "packages/bbui", + "packages/builder", + "packages/cli", + "packages/client", + "packages/frontend-core", + "packages/sdk", + "packages/server", + "packages/shared-core", + "packages/string-templates", + "packages/types", + "packages/worker", + "packages/pro/packages/pro" ] - } + }, + "resolutions": { + "@budibase/backend-core": "0.0.0", + "@budibase/shared-core": "0.0.0", + "@budibase/string-templates": "0.0.0", + "@budibase/types": "0.0.0" + }, + "dependencies": {} } diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 1f3feea7a9..f85687b007 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.6.23", + "version": "0.0.0", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -15,8 +15,6 @@ "prebuild": "rimraf dist/", "prepack": "cp package.json dist", "build": "tsc -p tsconfig.build.json", - "build:pro": "../../scripts/pro/build.sh", - "postbuild": "yarn run build:pro", "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput", "test": "bash scripts/test.sh", "test:watch": "jest --watchAll" @@ -24,7 +22,7 @@ "dependencies": { "@budibase/nano": "10.1.2", "@budibase/pouchdb-replication-stream": "1.2.10", - "@budibase/types": "^2.6.23", + "@budibase/types": "0.0.0", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", @@ -35,7 +33,7 @@ "correlation-id": "4.0.0", "dotenv": "16.0.1", "emitter-listener": "1.1.2", - "ioredis": "4.28.0", + "ioredis": "5.3.2", "joi": "17.6.0", "jsonwebtoken": "9.0.0", "koa-passport": "4.1.4", @@ -64,7 +62,6 @@ "@swc/jest": "^0.2.24", "@trendyol/jest-testcontainers": "^2.1.1", "@types/chance": "1.1.3", - "@types/ioredis": "4.28.0", "@types/jest": "29.5.0", "@types/koa": "2.13.4", "@types/lodash": "4.14.180", @@ -76,7 +73,7 @@ "@types/tar-fs": "2.0.1", "@types/uuid": "8.3.4", "chance": "1.1.8", - "ioredis-mock": "5.8.0", + "ioredis-mock": "8.7.0", "jest": "29.5.0", "jest-environment-node": "29.5.0", "jest-serial-runner": "^1.2.1", @@ -90,5 +87,19 @@ "tsconfig-paths": "4.0.0", "typescript": "4.7.3" }, + "nx": { + "targets": { + "build": { + "dependsOn": [ + { + "projects": [ + "@budibase/types" + ], + "target": "build" + } + ] + } + } + }, "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc" } diff --git a/packages/backend-core/src/cache/tests/writethrough.spec.ts b/packages/backend-core/src/cache/tests/writethrough.spec.ts index e4c7cc6e64..92b073ed64 100644 --- a/packages/backend-core/src/cache/tests/writethrough.spec.ts +++ b/packages/backend-core/src/cache/tests/writethrough.spec.ts @@ -72,16 +72,12 @@ describe("writethrough", () => { writethrough.put({ ...current, value: 4 }), ]) + // with a lock, this will work const newRev = responses.map(x => x.rev).find(x => x !== current._rev) expect(newRev).toBeDefined() expect(responses.map(x => x.rev)).toEqual( expect.arrayContaining([current._rev, current._rev, newRev]) ) - expectFunctionWasCalledTimesWith( - mocks.alerts.logWarn, - 2, - "Ignoring redlock conflict in write-through cache" - ) const output = await db.get(current._id) expect(output.value).toBe(4) diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index aa40f13775..be49b9f261 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -21,7 +21,7 @@ export enum ViewName { AUTOMATION_LOGS = "automation_logs", ACCOUNT_BY_EMAIL = "account_by_email", PLATFORM_USERS_LOWERCASE = "platform_users_lowercase", - USER_BY_GROUP = "by_group_user", + USER_BY_GROUP = "user_by_group", APP_BACKUP_BY_TRIGGER = "by_trigger", } diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts index 2bb8f815cf..ba2533cf4a 100644 --- a/packages/backend-core/src/constants/misc.ts +++ b/packages/backend-core/src/constants/misc.ts @@ -16,6 +16,7 @@ export enum Header { LICENSE_KEY = "x-budibase-license-key", API_VER = "x-budibase-api-version", APP_ID = "x-budibase-app-id", + SESSION_ID = "x-budibase-session-id", TYPE = "x-budibase-type", PREVIEW_ROLE = "x-budibase-role", TENANT_ID = "x-budibase-tenant-id", diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 94d78e94ff..29ca4123f5 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -12,7 +12,7 @@ import { isDocument, } from "@budibase/types" import { getCouchInfo } from "./connections" -import { directCouchCall } from "./utils" +import { directCouchUrlCall } from "./utils" import { getPouchDB } from "./pouchDB" import { WriteStream, ReadStream } from "fs" import { newid } from "../../docIds/newid" @@ -46,6 +46,8 @@ export class DatabaseImpl implements Database { private readonly instanceNano?: Nano.ServerScope private readonly pouchOpts: DatabaseOpts + private readonly couchInfo = getCouchInfo() + constructor(dbName?: string, opts?: DatabaseOpts, connection?: string) { if (dbName == null) { throw new Error("Database name cannot be undefined.") @@ -53,8 +55,8 @@ export class DatabaseImpl implements Database { this.name = dbName this.pouchOpts = opts || {} if (connection) { - const couchInfo = getCouchInfo(connection) - this.instanceNano = buildNano(couchInfo) + this.couchInfo = getCouchInfo(connection) + this.instanceNano = buildNano(this.couchInfo) } if (!DatabaseImpl.nano) { DatabaseImpl.init() @@ -67,7 +69,11 @@ export class DatabaseImpl implements Database { } async exists() { - let response = await directCouchCall(`/${this.name}`, "HEAD") + const response = await directCouchUrlCall({ + url: `${this.couchInfo.url}/${this.name}`, + method: "HEAD", + cookie: this.couchInfo.cookie, + }) return response.status === 200 } diff --git a/packages/backend-core/src/db/couch/connections.ts b/packages/backend-core/src/db/couch/connections.ts index 06c661f350..4214c7cdc6 100644 --- a/packages/backend-core/src/db/couch/connections.ts +++ b/packages/backend-core/src/db/couch/connections.ts @@ -4,21 +4,21 @@ export const getCouchInfo = (connection?: string) => { const urlInfo = getUrlInfo(connection) let username let password - if (env.COUCH_DB_USERNAME) { - // set from env - username = env.COUCH_DB_USERNAME - } else if (urlInfo.auth.username) { + if (urlInfo.auth?.username) { // set from url username = urlInfo.auth.username + } else if (env.COUCH_DB_USERNAME) { + // set from env + username = env.COUCH_DB_USERNAME } else if (!env.isTest()) { throw new Error("CouchDB username not set") } - if (env.COUCH_DB_PASSWORD) { - // set from env - password = env.COUCH_DB_PASSWORD - } else if (urlInfo.auth.password) { + if (urlInfo.auth?.password) { // set from url password = urlInfo.auth.password + } else if (env.COUCH_DB_PASSWORD) { + // set from env + password = env.COUCH_DB_PASSWORD } else if (!env.isTest()) { throw new Error("CouchDB password not set") } diff --git a/packages/backend-core/src/db/couch/utils.ts b/packages/backend-core/src/db/couch/utils.ts index 426bf92158..51b2a38998 100644 --- a/packages/backend-core/src/db/couch/utils.ts +++ b/packages/backend-core/src/db/couch/utils.ts @@ -9,6 +9,20 @@ export async function directCouchCall( ) { let { url, cookie } = getCouchInfo() const couchUrl = `${url}/${path}` + return await directCouchUrlCall({ url: couchUrl, cookie, method, body }) +} + +export async function directCouchUrlCall({ + url, + cookie, + method, + body, +}: { + url: string + cookie: string + method: string + body?: any +}) { const params: any = { method: method, headers: { @@ -19,7 +33,7 @@ export async function directCouchCall( params.body = JSON.stringify(body) params.headers["Content-Type"] = "application/json" } - return await fetch(checkSlashesInUrl(encodeURI(couchUrl)), params) + return await fetch(checkSlashesInUrl(encodeURI(url)), params) } export async function directCouchQuery( diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 9163dfeba6..eab8cd4c45 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -97,7 +97,6 @@ const environment = { REDIS_URL: process.env.REDIS_URL || "localhost:6379", REDIS_PASSWORD: process.env.REDIS_PASSWORD, REDIS_CLUSTERED: process.env.REDIS_CLUSTERED, - MOCK_REDIS: process.env.MOCK_REDIS, MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, AWS_REGION: process.env.AWS_REGION, @@ -129,6 +128,7 @@ const environment = { PLUGIN_BUCKET_NAME: process.env.PLUGIN_BUCKET_NAME || DefaultBucketName.PLUGINS, USE_COUCH: process.env.USE_COUCH || true, + MOCK_REDIS: process.env.MOCK_REDIS, DEFAULT_LICENSE: process.env.DEFAULT_LICENSE, SERVICE: process.env.SERVICE || "budibase", LOG_LEVEL: process.env.LOG_LEVEL || "info", diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index 40233b3827..7b98674788 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -21,6 +21,7 @@ export * as context from "./context" export * as cache from "./cache" export * as objectStore from "./objectStore" export * as redis from "./redis" +export { Client as RedisClient } from "./redis" export * as locks from "./redis/redlockImpl" export * as utils from "./utils" export * as errors from "./errors" diff --git a/packages/backend-core/src/logging/pino/logger.ts b/packages/backend-core/src/logging/pino/logger.ts index cebc78ffc7..c96bc83e04 100644 --- a/packages/backend-core/src/logging/pino/logger.ts +++ b/packages/backend-core/src/logging/pino/logger.ts @@ -96,6 +96,7 @@ if (!env.DISABLE_PINO_LOGGER) { const mergingObject: any = { err: error, + pid: process.pid, ...contextObject, } diff --git a/packages/backend-core/src/redis/init.ts b/packages/backend-core/src/redis/init.ts index 485268edad..55ffe3dd12 100644 --- a/packages/backend-core/src/redis/init.ts +++ b/packages/backend-core/src/redis/init.ts @@ -6,7 +6,8 @@ let userClient: Client, appClient: Client, cacheClient: Client, writethroughClient: Client, - lockClient: Client + lockClient: Client, + socketClient: Client async function init() { userClient = await new Client(utils.Databases.USER_CACHE).init() @@ -14,9 +15,10 @@ async function init() { appClient = await new Client(utils.Databases.APP_METADATA).init() cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init() lockClient = await new Client(utils.Databases.LOCKS).init() - writethroughClient = await new Client( - utils.Databases.WRITE_THROUGH, - utils.SelectableDatabase.WRITE_THROUGH + writethroughClient = await new Client(utils.Databases.WRITE_THROUGH).init() + socketClient = await new Client( + utils.Databases.SOCKET_IO, + utils.SelectableDatabase.SOCKET_IO ).init() } @@ -27,6 +29,7 @@ export async function shutdown() { if (cacheClient) await cacheClient.finish() if (writethroughClient) await writethroughClient.finish() if (lockClient) await lockClient.finish() + if (socketClient) await socketClient.finish() } process.on("exit", async () => { @@ -74,3 +77,10 @@ export async function getLockClient() { } return lockClient } + +export async function getSocketClient() { + if (!socketClient) { + await init() + } + return socketClient +} diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 2d54b51a9f..5056a5d549 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -1,6 +1,15 @@ import env from "../environment" -// ioredis mock is all in memory -const Redis = env.MOCK_REDIS ? require("ioredis-mock") : require("ioredis") +import Redis from "ioredis" +// mock-redis doesn't have any typing +let MockRedis: any | undefined +if (env.MOCK_REDIS) { + try { + // ioredis mock is all in memory + MockRedis = require("ioredis-mock") + } catch (err) { + console.log("Mock redis unavailable") + } +} import { addDbPrefix, removeDbPrefix, @@ -18,7 +27,7 @@ const DEFAULT_SELECT_DB = SelectableDatabase.DEFAULT // for testing just generate the client once let CLOSED = false let CLIENTS: { [key: number]: any } = {} - +0 let CONNECTED = false // mock redis always connected @@ -55,6 +64,7 @@ function connectionError( * will return the ioredis client which will be ready to use. */ function init(selectDb = DEFAULT_SELECT_DB) { + const RedisCore = env.MOCK_REDIS && MockRedis ? MockRedis : Redis let timeout: NodeJS.Timeout CLOSED = false let client = pickClient(selectDb) @@ -64,7 +74,7 @@ function init(selectDb = DEFAULT_SELECT_DB) { } // testing uses a single in memory client if (env.MOCK_REDIS) { - CLIENTS[selectDb] = new Redis(getRedisOptions()) + CLIENTS[selectDb] = new RedisCore(getRedisOptions()) } // start the timer - only allowed 5 seconds to connect timeout = setTimeout(() => { @@ -84,11 +94,11 @@ function init(selectDb = DEFAULT_SELECT_DB) { const { redisProtocolUrl, opts, host, port } = getRedisOptions() if (CLUSTERED) { - client = new Redis.Cluster([{ host, port }], opts) + client = new RedisCore.Cluster([{ host, port }], opts) } else if (redisProtocolUrl) { - client = new Redis(redisProtocolUrl) + client = new RedisCore(redisProtocolUrl) } else { - client = new Redis(opts) + client = new RedisCore(opts) } // attach handlers client.on("end", (err: Error) => { @@ -183,6 +193,9 @@ class RedisWrapper { CLOSED = false init(this._select) await waitForConnection(this._select) + if (this._select && !env.isTest()) { + this.getClient().select(this._select) + } return this } @@ -209,6 +222,11 @@ class RedisWrapper { return this.getClient().keys(addDbPrefix(db, pattern)) } + async exists(key: string) { + const db = this._db + return await this.getClient().exists(addDbPrefix(db, key)) + } + async get(key: string) { const db = this._db let response = await this.getClient().get(addDbPrefix(db, key)) diff --git a/packages/backend-core/src/redis/redlockImpl.ts b/packages/backend-core/src/redis/redlockImpl.ts index 55b891ea84..7fe61a409e 100644 --- a/packages/backend-core/src/redis/redlockImpl.ts +++ b/packages/backend-core/src/redis/redlockImpl.ts @@ -4,10 +4,10 @@ import { LockOptions, LockType } from "@budibase/types" import * as context from "../context" import env from "../environment" -const getClient = async ( +async function getClient( type: LockType, opts?: Redlock.Options -): Promise => { +): Promise { if (type === LockType.CUSTOM) { return newRedlock(opts) } @@ -18,6 +18,9 @@ const getClient = async ( case LockType.TRY_ONCE: { return newRedlock(OPTIONS.TRY_ONCE) } + case LockType.TRY_TWICE: { + return newRedlock(OPTIONS.TRY_TWICE) + } case LockType.DEFAULT: { return newRedlock(OPTIONS.DEFAULT) } @@ -35,6 +38,9 @@ const OPTIONS = { // immediately throws an error if the lock is already held retryCount: 0, }, + TRY_TWICE: { + retryCount: 1, + }, TEST: { // higher retry count in unit tests // due to high contention. @@ -62,7 +68,7 @@ const OPTIONS = { }, } -const newRedlock = async (opts: Redlock.Options = {}) => { +export async function newRedlock(opts: Redlock.Options = {}) { let options = { ...OPTIONS.DEFAULT, ...opts } const redisWrapper = await getLockClient() const client = redisWrapper.getClient() @@ -81,22 +87,26 @@ type RedlockExecution = | SuccessfulRedlockExecution | UnsuccessfulRedlockExecution -export const doWithLock = async ( +function getLockName(opts: LockOptions) { + // determine lock name + // by default use the tenantId for uniqueness, unless using a system lock + const prefix = opts.systemLock ? "system" : context.getTenantId() + let name: string = `lock:${prefix}_${opts.name}` + // add additional unique name if required + if (opts.resource) { + name = name + `_${opts.resource}` + } + return name +} + +export async function doWithLock( opts: LockOptions, task: () => Promise -): Promise> => { +): Promise> { const redlock = await getClient(opts.type, opts.customOptions) let lock try { - // determine lock name - // by default use the tenantId for uniqueness, unless using a system lock - const prefix = opts.systemLock ? "system" : context.getTenantId() - let name: string = `lock:${prefix}_${opts.name}` - - // add additional unique name if required - if (opts.resource) { - name = name + `_${opts.resource}` - } + const name = getLockName(opts) // create the lock lock = await redlock.lock(name, opts.ttl) @@ -112,7 +122,6 @@ export const doWithLock = async ( if (opts.type === LockType.TRY_ONCE) { // don't throw for try-once locks, they will always error // due to retry count (0) exceeded - console.warn(e) return { executed: false } } else { console.error(e) diff --git a/packages/backend-core/src/redis/utils.ts b/packages/backend-core/src/redis/utils.ts index 2c49ee4941..34b7275a2b 100644 --- a/packages/backend-core/src/redis/utils.ts +++ b/packages/backend-core/src/redis/utils.ts @@ -27,6 +27,7 @@ export enum Databases { GENERIC_CACHE = "data_cache", WRITE_THROUGH = "writeThrough", LOCKS = "locks", + SOCKET_IO = "socket_io", } /** @@ -40,7 +41,7 @@ export enum Databases { */ export enum SelectableDatabase { DEFAULT = 0, - WRITE_THROUGH = 1, + SOCKET_IO = 1, UNUSED_1 = 2, UNUSED_2 = 3, UNUSED_3 = 4, @@ -94,7 +95,7 @@ export function getRedisOptions() { opts.port = port opts.password = password } - return { opts, host, port, redisProtocolUrl } + return { opts, host, port: parseInt(port), redisProtocolUrl } } export function addDbPrefix(db: string, key: string) { diff --git a/packages/backend-core/src/utils/tests/utils.spec.ts b/packages/backend-core/src/utils/tests/utils.spec.ts index ededa48628..5a0ac4f283 100644 --- a/packages/backend-core/src/utils/tests/utils.spec.ts +++ b/packages/backend-core/src/utils/tests/utils.spec.ts @@ -5,6 +5,7 @@ import * as db from "../../db" import { Header } from "../../constants" import { newid } from "../../utils" import env from "../../environment" +import { BBContext } from "@budibase/types" describe("utils", () => { const config = new DBTestConfiguration() @@ -106,4 +107,85 @@ describe("utils", () => { expect(actual).toBe(undefined) }) }) + + describe("isServingBuilder", () => { + let ctx: BBContext + + const expectResult = (result: boolean) => + expect(utils.isServingBuilder(ctx)).toBe(result) + + beforeEach(() => { + ctx = structures.koa.newContext() + }) + + it("returns true if current path is in builder", async () => { + ctx.path = "/builder/app/app_" + expectResult(true) + }) + + it("returns false if current path doesn't have '/' suffix", async () => { + ctx.path = "/builder/app" + expectResult(false) + + ctx.path = "/xx" + expectResult(false) + }) + }) + + describe("isServingBuilderPreview", () => { + let ctx: BBContext + + const expectResult = (result: boolean) => + expect(utils.isServingBuilderPreview(ctx)).toBe(result) + + beforeEach(() => { + ctx = structures.koa.newContext() + }) + + it("returns true if current path is in builder preview", async () => { + ctx.path = "/app/preview/xx" + expectResult(true) + }) + + it("returns false if current path is not in builder preview", async () => { + ctx.path = "/builder" + expectResult(false) + + ctx.path = "/xx" + expectResult(false) + }) + }) + + describe("isPublicAPIRequest", () => { + let ctx: BBContext + + const expectResult = (result: boolean) => + expect(utils.isPublicApiRequest(ctx)).toBe(result) + + beforeEach(() => { + ctx = structures.koa.newContext() + }) + + it("returns true if current path remains to public API", async () => { + ctx.path = "/api/public/v1/invoices" + expectResult(true) + + ctx.path = "/api/public/v1" + expectResult(true) + + ctx.path = "/api/public/v2" + expectResult(true) + + ctx.path = "/api/public/v21" + expectResult(true) + }) + + it("returns false if current path doesn't remain to public API", async () => { + ctx.path = "/api/public" + expectResult(false) + + ctx.path = "/xx" + expectResult(false) + }) + }) }) diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index 75b098093b..82da95983a 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -1,11 +1,5 @@ -import { getAllApps, queryGlobalView } from "../db" -import { - Header, - MAX_VALID_DATE, - DocumentType, - SEPARATOR, - ViewName, -} from "../constants" +import { getAllApps } from "../db" +import { Header, MAX_VALID_DATE, DocumentType, SEPARATOR } from "../constants" import env from "../environment" import * as tenancy from "../tenancy" import * as context from "../context" @@ -23,7 +17,9 @@ const APP_PREFIX = DocumentType.APP + SEPARATOR const PROD_APP_PREFIX = "/app/" const BUILDER_PREVIEW_PATH = "/app/preview" -const BUILDER_REFERER_PREFIX = "/builder/app/" +const BUILDER_PREFIX = "/builder" +const BUILDER_APP_PREFIX = `${BUILDER_PREFIX}/app/` +const PUBLIC_API_PREFIX = "/api/public/v" function confirmAppId(possibleAppId: string | undefined) { return possibleAppId && possibleAppId.startsWith(APP_PREFIX) @@ -69,6 +65,18 @@ export function isServingApp(ctx: Ctx) { return false } +export function isServingBuilder(ctx: Ctx): boolean { + return ctx.path.startsWith(BUILDER_APP_PREFIX) +} + +export function isServingBuilderPreview(ctx: Ctx): boolean { + return ctx.path.startsWith(BUILDER_PREVIEW_PATH) +} + +export function isPublicApiRequest(ctx: Ctx): boolean { + return ctx.path.startsWith(PUBLIC_API_PREFIX) +} + /** * Given a request tries to find the appId, which can be located in various places * @param {object} ctx The main request body to look through. @@ -110,7 +118,7 @@ export async function getAppIdFromCtx(ctx: Ctx) { // make sure this is performed after prod app url resolution, in case the // referer header is present from a builder redirect const referer = ctx.request.headers.referer - if (!appId && referer?.includes(BUILDER_REFERER_PREFIX)) { + if (!appId && referer?.includes(BUILDER_APP_PREFIX)) { const refererId = parseAppIdFromUrl(ctx.request.headers.referer) appId = confirmAppId(refererId) } diff --git a/packages/backend-core/tests/core/utilities/mocks/licenses.ts b/packages/backend-core/tests/core/utilities/mocks/licenses.ts index 839b22e5f9..4272e78eb8 100644 --- a/packages/backend-core/tests/core/utilities/mocks/licenses.ts +++ b/packages/backend-core/tests/core/utilities/mocks/licenses.ts @@ -90,6 +90,10 @@ export const useScimIntegration = () => { return useFeature(Feature.SCIM) } +export const useSyncAutomations = () => { + return useFeature(Feature.SYNC_AUTOMATIONS) +} + // QUOTAS export const setAutomationLogsQuota = (value: number) => { diff --git a/packages/backend-core/tsconfig.json b/packages/backend-core/tsconfig.json index e95fb9ab4d..2b1419b051 100644 --- a/packages/backend-core/tsconfig.json +++ b/packages/backend-core/tsconfig.json @@ -7,11 +7,6 @@ "@budibase/types": ["../types/src"] } }, - "references": [ - { "path": "../types" } - ], - "exclude": [ - "node_modules", - "dist", - ] -} \ No newline at end of file + + "exclude": ["node_modules", "dist"] +} diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 32c47b3c49..b03c83d71b 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.6.23", + "version": "0.0.0", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,8 +38,8 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "1.2.1", - "@budibase/shared-core": "^2.6.23", - "@budibase/string-templates": "^2.6.23", + "@budibase/shared-core": "0.0.0", + "@budibase/string-templates": "0.0.0", "@spectrum-css/accordion": "3.0.24", "@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actiongroup": "1.0.1", @@ -90,5 +90,19 @@ "resolutions": { "loader-utils": "1.4.1" }, + "nx": { + "targets": { + "build": { + "dependsOn": [ + { + "projects": [ + "@budibase/string-templates" + ], + "target": "build" + } + ] + } + } + }, "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc" } diff --git a/packages/bbui/src/ActionButton/ActionButton.svelte b/packages/bbui/src/ActionButton/ActionButton.svelte index e9eb3b3471..427a98f888 100644 --- a/packages/bbui/src/ActionButton/ActionButton.svelte +++ b/packages/bbui/src/ActionButton/ActionButton.svelte @@ -102,7 +102,9 @@ margin-left: 0; transition: color ease-out 130ms; } - .is-selected:not(.spectrum-ActionButton--emphasized):not(.spectrum-ActionButton--quiet) { + .is-selected:not(.spectrum-ActionButton--emphasized):not( + .spectrum-ActionButton--quiet + ) { background: var(--spectrum-global-color-gray-300); border-color: var(--spectrum-global-color-gray-500); } diff --git a/packages/bbui/src/Actions/position_dropdown.js b/packages/bbui/src/Actions/position_dropdown.js index 2cb681670b..01555446d9 100644 --- a/packages/bbui/src/Actions/position_dropdown.js +++ b/packages/bbui/src/Actions/position_dropdown.js @@ -56,6 +56,8 @@ export default function positionDropdown(element, opts) { styles.left = anchorBounds.left + anchorBounds.width - elementBounds.width } else if (align === "right-outside") { styles.left = anchorBounds.right + offset + } else if (align === "left-outside") { + styles.left = anchorBounds.left - elementBounds.width - offset } else { styles.left = anchorBounds.left } diff --git a/packages/bbui/src/Avatar/Avatar.svelte b/packages/bbui/src/Avatar/Avatar.svelte index 1e4cefd8ce..0faf50f55a 100644 --- a/packages/bbui/src/Avatar/Avatar.svelte +++ b/packages/bbui/src/Avatar/Avatar.svelte @@ -13,10 +13,12 @@ export let url = "" export let disabled = false export let initials = "JD" + export let color = null const DefaultColor = "#3aab87" - $: color = getColor(initials) + $: avatarColor = color || getColor(initials) + $: style = getStyle(size, avatarColor) const getColor = initials => { if (!initials?.length) { @@ -26,6 +28,12 @@ const hue = ((code % 26) / 26) * 360 return `hsl(${hue}, 50%, 50%)` } + + const getStyle = (sizeKey, color) => { + const size = `var(${sizes.get(sizeKey)})` + const fontSize = `calc(${size} / 2)` + return `width:${size}; height:${size}; font-size:${fontSize}; background:${color};` + } {#if url} @@ -37,13 +45,7 @@ style="width: var({sizes.get(size)}); height: var({sizes.get(size)});" /> {:else} -
+
{initials || ""}
{/if} diff --git a/packages/bbui/src/Button/Button.svelte b/packages/bbui/src/Button/Button.svelte index f8a6004f8f..efd5f33bd2 100644 --- a/packages/bbui/src/Button/Button.svelte +++ b/packages/bbui/src/Button/Button.svelte @@ -2,6 +2,7 @@ import "@spectrum-css/button/dist/index-vars.css" import Tooltip from "../Tooltip/Tooltip.svelte" + export let type export let disabled = false export let size = "M" export let cta = false @@ -21,6 +22,7 @@ - -
- + {#if !headless} +
+
+ {title} + + + +
+
+ + +
+
+ {/if} {/if} diff --git a/packages/bbui/src/FancyForm/FancyField.svelte b/packages/bbui/src/FancyForm/FancyField.svelte index 89f2dec7d0..0c99394599 100644 --- a/packages/bbui/src/FancyForm/FancyField.svelte +++ b/packages/bbui/src/FancyForm/FancyField.svelte @@ -1,7 +1,7 @@ diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 70cb56c77d..6ab750c3d6 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -11,8 +11,8 @@ ActionButton, Drawer, Modal, - Detail, notifications, + Icon, } from "@budibase/bbui" import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte" import { automationStore, selectedAutomation } from "builderStore" @@ -27,9 +27,18 @@ import CronBuilder from "./CronBuilder.svelte" import Editor from "components/integration/QueryEditor.svelte" import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte" + import CodeEditor from "components/common/CodeEditor/CodeEditor.svelte" + import { + bindingsToCompletions, + jsAutocomplete, + EditorModes, + } from "components/common/CodeEditor" import FilterDrawer from "components/design/settings/controls/FilterEditor/FilterDrawer.svelte" import { LuceneUtils } from "@budibase/frontend-core" - import { getSchemaForTable } from "builderStore/dataBinding" + import { + getSchemaForTable, + getEnvironmentBindings, + } from "builderStore/dataBinding" import { Utils } from "@budibase/frontend-core" import { TriggerStepID, ActionStepID } from "constants/backend/automations" import { onMount } from "svelte" @@ -43,7 +52,6 @@ let webhookModal let drawer let fillWidth = true - let codeBindingOpen = false let inputData $: filters = lookForFilters(schemaProperties) || [] @@ -61,11 +69,63 @@ $: isTrigger = block?.type === "TRIGGER" $: isUpdateRow = stepId === ActionStepID.UPDATE_ROW + /** + * TODO - Remove after November 2023 + * ******************************* + * Code added to provide backwards compatibility between Values 1,2,3,4,5 + * and the new JSON body. + */ + let deprecatedSchemaProperties + $: { + if (block?.stepId === "integromat" || block?.stepId === "zapier") { + deprecatedSchemaProperties = schemaProperties.filter( + prop => !prop[0].startsWith("value") + ) + if (!deprecatedSchemaProperties.map(entry => entry[0]).includes("body")) { + deprecatedSchemaProperties.push([ + "body", + { + title: "Payload", + type: "json", + }, + ]) + } + } else { + deprecatedSchemaProperties = schemaProperties + } + } + /****************************************************/ + const getInputData = (testData, blockInputs) => { let newInputData = testData || blockInputs if (block.event === "app:trigger" && !newInputData?.fields) { newInputData = cloneDeep(blockInputs) } + + /** + * TODO - Remove after November 2023 + * ******************************* + * Code added to provide backwards compatibility between Values 1,2,3,4,5 + * and the new JSON body. + */ + if ( + (block?.stepId === "integromat" || block?.stepId === "zapier") && + !newInputData?.body?.value + ) { + let deprecatedValues = { + ...newInputData, + } + delete deprecatedValues.url + delete deprecatedValues.body + newInputData = { + url: newInputData.url, + body: { + value: JSON.stringify(deprecatedValues), + }, + } + } + /**********************************/ + inputData = newInputData setDefaultEnumValues() } @@ -158,6 +218,19 @@ } const outputs = Object.entries(schema) + let bindingIcon = "" + let bindindingRank = 0 + + if (idx === 0) { + bindingIcon = automation.trigger.icon + } else if (isLoopBlock) { + bindingIcon = "Reuse" + bindindingRank = idx + 1 + } else { + bindingIcon = allSteps[idx].icon + bindindingRank = idx - loopBlockCount + } + bindings = bindings.concat( outputs.map(([name, value]) => { let runtimeName = isLoopBlock @@ -166,17 +239,24 @@ ? `steps[${idx - loopBlockCount}].${name}` : `steps.${idx - loopBlockCount}.${name}` const runtime = idx === 0 ? `trigger.${name}` : runtimeName + const categoryName = + idx === 0 + ? "Trigger outputs" + : isLoopBlock + ? "Loop Outputs" + : `Step ${idx - loopBlockCount} outputs` return { - label: runtime, + readableBinding: runtime, + runtimeBinding: runtime, type: value.type, description: value.description, - category: - idx === 0 - ? "Trigger outputs" - : isLoopBlock - ? "Loop Outputs" - : `Step ${idx - loopBlockCount} outputs`, - path: runtime, + icon: bindingIcon, + category: categoryName, + display: { + type: value.type, + name: name, + rank: bindindingRank, + }, } }) ) @@ -185,15 +265,12 @@ // Environment bindings if ($licensing.environmentVariablesEnabled) { bindings = bindings.concat( - $environment.variables.map(variable => { + getEnvironmentBindings().map(binding => { return { - label: `env.${variable.name}`, - path: `env.${variable.name}`, - icon: "Key", - category: "Environment", + ...binding, display: { - type: "string", - name: variable.name, + ...binding.display, + rank: 98, }, } }) @@ -239,7 +316,7 @@
- {#each schemaProperties as [key, value]} + {#each deprecatedSchemaProperties as [key, value]}
{#if key !== "fields"}