diff --git a/.all-contributorsrc b/.all-contributorsrc deleted file mode 100644 index 3a416f917e..0000000000 --- a/.all-contributorsrc +++ /dev/null @@ -1,194 +0,0 @@ -{ - "files": [ - "README.md" - ], - "imageSize": 100, - "commit": false, - "contributors": [ - { - "login": "shogunpurple", - "name": "Martin McKeaveney", - "avatar_url": "https://avatars1.githubusercontent.com/u/11256663?v=4", - "profile": "http://martinmck.com", - "contributions": [ - "code", - "doc", - "test", - "infra" - ] - }, - { - "login": "mike12345567", - "name": "Michael Drury", - "avatar_url": "https://avatars2.githubusercontent.com/u/4407001?v=4", - "profile": "http://www.michaeldrury.co.uk/", - "contributions": [ - "doc", - "code", - "test", - "infra" - ] - }, - { - "login": "aptkingston", - "name": "Andrew Kingston", - "avatar_url": "https://avatars3.githubusercontent.com/u/9075550?v=4", - "profile": "https://github.com/aptkingston", - "contributions": [ - "doc", - "code", - "test", - "design" - ] - }, - { - "login": "mjashanks", - "name": "Michael Shanks", - "avatar_url": "https://avatars3.githubusercontent.com/u/3524181?v=4", - "profile": "https://budibase.com/", - "contributions": [ - "doc", - "code", - "test" - ] - }, - { - "login": "kevmodrome", - "name": "Kevin Åberg Kultalahti", - "avatar_url": "https://avatars3.githubusercontent.com/u/534488?v=4", - "profile": "https://github.com/kevmodrome", - "contributions": [ - "doc", - "code", - "test" - ] - }, - { - "login": "joebudi", - "name": "Joe", - "avatar_url": "https://avatars2.githubusercontent.com/u/49767913?v=4", - "profile": "https://www.budibase.com/", - "contributions": [ - "doc", - "code", - "content", - "design" - ] - }, - { - "login": "Rory-Powell", - "name": "Rory Powell", - "avatar_url": "https://avatars.githubusercontent.com/u/8755148?v=4", - "profile": "https://github.com/Rory-Powell", - "contributions": [ - "code", - "doc", - "test" - ] - }, - { - "login": "PClmnt", - "name": "Peter Clement", - "avatar_url": "https://avatars.githubusercontent.com/u/5665926?v=4", - "profile": "https://github.com/PClmnt", - "contributions": [ - "code", - "doc", - "test" - ] - }, - { - "login": "Conor-Mack", - "name": "Conor_Mack", - "avatar_url": "https://avatars1.githubusercontent.com/u/36074859?v=4", - "profile": "https://github.com/Conor-Mack", - "contributions": [ - "code", - "test" - ] - }, - { - "login": "pngwn", - "name": "pngwn", - "avatar_url": "https://avatars1.githubusercontent.com/u/12937446?v=4", - "profile": "https://github.com/pngwn", - "contributions": [ - "code", - "test" - ] - }, - { - "login": "HugoLd", - "name": "HugoLd", - "avatar_url": "https://avatars0.githubusercontent.com/u/26521848?v=4", - "profile": "https://github.com/HugoLd", - "contributions": [ - "code" - ] - }, - { - "login": "victoriasloan", - "name": "victoriasloan", - "avatar_url": "https://avatars.githubusercontent.com/u/9913651?v=4", - "profile": "https://github.com/victoriasloan", - "contributions": [ - "code" - ] - }, - { - "login": "yashank09", - "name": "yashank09", - "avatar_url": "https://avatars.githubusercontent.com/u/37672190?v=4", - "profile": "https://github.com/yashank09", - "contributions": [ - "code" - ] - }, - { - "login": "SOVLOOKUP", - "name": "SOVLOOKUP", - "avatar_url": "https://avatars.githubusercontent.com/u/53158137?v=4", - "profile": "https://github.com/SOVLOOKUP", - "contributions": [ - "code" - ] - }, - { - "login": "seoulaja", - "name": "seoulaja", - "avatar_url": "https://avatars.githubusercontent.com/u/15101654?v=4", - "profile": "https://github.com/seoulaja", - "contributions": [ - "translation" - ] - }, - { - "login": "mslourens", - "name": "Maurits Lourens", - "avatar_url": "https://avatars.githubusercontent.com/u/1907152?v=4", - "profile": "https://github.com/mslourens", - "contributions": [ - "test", - "code" - ] - }, - { - "login": "Rory-Powell", - "name": "Rory Powell", - "avatar_url": "https://avatars.githubusercontent.com/u/8755148?v=4", - "profile": "https://github.com/Rory-Powell", - "contributions": [ - "infra", - "test", - "code" - ] - } - ], - "contributorsPerLine": 7, - "projectName": "budibase", - "projectOwner": "Budibase", - "repoType": "github", - "repoHost": "https://github.com", - "skipCi": true, - "commitConvention": "none" -} diff --git a/.eslintignore b/.eslintignore index 021fe8e367..a79f9e2879 100644 --- a/.eslintignore +++ b/.eslintignore @@ -8,3 +8,6 @@ packages/backend-core/coverage packages/server/client packages/builder/.routify packages/sdk/sdk +packages/account-portal/packages/server/build +packages/account-portal/packages/ui/.routify +packages/account-portal/packages/ui/build \ No newline at end of file diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 9b75a2e73a..029dd5af42 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -1,139 +1,45 @@ # Budibase CI Pipelines -Welcome to the budibase CI pipelines directory. This document details what each of the CI pipelines are for, and come common combinations. +Welcome to the Budibase CI pipelines directory. This document details what each of the CI pipelines are for, and come common combinations. ## All CI Pipelines -### Note - -- When running workflow dispatch jobs, ensure you always run them off the `master` branch. It defaults to `develop`, so double check before running any jobs. The exception to this case is the `deploy-release` job which requires the develop branch. - ### Standard CI Build Job (budibase_ci.yml) Triggers: -- PR or push to develop - PR or push to master -The standard CI Build job is what runs when you raise a PR to develop or master. +The standard CI Build job is what runs when you raise a PR to master. - Installs all dependencies, - builds the project - run the unit tests - Generate test coverage metrics with codecov - Run the integration tests +- Check that the pro and account portal submodules are pointing to the lastest master head -### Release Develop Job (release-develop.yml) +### Release Job (tag-release.yml) Triggers: -- Push to develop +- Manually triggered -The job responsible for building, tagging and pushing docker images out to the test and release environments. +This job is responsible for building and pushing all the production services, packages and images. This is done via [budibase-deploys](https://github.com/Budibase/budibase-deploys/actions/workflows/release.yml). -- Installs all dependencies -- builds the project -- run the unit tests -- publish the budibase JS packages under a prerelease tag to NPM -- build, tag and push docker images under the `develop` tag to docker hub +An input is required, indicating if the new version will be a `patch`, `minor` or `major` bump. -These images will then be pulled by the test and release environments, updating the latest automatically. Discord notifications are sent to the #infra channel when this occurs. - -### Release Job (release.yml) - -Triggers: - -- Push to master - -This job is responsible for building and pushing the latest code to NPM and docker hub, so that it can be deployed. - -- Installs all dependencies -- builds the project -- run the unit tests -- publish the budibase JS packages under a release tag to NPM (always incremented by patch versions) -- build, tag and push docker images under the `v.x.x.x` (the tag of the NPM release) tag to docker hub - -### Release Selfhost Job (release-selfhost.yml) - -Triggers: - -- Manual Workflow Dispatch Trigger - -This job is responsible for delivering the latest version of budibase to those that are self-hosting. - -This job relies on the release job to have run first, so the latest image is pushed to dockerhub. This job then will pull the latest version from `lerna.json` and try to find an image in dockerhub corresponding to that version. For example, if the version in `lerna.json` is `1.0.0`: - -- Pull the images for all budibase services tagged `v1.0.0` from dockerhub -- Tag these images as `latest` -- Push them back to dockerhub. This now means anyone who pulls `latest` (self hosters using docker-compose) will get the latest version. -- Build and release the budibase helm chart for kubernetes users -- Perform a github release with the latest version. You can see previous releases here (https://github.com/Budibase/budibase/releases) - -### Deploy Release (deploy-release.yml) - -Triggers: - -- Manual Workflow Dispatch Trigger - -This job is responsible for deploying to our release, cloud kubernetes environment. You must run the release job first, to ensure that the latest images have been built and pushed to docker hub. After kicking off this job, the following will occur: - -- Checks out the release branch -- Pulls the latest `values.yaml` from budibase infra, a private repo containing budibases infrastructure configuration -- Gets the latest budibase version from `lerna.json`, if it hasn't been specified in the workflow when you kicked it off -- Configures AWS Credentials -- Deploys the helm chart in the budibase repo to our preproduction EKS cluster, injecting the `values.yaml` we pulled from budibase-infra -- Fires off a discord webhook in the #infra channel to show that the deployment completely successfully. - -### Deploy Preprod (deploy-preprod.yml) - -Triggers: - -- Manual Workflow Dispatch Trigger - -This job is responsible for deploying to our preprod, cloud kubernetes environment. You must run the release job first, to ensure that the latest images have been built and pushed to docker hub. After kicking off this job, the following will occur: - -- Checks out the master branch -- Pulls the latest `values.yaml` from budibase infra, a private repo containing budibases infrastructure configuration -- Gets the latest budibase version from `lerna.json`, if it hasn't been specified in the workflow when you kicked it off -- Configures AWS Credentials -- Deploys the helm chart in the budibase repo to our preprod EKS cluster, injecting the `values.yaml` we pulled from budibase-infra -- Fires off a discord webhook in the #infra channel to show that the deployment completely successfully. - -### Deploy Production (deploy-cloud.yml) - -Triggers: - -- Manual Workflow Dispatch Trigger - -This job is responsible for deploying to our production, cloud kubernetes environment. You must run the release job first, to ensure that the latest images have been built and pushed to docker hub. You can also manually enter a version number for this job, so you can perform rollbacks or upgrade to a specific version. After kicking off this job, the following will occur: - -- Checks out the master branch -- Pulls the latest `values.yaml` from budibase infra, a private repo containing budibases infrastructure configuration -- Gets the latest budibase version from `lerna.json`, if it hasn't been specified in the workflow when you kicked it off -- Configures AWS Credentials -- Deploys the helm chart in the budibase repo to our production EKS cluster, injecting the `values.yaml` we pulled from budibase-infra -- Fires off a discord webhook in the #infra channel to show that the deployment completely successfully. +More documentation can be found in here: https://budibase.atlassian.net/wiki/spaces/DEVOPS/pages/347930625/Production+release ## Common Workflows ### Deploy Changes to Production (Release) -- Merge `develop` into `master` -- Wait for budibase CI job and release job to run -- Run cloud deploy job -- Run release selfhost job - -### Deploy Changes to Production (Hotfix) - -- Branch off `master` -- Perform your hotfix -- Merge back into `master` -- Wait for budibase CI job and release job to run -- Run cloud deploy job -- Run release selfhost job +- Merge your changes into `master` +- Run `tag-release.yml` +- Check the progress in [budibase-deploys](https://github.com/Budibase/budibase-deploys/actions/workflows/release.yml) ### Rollback A Bad Cloud Deployment -- Kick off cloud deploy job -- Ensure you are running off master -- Enter the version number of the last known good version of budibase. For example `1.0.0` +Rollback documentation can be found in here. +https://budibase.atlassian.net/wiki/spaces/DEVOPS/pages/347930625/Production+release#Rollback diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index cb713c93ac..d6bbf19940 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -38,10 +38,10 @@ jobs: submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - - name: Use Node.js 18.x + - name: Use Node.js 20.x uses: actions/setup-node@v3 with: - node-version: 18.x + node-version: 20.x cache: yarn - run: yarn --frozen-lockfile - run: yarn lint @@ -56,10 +56,10 @@ jobs: token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Use Node.js 18.x + - name: Use Node.js 20.x uses: actions/setup-node@v3 with: - node-version: 18.x + node-version: 20.x cache: yarn - run: yarn --frozen-lockfile @@ -76,6 +76,18 @@ jobs: yarn check:types fi + helm-lint: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Use Node.js 20.x + uses: azure/setup-helm@v3 + - run: cd charts/budibase && helm lint . + test-libraries: runs-on: ubuntu-latest steps: @@ -86,10 +98,10 @@ jobs: token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Use Node.js 18.x + - name: Use Node.js 20.x uses: actions/setup-node@v3 with: - node-version: 18.x + node-version: 20.x cache: yarn - run: yarn --frozen-lockfile - name: Test @@ -110,10 +122,10 @@ jobs: token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Use Node.js 18.x + - name: Use Node.js 20.x uses: actions/setup-node@v3 with: - node-version: 18.x + node-version: 20.x cache: yarn - run: yarn --frozen-lockfile - name: Test worker @@ -134,10 +146,10 @@ jobs: token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Use Node.js 18.x + - name: Use Node.js 20.x uses: actions/setup-node@v3 with: - node-version: 18.x + node-version: 20.x cache: yarn - run: yarn --frozen-lockfile - name: Test server @@ -159,10 +171,10 @@ jobs: token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Use Node.js 18.x + - name: Use Node.js 20.x uses: actions/setup-node@v3 with: - node-version: 18.x + node-version: 20.x cache: yarn - run: yarn --frozen-lockfile - name: Test @@ -182,10 +194,10 @@ jobs: submodules: ${{ env.IS_OSS_CONTRIBUTOR == 'false' }} token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} - - name: Use Node.js 18.x + - name: Use Node.js 20.x uses: actions/setup-node@v3 with: - node-version: 18.x + node-version: 20.x cache: yarn - run: yarn --frozen-lockfile - name: Build packages @@ -204,7 +216,7 @@ jobs: check-pro-submodule: runs-on: ubuntu-latest - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' + if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') steps: - name: Checkout repo and submodules uses: actions/checkout@v3 @@ -246,7 +258,57 @@ jobs: if (submoduleCommit !== baseCommit) { console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.'); - console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/develop/docs/getting_started.md') + console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/master/docs/getting_started.md') + process.exit(1); + } else { + console.log('All good, the submodule had been merged and setup correctly!') + } + + check-accountportal-submodule: + runs-on: ubuntu-latest + if: inputs.run_as_oss != true && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') + steps: + - name: Checkout repo and submodules + uses: actions/checkout@v3 + with: + submodules: true + token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} + fetch-depth: 0 + + - name: Check account portal commit + id: get_accountportal_commits + run: | + cd packages/account-portal + accountportal_commit=$(git rev-parse HEAD) + + branch="${{ github.base_ref || github.ref_name }}" + echo "Running on branch '$branch' (base_ref=${{ github.base_ref }}, ref_name=${{ github.head_ref }})" + + base_commit=$(git rev-parse origin/master) + + if [[ ! -z $base_commit ]]; then + echo "target_branch=$branch" + echo "target_branch=$branch" >> "$GITHUB_OUTPUT" + echo "accountportal_commit=$accountportal_commit" + echo "accountportal_commit=$accountportal_commit" >> "$GITHUB_OUTPUT" + echo "base_commit=$base_commit" + echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT" + else + echo "Nothing to do - branch to branch merge." + fi + + - name: Check submodule merged to base branch + if: ${{ steps.get_accountportal_commits.outputs.base_commit != '' }} + uses: actions/github-script@v4 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const submoduleCommit = '${{ steps.get_accountportal_commits.outputs.accountportal_commit }}'; + const baseCommit = '${{ steps.get_accountportal_commits.outputs.base_commit }}'; + + if (submoduleCommit !== baseCommit) { + console.error('Submodule commit does not match the latest commit on the "${{ steps.get_accountportal_commits.outputs.target_branch }}" branch.'); + console.error('Refer to the account portal repo to merge your changes: https://github.com/Budibase/account-portal/blob/master/docs/index.md') process.exit(1); } else { console.log('All good, the submodule had been merged and setup correctly!') diff --git a/.github/workflows/close-featurebranch.yml b/.github/workflows/close-featurebranch.yml index 46cb781730..5da3eb52cd 100644 --- a/.github/workflows/close-featurebranch.yml +++ b/.github/workflows/close-featurebranch.yml @@ -2,9 +2,7 @@ name: close-featurebranch on: pull_request: - types: [closed] - branches: - - master + types: [closed, unlabeled] workflow_dispatch: inputs: BRANCH: @@ -14,6 +12,9 @@ on: jobs: release: + if: | + (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'feature-branch')) || + github.event.label.name == 'feature-branch' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index c70f2fff20..a5636fe912 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -2,12 +2,19 @@ name: deploy-featurebranch on: pull_request: - branches: - - master + types: [ + labeled, + # default types below (https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request) + opened, + synchronize, + reopened, + ] jobs: release: - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase' + if: | + (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') && + contains(github.event.pull_request.labels.*.name, 'feature-branch') runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/force-release.yml b/.github/workflows/force-release.yml new file mode 100644 index 0000000000..8a9d444f51 --- /dev/null +++ b/.github/workflows/force-release.yml @@ -0,0 +1,46 @@ +name: Forced release +concurrency: + group: tag-release + cancel-in-progress: false + +on: + workflow_dispatch: + +jobs: + ensure-is-master-tag: + name: Ensure is a master tag + runs-on: qa-arc-runner-set + steps: + - name: Checkout monorepo + uses: actions/checkout@v4 + with: + token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} + fetch-tags: true + fetch-depth: 0 + + - name: Fail if ref is not a tag + run: | + if ! git show-ref -q --verify "refs/tags/${{ github.ref_name }}" 2>/dev/null; then + echo "'${{ github.ref_name }}' is not a valid tag." + exit 1 + fi + - name: Fail if tag is not in master + run: | + if ! git merge-base --is-ancestor ${{ github.ref_name }} origin/master; then + echo "Tag is not in master. Release can only execute tags that are present on the master branch" + exit 1 + fi + + trigger-release: + needs: [ensure-is-master-tag] + runs-on: ubuntu-latest + steps: + - uses: peter-evans/repository-dispatch@v2 + with: + repository: budibase/budibase-deploys + event-type: release-prod + token: ${{ secrets.GH_ACCESS_TOKEN }} + client-payload: |- + { + "TAG": "${{ github.ref_name }}" + } diff --git a/.github/workflows/stale_bot.yml b/.github/workflows/stale_bot.yml index 8f3ab9c74c..411a70a463 100644 --- a/.github/workflows/stale_bot.yml +++ b/.github/workflows/stale_bot.yml @@ -16,8 +16,8 @@ jobs: days-before-pr-stale: 7 stale-issue-label: stale exempt-pr-labels: pinned,security,roadmap - days-before-pr-close: 7 + days-before-issue-close: 30 - uses: actions/stale@v8 with: @@ -26,6 +26,7 @@ jobs: days-before-stale: 30 only-issue-labels: bug,High priority stale-issue-label: warn + days-before-close: 30 - uses: actions/stale@v8 with: @@ -34,6 +35,7 @@ jobs: days-before-stale: 90 only-issue-labels: bug,Medium priority stale-issue-label: warn + days-before-close: 30 - uses: actions/stale@v8 with: @@ -43,5 +45,4 @@ jobs: stale-issue-label: stale only-issue-labels: bug stale-issue-message: "This issue has been automatically marked as stale because it has not had any activity for six months." - days-before-close: 30 diff --git a/.gitignore b/.gitignore index 02e0ca300d..3eb705dbbf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -builder/* .data/ .temp/ packages/server/runtime_apps/ @@ -41,8 +40,11 @@ bower_components build/Release # Dependency directories -/node_modules/ jspm_packages/ +*.min.js +*.map +node_modules/ +dist/ # TypeScript v1 declaration files typings/ diff --git a/.gitmodules b/.gitmodules index 2dd6ea53f2..cb6d1c5dc8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "packages/pro"] path = packages/pro url = git@github.com:Budibase/budibase-pro.git +[submodule "packages/account-portal"] + path = packages/account-portal + url = git@github.com:Budibase/account-portal.git diff --git a/.nvmrc b/.nvmrc index 7950a44576..790e1105f2 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v18.17.0 +v20.10.0 diff --git a/.prettierignore b/.prettierignore index ce7617224b..2444f0e753 100644 --- a/.prettierignore +++ b/.prettierignore @@ -8,4 +8,7 @@ packages/worker/coverage packages/backend-core/coverage packages/builder/.routify packages/sdk/sdk -packages/pro/coverage \ No newline at end of file +packages/pro/coverage +packages/account-portal/packages/ui/build +packages/account-portal/packages/ui/.routify +packages/account-portal/packages/server/build \ No newline at end of file diff --git a/.tool-versions b/.tool-versions index a909d60941..946d5198ce 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ -nodejs 18.17.0 +nodejs 20.10.0 python 3.10.0 yarn 1.22.19 diff --git a/.vscode/settings.json b/.vscode/settings.json index ece537efac..e22d5a8866 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,7 +1,7 @@ { "editor.formatOnSave": true, "editor.codeActionsOnSave": { - "source.fixAll": true + "source.fixAll": "explicit" }, "editor.defaultFormatter": "esbenp.prettier-vscode", "[json]": { diff --git a/LICENSE b/LICENSE index a017209adf..cbb55109f4 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,9 @@ -Copyright 2019-2021, Budibase Ltd. +Copyright 2019-2023, Budibase Ltd. Each Budibase package has its own license, please check the license file in each package. You can consider Budibase to be GPLv3 licensed overall. The apps that you build with Budibase do not package any GPLv3 licensed code, thus do not fall under those restrictions. + +Budibase ships with Structured Query Server, by The Neighbourhoodie Software GmbH. This license for this can be found at ./SQS_LICENSE diff --git a/SQS_LICENSE b/SQS_LICENSE new file mode 100644 index 0000000000..0315ee9527 --- /dev/null +++ b/SQS_LICENSE @@ -0,0 +1,31 @@ +FORM OF CUSTOMER LICENCE + +Budibase hereby grants the Customer a worldwide, royalty free, non-exclusive, +perpetual (for the lifetime of the intellectual property rights contained in the Product) +right and title to utilise the binary code of the The Neighbourhoodie Software GmbH +Structured Query Server software product (Product) for its own internal business +purposes (the Purpose) only (the Licence). The Product has the function of bringing a +CouchDB database (NoSQL database) into an SQL database form (SQLite) and thereby +making it usable for complex queries - which originally could only be displayed in an +SQL database. By indexing in SQLite and a server that is tailored to it, the Product +enables the use of CouchDB with SQL queries. +The Licence shall not permit sub-licensing, resale or transfer of the Product to third +parties, other than sub-licensing to the Customer’s direct contractors for the purposes +of utilizing the Product as contemplated above. +The Licence shall not permit the adaptation, modification, decompilation, reverse +engineering or similar activities with respect to the Product. +This licence is granted to the Customer only, although Customer and its Affiliates’ +employees, servants and agents shall be entitled to utilize the Product within the scope +of the Licence for the Customer’s Purpose only. +Reproduction is not permitted to users, except for reproductions that are necessary for +the use of the product under the licence described above. These conditions apply to the +product regardless of the form in which we make the product available and on which +devices it is installed and/or with which devices it is ultimately used. Depending on the +product variant or intended use, certain technical requirements in the IT infrastructure +must be satisfied as a prerequisite for use. +The law of the Northern Ireland applies exclusively to this licence, and the courts of +Northern Ireland shall have exclusive jurisdiction, save that we reserve a right to sue +you in the jurisdiction in which you are based. The application of the UN Sales +Convention (CISG) is excluded. +The invalidity of any part of this licence does not affect the validity of the remaining +regulations. diff --git a/charts/budibase/README.md b/charts/budibase/README.md index d8191026ce..342011bdb1 100644 --- a/charts/budibase/README.md +++ b/charts/budibase/README.md @@ -157,6 +157,17 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml | services.apps.replicaCount | int | `1` | The number of apps replicas to run. | | services.apps.resources | object | `{}` | The resources to use for apps pods. See for more information on how to set these. | | services.apps.startupProbe | object | HTTP health checks. | Startup probe configuration for apps pods. You shouldn't need to change this, but if you want to you can find more information here: | +| services.automationWorkers.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaling for the apps service. | +| services.automationWorkers.autoscaling.maxReplicas | int | `10` | | +| services.automationWorkers.autoscaling.minReplicas | int | `1` | | +| services.automationWorkers.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the automation worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the automation worker pods. | +| services.automationWorkers.enabled | bool | `true` | Whether or not to enable the automation worker service. If you disable this, automations will be processed by the apps service. | +| services.automationWorkers.livenessProbe | object | HTTP health checks. | Liveness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | +| services.automationWorkers.logLevel | string | `"info"` | The log level for the automation worker service. | +| services.automationWorkers.readinessProbe | object | HTTP health checks. | Readiness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | +| services.automationWorkers.replicaCount | int | `1` | The number of automation worker replicas to run. | +| services.automationWorkers.resources | object | `{}` | The resources to use for automation worker pods. See for more information on how to set these. | +| services.automationWorkers.startupProbe | object | HTTP health checks. | Startup probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: | | services.couchdb.backup.enabled | bool | `false` | Whether or not to enable periodic CouchDB backups. This works by replicating to another CouchDB instance. | | services.couchdb.backup.interval | string | `""` | Backup interval in seconds | | services.couchdb.backup.resources | object | `{}` | The resources to use for CouchDB backup pods. See for more information on how to set these. | diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 7358e474ca..c7c4481122 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -192,7 +192,14 @@ spec: - name: NODE_TLS_REJECT_UNAUTHORIZED value: {{ .Values.services.tlsRejectUnauthorized }} {{ end }} - + {{- if .Values.services.automationWorkers.enabled }} + - name: APP_FEATURES + value: "api" + {{- end }} + {{- range .Values.services.apps.extraEnv }} + - name: {{ .name }} + value: {{ .value | quote }} + {{- end }} image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always {{- if .Values.services.apps.startupProbe }} @@ -220,6 +227,14 @@ spec: resources: {{- toYaml . | nindent 10 }} {{ end }} + {{ if .Values.services.apps.command }} + command: + {{- toYaml .Values.services.apps.command | nindent 10 }} + {{ end }} + {{ if .Values.services.apps.args }} + args: + {{- toYaml .Values.services.apps.args | nindent 10 }} + {{ end }} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} @@ -237,4 +252,10 @@ spec: {{ end }} restartPolicy: Always serviceAccountName: "" + {{ if .Values.services.apps.ndots }} + dnsConfig: + options: + - name: ndots + value: {{ .Values.services.apps.ndots | quote }} + {{ end }} status: {} diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml new file mode 100644 index 0000000000..36c3a8ffbf --- /dev/null +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -0,0 +1,262 @@ +{{- if .Values.services.automationWorkers.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: +{{ if .Values.services.automationWorkers.deploymentAnnotations }} +{{- toYaml .Values.services.automationWorkers.deploymentAnnotations | indent 4 -}} +{{ end }} + labels: + io.kompose.service: automation-worker-service +{{ if .Values.services.automationWorkers.deploymentLabels }} +{{- toYaml .Values.services.automationWorkers.deploymentLabels | indent 4 -}} +{{ end }} + name: automation-worker-service +spec: + replicas: {{ .Values.services.automationWorkers.replicaCount }} + selector: + matchLabels: + io.kompose.service: automation-worker-service + strategy: + type: RollingUpdate + template: + metadata: + annotations: +{{ if .Values.services.automationWorkers.templateAnnotations }} +{{- toYaml .Values.services.automationWorkers.templateAnnotations | indent 8 -}} +{{ end }} + labels: + io.kompose.service: automation-worker-service +{{ if .Values.services.automationWorkers.templateLabels }} +{{- toYaml .Values.services.automationWorkers.templateLabels | indent 8 -}} +{{ end }} + spec: + containers: + - env: + - name: BUDIBASE_ENVIRONMENT + value: {{ .Values.globals.budibaseEnv }} + - name: DEPLOYMENT_ENVIRONMENT + value: "kubernetes" + - name: COUCH_DB_URL + {{ if .Values.services.couchdb.url }} + value: {{ .Values.services.couchdb.url }} + {{ else }} + value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} + {{ end }} + {{ if .Values.services.couchdb.enabled }} + - name: COUCH_DB_USER + valueFrom: + secretKeyRef: + name: {{ template "couchdb.fullname" . }} + key: adminUsername + - name: COUCH_DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "couchdb.fullname" . }} + key: adminPassword + {{ end }} + - name: ENABLE_ANALYTICS + value: {{ .Values.globals.enableAnalytics | quote }} + - name: API_ENCRYPTION_KEY + value: {{ .Values.globals.apiEncryptionKey | quote }} + - name: HTTP_LOGGING + value: {{ .Values.services.automationWorkers.httpLogging | quote }} + - name: INTERNAL_API_KEY + valueFrom: + secretKeyRef: + name: {{ template "budibase.fullname" . }} + key: internalApiKey + - name: INTERNAL_API_KEY_FALLBACK + value: {{ .Values.globals.internalApiKeyFallback | quote }} + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: {{ template "budibase.fullname" . }} + key: jwtSecret + - name: JWT_SECRET_FALLBACK + value: {{ .Values.globals.jwtSecretFallback | quote }} + {{ if .Values.services.objectStore.region }} + - name: AWS_REGION + value: {{ .Values.services.objectStore.region }} + {{ end }} + - name: MINIO_ENABLED + value: {{ .Values.services.objectStore.minio | quote }} + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ template "budibase.fullname" . }} + key: objectStoreAccess + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ template "budibase.fullname" . }} + key: objectStoreSecret + - name: CLOUDFRONT_CDN + value: {{ .Values.services.objectStore.cloudfront.cdn | quote }} + - name: CLOUDFRONT_PUBLIC_KEY_ID + value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }} + - name: CLOUDFRONT_PRIVATE_KEY_64 + value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }} + - name: MINIO_URL + value: {{ .Values.services.objectStore.url }} + - name: PLUGIN_BUCKET_NAME + value: {{ .Values.services.objectStore.pluginBucketName | quote }} + - name: APPS_BUCKET_NAME + value: {{ .Values.services.objectStore.appsBucketName | quote }} + - name: GLOBAL_BUCKET_NAME + value: {{ .Values.services.objectStore.globalBucketName | quote }} + - name: BACKUPS_BUCKET_NAME + value: {{ .Values.services.objectStore.backupsBucketName | quote }} + - name: PORT + value: {{ .Values.services.automationWorkers.port | quote }} + {{ if .Values.services.worker.publicApiRateLimitPerSecond }} + - name: API_REQ_LIMIT_PER_SEC + value: {{ .Values.globals.automationWorkers.publicApiRateLimitPerSecond | quote }} + {{ end }} + - name: MULTI_TENANCY + value: {{ .Values.globals.multiTenancy | quote }} + - name: OFFLINE_MODE + value: {{ .Values.globals.offlineMode | quote }} + - name: LOG_LEVEL + value: {{ .Values.services.automationWorkers.logLevel | quote }} + - name: REDIS_PASSWORD + value: {{ .Values.services.redis.password }} + - name: REDIS_URL + {{ if .Values.services.redis.url }} + value: {{ .Values.services.redis.url }} + {{ else }} + value: redis-service:{{ .Values.services.redis.port }} + {{ end }} + - name: SELF_HOSTED + value: {{ .Values.globals.selfHosted | quote }} + - name: POSTHOG_TOKEN + value: {{ .Values.globals.posthogToken | quote }} + - name: WORKER_URL + value: http://worker-service:{{ .Values.services.worker.port }} + - name: PLATFORM_URL + value: {{ .Values.globals.platformUrl | quote }} + - name: ACCOUNT_PORTAL_URL + value: {{ .Values.globals.accountPortalUrl | quote }} + - name: ACCOUNT_PORTAL_API_KEY + value: {{ .Values.globals.accountPortalApiKey | quote }} + - name: COOKIE_DOMAIN + value: {{ .Values.globals.cookieDomain | quote }} + - name: HTTP_MIGRATIONS + value: {{ .Values.globals.httpMigrations | quote }} + - name: GOOGLE_CLIENT_ID + value: {{ .Values.globals.google.clientId | quote }} + - name: GOOGLE_CLIENT_SECRET + value: {{ .Values.globals.google.secret | quote }} + - name: AUTOMATION_MAX_ITERATIONS + value: {{ .Values.globals.automationMaxIterations | quote }} + - name: TENANT_FEATURE_FLAGS + value: {{ .Values.globals.tenantFeatureFlags | quote }} + - name: ENCRYPTION_KEY + value: {{ .Values.globals.bbEncryptionKey | quote }} + {{ if .Values.globals.bbAdminUserEmail }} + - name: BB_ADMIN_USER_EMAIL + value: {{ .Values.globals.bbAdminUserEmail | quote }} + {{ end }} + {{ if .Values.globals.bbAdminUserPassword }} + - name: BB_ADMIN_USER_PASSWORD + value: {{ .Values.globals.bbAdminUserPassword | quote }} + {{ end }} + {{ if .Values.globals.pluginsDir }} + - name: PLUGINS_DIR + value: {{ .Values.globals.pluginsDir | quote }} + {{ end }} + {{ if .Values.services.automationWorkers.nodeDebug }} + - name: NODE_DEBUG + value: {{ .Values.services.automationWorkers.nodeDebug | quote }} + {{ end }} + {{ if .Values.globals.datadogApmEnabled }} + - name: DD_LOGS_INJECTION + value: {{ .Values.globals.datadogApmEnabled | quote }} + - name: DD_APM_ENABLED + value: {{ .Values.globals.datadogApmEnabled | quote }} + - name: DD_APM_DD_URL + value: https://trace.agent.datadoghq.eu + {{ end }} + {{ if .Values.globals.globalAgentHttpProxy }} + - name: GLOBAL_AGENT_HTTP_PROXY + value: {{ .Values.globals.globalAgentHttpProxy | quote }} + {{ end }} + {{ if .Values.globals.globalAgentHttpsProxy }} + - name: GLOBAL_AGENT_HTTPS_PROXY + value: {{ .Values.globals.globalAgentHttpsProxy | quote }} + {{ end }} + {{ if .Values.globals.globalAgentNoProxy }} + - name: GLOBAL_AGENT_NO_PROXY + value: {{ .Values.globals.globalAgentNoProxy | quote }} + {{ end }} + {{ if .Values.services.tlsRejectUnauthorized }} + - name: NODE_TLS_REJECT_UNAUTHORIZED + value: {{ .Values.services.tlsRejectUnauthorized }} + {{ end }} + - name: APP_FEATURES + value: "automations" + {{- range .Values.services.automationWorkers.extraEnv }} + - name: {{ .name }} + value: {{ .value | quote }} + {{- end }} + + image: budibase/apps:{{ .Values.globals.appVersion | default .Chart.AppVersion }} + imagePullPolicy: Always + {{- if .Values.services.automationWorkers.startupProbe }} + {{- with .Values.services.automationWorkers.startupProbe }} + startupProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.automationWorkers.livenessProbe }} + {{- with .Values.services.automationWorkers.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + {{- if .Values.services.automationWorkers.readinessProbe }} + {{- with .Values.services.automationWorkers.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- end }} + name: bbautomationworker + ports: + - containerPort: {{ .Values.services.automationWorkers.port }} + {{ with .Values.services.automationWorkers.resources }} + resources: + {{- toYaml . | nindent 10 }} + {{ end }} + {{ if .Values.services.automationWorkers.command }} + command: + {{- toYaml .Values.services.automationWorkers.command | nindent 10 }} + {{ end }} + {{ if .Values.services.automationWorkers.args }} + args: + {{- toYaml .Values.services.automationWorkers.args | nindent 10 }} + {{ end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{ if .Values.schedulerName }} + schedulerName: {{ .Values.schedulerName | quote }} + {{ end }} + {{ if .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.imagePullSecrets | nindent 6 }} + {{ end }} + restartPolicy: Always + serviceAccountName: "" + {{ if .Values.services.automationWorkers.ndots }} + dnsConfig: + options: + - name: ndots + value: {{ .Values.services.automationWorkers.ndots | quote }} + {{ end }} +status: {} +{{- end }} \ No newline at end of file diff --git a/charts/budibase/templates/automation-worker-service-hpa.yaml b/charts/budibase/templates/automation-worker-service-hpa.yaml new file mode 100644 index 0000000000..f29223b61b --- /dev/null +++ b/charts/budibase/templates/automation-worker-service-hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.services.automationWorkers.autoscaling.enabled }} +apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "budibase.fullname" . }}-apps + labels: + {{- include "budibase.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: automation-worker-service + minReplicas: {{ .Values.services.automationWorkers.autoscaling.minReplicas }} + maxReplicas: {{ .Values.services.automationWorkers.autoscaling.maxReplicas }} + metrics: + {{- if .Values.services.automationWorkers.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.services.automationWorkers.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.services.automationWorkers.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.services.automationWorkers.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/charts/budibase/templates/proxy-service-deployment.yaml b/charts/budibase/templates/proxy-service-deployment.yaml index 706e9b4b73..233028cafe 100644 --- a/charts/budibase/templates/proxy-service-deployment.yaml +++ b/charts/budibase/templates/proxy-service-deployment.yaml @@ -100,5 +100,19 @@ spec: {{ end }} restartPolicy: Always serviceAccountName: "" + {{ if .Values.services.proxy.command }} + command: + {{- toYaml .Values.services.proxy.command | nindent 8 }} + {{ end }} + {{ if .Values.services.proxy.args }} + args: + {{- toYaml .Values.services.proxy.args | nindent 8 }} + {{ end }} volumes: + {{ if .Values.services.proxy.ndots }} + dnsConfig: + options: + - name: ndots + value: {{ .Values.services.proxy.ndots | quote }} + {{ end }} status: {} diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index 6427aa70e8..2f97508ae3 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -182,6 +182,10 @@ spec: - name: NODE_TLS_REJECT_UNAUTHORIZED value: {{ .Values.services.tlsRejectUnauthorized }} {{ end }} + {{- range .Values.services.worker.extraEnv }} + - name: {{ .name }} + value: {{ .value | quote }} + {{- end }} image: budibase/worker:{{ .Values.globals.appVersion | default .Chart.AppVersion }} imagePullPolicy: Always {{- if .Values.services.worker.startupProbe }} @@ -209,6 +213,14 @@ spec: resources: {{- toYaml . | nindent 10 }} {{ end }} + {{ if .Values.services.worker.command }} + command: + {{- toYaml .Values.services.worker.command | nindent 10 }} + {{ end }} + {{ if .Values.services.worker.args }} + args: + {{- toYaml .Values.services.worker.args | nindent 10 }} + {{ end }} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} @@ -226,4 +238,10 @@ spec: {{ end }} restartPolicy: Always serviceAccountName: "" + {{ if .Values.services.worker.ndots }} + dnsConfig: + options: + - name: ndots + value: {{ .Values.services.worker.ndots | quote }} + {{ end }} status: {} diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 13054e75fc..09262df463 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -220,6 +220,9 @@ services: # # for more information on how to set these. resources: {} + # -- Extra environment variables to set for apps pods. Takes a list of + # name=value pairs. + extraEnv: [] # -- Startup probe configuration for apps pods. You shouldn't need to # change this, but if you want to you can find more information here: # @@ -272,6 +275,78 @@ services: # and resources set for the apps pods. targetCPUUtilizationPercentage: 80 + automationWorkers: + # -- Whether or not to enable the automation worker service. If you disable this, + # automations will be processed by the apps service. + enabled: true + # @ignore (you shouldn't need to change this) + port: 4002 + # -- The number of automation worker replicas to run. + replicaCount: 1 + # -- The log level for the automation worker service. + logLevel: info + # -- The resources to use for automation worker pods. See + # + # for more information on how to set these. + resources: {} + # -- Extra environment variables to set for automation worker pods. Takes a list of + # name=value pairs. + extraEnv: [] + # -- Startup probe configuration for automation worker pods. You shouldn't + # need to change this, but if you want to you can find more information + # here: + # + # @default -- HTTP health checks. + startupProbe: + # @ignore + httpGet: + path: /health + port: 4002 + scheme: HTTP + # @ignore + failureThreshold: 30 + # @ignore + periodSeconds: 3 + # -- Readiness probe configuration for automation worker pods. You shouldn't + # need to change this, but if you want to you can find more information + # here: + # + # @default -- HTTP health checks. + readinessProbe: + # @ignore + httpGet: + path: /health + port: 4002 + scheme: HTTP + # @ignore + periodSeconds: 3 + # @ignore + failureThreshold: 1 + # -- Liveness probe configuration for automation worker pods. You shouldn't + # need to change this, but if you want to you can find more information + # here: + # + # @default -- HTTP health checks. + livenessProbe: + # @ignore + httpGet: + path: /health + port: 4002 + scheme: HTTP + # @ignore + failureThreshold: 3 + # @ignore + periodSeconds: 30 + autoscaling: + # -- Whether to enable horizontal pod autoscaling for the apps service. + enabled: false + minReplicas: 1 + maxReplicas: 10 + # -- Target CPU utilization percentage for the automation worker service. + # Note that for autoscaling to work, you will need to have metrics-server + # configured, and resources set for the automation worker pods. + targetCPUUtilizationPercentage: 80 + worker: # @ignore (you shouldn't need to change this) port: 4003 @@ -285,6 +360,9 @@ services: # # for more information on how to set these. resources: {} + # -- Extra environment variables to set for worker pods. Takes a list of + # name=value pairs. + extraEnv: [] # -- Startup probe configuration for worker pods. You shouldn't need to # change this, but if you want to you can find more information here: # diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 77afd9453b..311afbe706 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -84,13 +84,13 @@ Component libraries are collections of components as well as the definition of t - If the project diverges from your branch, please rebase instead of merging. This makes the commit graph easier to read. -- Once your work is completed, please raise a PR against the `develop` branch with some information about what has changed and why. +- Once your work is completed, please raise a PR against the `master` branch with some information about what has changed and why. ### Getting Started For Contributors #### 1. Prerequisites -- NodeJS version `18.x.x` +- NodeJS version `20.x.x` - Python version `3.x` ### Using asdf (recommended) @@ -246,7 +246,7 @@ From here - to develop a change in pro, you can follow the below flow: cd packages/pro # get the base branch you are working from (same as monorepo) git fetch -git checkout +git checkout master # create a branch, named the same as the branch in your monorepo git checkout -b ... make changes diff --git a/examples/nextjs-api-sales/package.json b/examples/nextjs-api-sales/package.json index 41ce52e952..9303874a77 100644 --- a/examples/nextjs-api-sales/package.json +++ b/examples/nextjs-api-sales/package.json @@ -22,6 +22,6 @@ "@types/react": "17.0.39", "eslint": "8.10.0", "eslint-config-next": "12.1.0", - "typescript": "4.6.2" + "typescript": "5.2.2" } -} \ No newline at end of file +} diff --git a/hosting/couchdb/runner.sh b/hosting/couchdb/runner.sh index b576c886c2..e56b8e0e7f 100644 --- a/hosting/couchdb/runner.sh +++ b/hosting/couchdb/runner.sh @@ -30,10 +30,18 @@ elif [[ "${TARGETBUILD}" = "single" ]]; then # mount, so we use that for all persistent data. sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini +elif [[ "${TARGETBUILD}" = "docker-compose" ]]; then + # We remove the database_dir and view_index_dir settings from the local.ini + # in docker-compose because it will default to /opt/couchdb/data which is what + # our docker-compose was using prior to us switching to using our own CouchDB + # image. + sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini + sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini + sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini elif [[ -n $KUBERNETES_SERVICE_HOST ]]; then # In Kubernetes the directory /opt/couchdb/data has a persistent volume # mount for storing database data. - sed -i "s#DATA_DIR#/opt/couchdb/data#g" /opt/clouseau/clouseau.ini + sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini # We remove the database_dir and view_index_dir settings from the local.ini # in Kubernetes because it will default to /opt/couchdb/data which is what diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index 8f66d211f7..34cae92dc7 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -57,7 +57,6 @@ services: depends_on: - redis-service - minio-service - - couch-init minio-service: restart: unless-stopped @@ -70,7 +69,7 @@ services: MINIO_BROWSER: "off" command: server /data --console-address ":9001" healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + test: "timeout 5s bash -c ':> /dev/tcp/127.0.0.1/9000' || exit 1" interval: 30s timeout: 20s retries: 3 @@ -98,30 +97,19 @@ services: couchdb-service: restart: unless-stopped - image: ibmcom/couchdb3 + image: budibase/couchdb + pull_policy: always environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} + - TARGETBUILD=docker-compose volumes: - couchdb3_data:/opt/couchdb/data - couch-init: - image: curlimages/curl - environment: - PUT_CALL: "curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984" - depends_on: - - couchdb-service - command: - [ - "sh", - "-c", - "sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;", - ] - redis-service: restart: unless-stopped image: redis - command: redis-server --requirepass ${REDIS_PASSWORD} + command: redis-server --requirepass "${REDIS_PASSWORD}" volumes: - redis_data:/data diff --git a/hosting/proxy/nginx.prod.conf b/hosting/proxy/nginx.prod.conf index 88f9645f80..65cc3ff390 100644 --- a/hosting/proxy/nginx.prod.conf +++ b/hosting/proxy/nginx.prod.conf @@ -257,6 +257,7 @@ http { access_log off; allow 127.0.0.1; + allow 10.0.0.0/8; deny all; location /nginx_status { diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index e9ff6c6596..67ac677984 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18-slim as build +FROM node:20-slim as build # install node-gyp dependencies RUN apt-get update && apt-get install -y --no-install-recommends g++ make python3 jq @@ -42,7 +42,7 @@ COPY packages/string-templates packages/string-templates FROM budibase/couchdb as runner ARG TARGETARCH ENV TARGETARCH $TARGETARCH -ENV NODE_MAJOR 18 +ENV NODE_MAJOR 20 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) # e.g. docker build --build-arg TARGETBUILD=aas .... ARG TARGETBUILD=single diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index f4b2b5b127..886da4c916 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -7,7 +7,7 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME [[ -z "${BUDIBASE_ENVIRONMENT}" ]] && export BUDIBASE_ENVIRONMENT=PRODUCTION [[ -z "${CLUSTER_PORT}" ]] && export CLUSTER_PORT=80 [[ -z "${DEPLOYMENT_ENVIRONMENT}" ]] && export DEPLOYMENT_ENVIRONMENT=docker -[[ -z "${MINIO_URL}" ]] && export MINIO_URL=http://127.0.0.1:9000 +[[ -z "${MINIO_URL}" ]] && [[ -z "${USE_S3}" ]] && export MINIO_URL=http://127.0.0.1:9000 [[ -z "${NODE_ENV}" ]] && export NODE_ENV=production [[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU [[ -z "${TENANT_FEATURE_FLAGS}" ]] && export TENANT_FEATURE_FLAGS="*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR" @@ -77,7 +77,12 @@ mkdir -p ${DATA_DIR}/minio chown -R couchdb:couchdb ${DATA_DIR}/couch redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 & /bbcouch-runner.sh & -/minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 & + +# only start minio if use s3 isn't passed +if [[ -z "${USE_S3}" ]]; then + /minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 & +fi + /etc/init.d/nginx restart if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then # Add monthly cron job to renew certbot certificate diff --git a/lerna.json b/lerna.json index d6a5f41281..71c53cd3fa 100644 --- a/lerna.json +++ b/lerna.json @@ -1,10 +1,13 @@ { - "version": "2.13.35", + "version": "2.14.2", "npmClient": "yarn", "packages": [ - "packages/*" + "packages/*", + "!packages/account-portal", + "packages/account-portal/packages/*" ], "useNx": true, + "concurrency": 20, "command": { "publish": { "ignoreChanges": [ diff --git a/package.json b/package.json index 2978483448..8c2b6b099c 100644 --- a/package.json +++ b/package.json @@ -6,25 +6,26 @@ "@babel/eslint-parser": "^7.22.5", "@babel/preset-env": "^7.22.5", "@esbuild-plugins/tsconfig-paths": "^0.1.2", - "@typescript-eslint/parser": "6.7.2", + "@types/node": "20.10.0", + "@typescript-eslint/parser": "6.9.0", "esbuild": "^0.18.17", "esbuild-node-externals": "^1.8.0", - "eslint": "^8.44.0", + "eslint": "^8.52.0", "eslint-plugin-import": "^2.29.0", "eslint-plugin-local-rules": "^2.0.0", - "eslint-plugin-svelte": "^2.32.2", + "eslint-plugin-svelte": "^2.34.0", "husky": "^8.0.3", "kill-port": "^1.6.1", "lerna": "7.1.1", "madge": "^6.0.0", - "minimist": "^1.2.8", "nx": "16.4.3", "nx-cloud": "16.0.5", "prettier": "2.8.8", "prettier-plugin-svelte": "^2.3.0", "svelte": "3.49.0", - "svelte-eslint-parser": "^0.32.0", - "typescript": "5.2.2" + "svelte-eslint-parser": "^0.33.1", + "typescript": "5.2.2", + "yargs": "^17.7.2" }, "scripts": { "preinstall": "node scripts/syncProPackage.js", @@ -39,13 +40,16 @@ "nuke": "yarn run nuke:packages && yarn run nuke:docker", "nuke:packages": "yarn run restore", "nuke:docker": "lerna run --stream dev:stack:nuke", - "clean": "lerna clean -y", + "clean": "lerna clean -y && echo Cleaning top level node modules 🧹 && rm -rf ./node_modules && echo Done! 🚀", "kill-builder": "kill-port 3000", "kill-server": "kill-port 4001 4002", - "kill-all": "yarn run kill-builder && yarn run kill-server", - "dev": "yarn run kill-all && lerna run --parallel prebuild && lerna run --stream dev:builder", - "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", - "dev:server": "yarn run kill-server && lerna run --stream dev:builder --scope @budibase/worker --scope @budibase/server", + "kill-accountportal": "kill-port 3001 4003", + "kill-all": "yarn run kill-builder && yarn run kill-server && yarn kill-accountportal", + "dev": "yarn run kill-all && lerna run --parallel prebuild && lerna run --stream dev --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server", + "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream dev --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server", + "dev:server": "yarn run kill-server && lerna run --stream dev --scope @budibase/worker --scope @budibase/server", + "dev:accountportal": "yarn kill-accountportal && lerna run dev --stream --scope @budibase/account-portal-ui --scope @budibase/account-portal-server", + "dev:all": "yarn run kill-all && lerna run --stream dev", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", "test": "lerna run --stream test --stream", @@ -79,11 +83,14 @@ "security:audit": "node scripts/audit.js", "postinstall": "husky install", "submodules:load": "git submodule init && git submodule update && yarn", - "submodules:unload": "git submodule deinit --all && yarn" + "submodules:unload": "git submodule deinit --all && yarn", + "add-app-migration": "node scripts/add-app-migration.js --title" }, "workspaces": { "packages": [ - "packages/*" + "packages/*", + "!packages/account-portal", + "packages/account-portal/packages/*" ] }, "resolutions": { @@ -93,7 +100,7 @@ "@budibase/types": "0.0.0" }, "engines": { - "node": ">=18.0.0 <19.0.0" + "node": ">=20.0.0 <21.0.0" }, "dependencies": {} } diff --git a/packages/account-portal b/packages/account-portal new file mode 160000 index 0000000000..b11e6b4737 --- /dev/null +++ b/packages/account-portal @@ -0,0 +1 @@ +Subproject commit b11e6b47370d9b77c63648b45929c86bfed6360c diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 306aabfe6a..343bc67449 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -21,7 +21,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/nano": "10.1.3", + "@budibase/nano": "10.1.4", "@budibase/pouchdb-replication-stream": "1.2.10", "@budibase/shared-core": "0.0.0", "@budibase/types": "0.0.0", @@ -32,6 +32,7 @@ "bcryptjs": "2.4.3", "bull": "4.10.1", "correlation-id": "4.0.0", + "dd-trace": "3.13.2", "dotenv": "16.0.1", "ioredis": "5.3.2", "joi": "17.6.0", @@ -64,7 +65,6 @@ "@types/cookies": "0.7.8", "@types/jest": "29.5.5", "@types/lodash": "4.14.200", - "@types/node": "18.17.0", "@types/node-fetch": "2.6.4", "@types/pouchdb": "6.4.0", "@types/redlock": "4.0.3", @@ -73,8 +73,8 @@ "@types/uuid": "8.3.4", "chance": "1.1.8", "ioredis-mock": "8.9.0", - "jest": "29.6.2", - "jest-environment-node": "29.6.2", + "jest": "29.7.0", + "jest-environment-node": "29.7.0", "jest-serial-runner": "1.2.1", "pino-pretty": "10.0.0", "pouchdb-adapter-memory": "7.2.2", diff --git a/packages/backend-core/src/cache/generic.ts b/packages/backend-core/src/cache/generic.ts index 7a2be5a0f0..3ac323a8d4 100644 --- a/packages/backend-core/src/cache/generic.ts +++ b/packages/backend-core/src/cache/generic.ts @@ -18,14 +18,15 @@ export enum TTL { ONE_DAY = 86400, } -function performExport(funcName: string) { - // @ts-ignore - return (...args: any) => GENERIC[funcName](...args) -} - -export const keys = performExport("keys") -export const get = performExport("get") -export const store = performExport("store") -export const destroy = performExport("delete") -export const withCache = performExport("withCache") -export const bustCache = performExport("bustCache") +export const keys = (...args: Parameters) => + GENERIC.keys(...args) +export const get = (...args: Parameters) => + GENERIC.get(...args) +export const store = (...args: Parameters) => + GENERIC.store(...args) +export const destroy = (...args: Parameters) => + GENERIC.delete(...args) +export const withCache = (...args: Parameters) => + GENERIC.withCache(...args) +export const bustCache = (...args: Parameters) => + GENERIC.bustCache(...args) diff --git a/packages/backend-core/src/cache/passwordReset.ts b/packages/backend-core/src/cache/passwordReset.ts index 7f5a93f149..db32b520f7 100644 --- a/packages/backend-core/src/cache/passwordReset.ts +++ b/packages/backend-core/src/cache/passwordReset.ts @@ -1,6 +1,6 @@ import * as redis from "../redis/init" import * as utils from "../utils" -import { Duration, DurationType } from "../utils" +import { Duration } from "../utils" const TTL_SECONDS = Duration.fromHours(1).toSeconds() @@ -32,7 +32,18 @@ export async function getCode(code: string): Promise { const client = await redis.getPasswordResetClient() const value = (await client.get(code)) as PasswordReset | undefined if (!value) { - throw "Provided information is not valid, cannot reset password - please try again." + throw new Error( + "Provided information is not valid, cannot reset password - please try again." + ) } return value } + +/** + * Given a reset code this will invalidate it. + * @param code The code provided via the email link. + */ +export async function invalidateCode(code: string): Promise { + const client = await redis.getPasswordResetClient() + await client.delete(code) +} diff --git a/packages/backend-core/src/cache/tests/writethrough.spec.ts b/packages/backend-core/src/cache/tests/writethrough.spec.ts index 97d3ece7a6..37887b4bd9 100644 --- a/packages/backend-core/src/cache/tests/writethrough.spec.ts +++ b/packages/backend-core/src/cache/tests/writethrough.spec.ts @@ -1,15 +1,16 @@ import { DBTestConfiguration } from "../../../tests/extra" -import { - structures, - expectFunctionWasCalledTimesWith, - mocks, -} from "../../../tests" +import { structures } from "../../../tests" import { Writethrough } from "../writethrough" import { getDB } from "../../db" +import { Document } from "@budibase/types" import tk from "timekeeper" tk.freeze(Date.now()) +interface ValueDoc extends Document { + value: any +} + const DELAY = 5000 describe("writethrough", () => { @@ -117,7 +118,7 @@ describe("writethrough", () => { describe("get", () => { it("should be able to retrieve", async () => { await config.doInTenant(async () => { - const response = await writethrough.get(docId) + const response = await writethrough.get(docId) expect(response.value).toBe(4) }) }) diff --git a/packages/backend-core/src/cache/writethrough.ts b/packages/backend-core/src/cache/writethrough.ts index c331d791a6..5cafe418d7 100644 --- a/packages/backend-core/src/cache/writethrough.ts +++ b/packages/backend-core/src/cache/writethrough.ts @@ -7,7 +7,7 @@ import * as locks from "../redis/redlockImpl" const DEFAULT_WRITE_RATE_MS = 10000 let CACHE: BaseCache | null = null -interface CacheItem { +interface CacheItem { doc: any lastWrite: number } @@ -24,7 +24,10 @@ function makeCacheKey(db: Database, key: string) { return db.name + key } -function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem { +function makeCacheItem( + doc: T, + lastWrite: number | null = null +): CacheItem { return { doc, lastWrite: lastWrite || Date.now() } } @@ -35,7 +38,7 @@ async function put( ) { const cache = await getCache() const key = doc._id - let cacheItem: CacheItem | undefined + let cacheItem: CacheItem | undefined if (key) { cacheItem = await cache.get(makeCacheKey(db, key)) } @@ -53,11 +56,8 @@ async function put( const writeDb = async (toWrite: any) => { // doc should contain the _id and _rev const response = await db.put(toWrite, { force: true }) - output = { - ...doc, - _id: response.id, - _rev: response.rev, - } + output._id = response.id + output._rev = response.rev } try { await writeDb(doc) @@ -84,12 +84,12 @@ async function put( return { ok: true, id: output._id, rev: output._rev } } -async function get(db: Database, id: string): Promise { +async function get(db: Database, id: string): Promise { const cache = await getCache() const cacheKey = makeCacheKey(db, id) - let cacheItem: CacheItem = await cache.get(cacheKey) + let cacheItem: CacheItem = await cache.get(cacheKey) if (!cacheItem) { - const doc = await db.get(id) + const doc = await db.get(id) cacheItem = makeCacheItem(doc) await cache.store(cacheKey, cacheItem) } @@ -123,8 +123,8 @@ export class Writethrough { return put(this.db, doc, writeRateMs) } - async get(id: string) { - return get(this.db, id) + async get(id: string) { + return get(this.db, id) } async remove(docOrId: any, rev?: any) { diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts index 8ef34196ed..aee099e10a 100644 --- a/packages/backend-core/src/constants/misc.ts +++ b/packages/backend-core/src/constants/misc.ts @@ -11,24 +11,7 @@ export enum Cookie { OIDC_CONFIG = "budibase:oidc:config", } -export enum Header { - API_KEY = "x-budibase-api-key", - LICENSE_KEY = "x-budibase-license-key", - API_VER = "x-budibase-api-version", - APP_ID = "x-budibase-app-id", - SESSION_ID = "x-budibase-session-id", - TYPE = "x-budibase-type", - PREVIEW_ROLE = "x-budibase-role", - TENANT_ID = "x-budibase-tenant-id", - VERIFICATION_CODE = "x-budibase-verification-code", - RETURN_VERIFICATION_CODE = "x-budibase-return-verification-code", - RESET_PASSWORD_CODE = "x-budibase-reset-password-code", - RETURN_RESET_PASSWORD_CODE = "x-budibase-return-reset-password-code", - TOKEN = "x-budibase-token", - CSRF_TOKEN = "x-csrf-token", - CORRELATION_ID = "x-budibase-correlation-id", - AUTHORIZATION = "authorization", -} +export { Header } from "@budibase/shared-core" export enum GlobalRole { OWNER = "owner", diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 983a4d20e1..36fd5dcb48 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -134,7 +134,7 @@ export async function doInContext(appId: string, task: any): Promise { } export async function doInTenant( - tenantId: string | null, + tenantId: string | undefined, task: () => T ): Promise { // make sure default always selected in single tenancy @@ -335,3 +335,11 @@ export function isScim(): boolean { const scimCall = context?.isScim return !!scimCall } + +export function getCurrentContext(): ContextMap | undefined { + try { + return Context.get() + } catch (e) { + return undefined + } +} diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts index a1606a17b9..f73dc9f5c7 100644 --- a/packages/backend-core/src/context/types.ts +++ b/packages/backend-core/src/context/types.ts @@ -1,4 +1,5 @@ import { IdentityContext } from "@budibase/types" +import { ExecutionTimeTracker } from "../timers" // keep this out of Budibase types, don't want to expose context info export type ContextMap = { @@ -9,4 +10,5 @@ export type ContextMap = { isScim?: boolean automationId?: string isMigrating?: boolean + jsExecutionTracker?: ExecutionTimeTracker } diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 8588a7157a..3fec573bb9 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -17,6 +17,7 @@ import { directCouchUrlCall } from "./utils" import { getPouchDB } from "./pouchDB" import { WriteStream, ReadStream } from "fs" import { newid } from "../../docIds/newid" +import { DDInstrumentedDatabase } from "../instrumentation" function buildNano(couchInfo: { url: string; cookie: string }) { return Nano({ @@ -35,10 +36,8 @@ export function DatabaseWithConnection( connection: string, opts?: DatabaseOpts ) { - if (!connection) { - throw new Error("Must provide connection details") - } - return new DatabaseImpl(dbName, opts, connection) + const db = new DatabaseImpl(dbName, opts, connection) + return new DDInstrumentedDatabase(db) } export class DatabaseImpl implements Database { diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts index 3e69d49f0e..197770298e 100644 --- a/packages/backend-core/src/db/db.ts +++ b/packages/backend-core/src/db/db.ts @@ -1,8 +1,9 @@ import { directCouchQuery, DatabaseImpl } from "./couch" import { CouchFindOptions, Database, DatabaseOpts } from "@budibase/types" +import { DDInstrumentedDatabase } from "./instrumentation" export function getDB(dbName: string, opts?: DatabaseOpts): Database { - return new DatabaseImpl(dbName, opts) + return new DDInstrumentedDatabase(new DatabaseImpl(dbName, opts)) } // we have to use a callback for this so that we can close diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts new file mode 100644 index 0000000000..ba5febcba6 --- /dev/null +++ b/packages/backend-core/src/db/instrumentation.ts @@ -0,0 +1,156 @@ +import { + DocumentScope, + DocumentDestroyResponse, + DocumentInsertResponse, + DocumentBulkResponse, + OkResponse, +} from "@budibase/nano" +import { + AllDocsResponse, + AnyDocument, + Database, + DatabaseDumpOpts, + DatabasePutOpts, + DatabaseQueryOpts, + Document, +} from "@budibase/types" +import tracer from "dd-trace" +import { Writable } from "stream" + +export class DDInstrumentedDatabase implements Database { + constructor(private readonly db: Database) {} + + get name(): string { + return this.db.name + } + + exists(): Promise { + return tracer.trace("db.exists", span => { + span?.addTags({ db_name: this.name }) + return this.db.exists() + }) + } + + checkSetup(): Promise> { + return tracer.trace("db.checkSetup", span => { + span?.addTags({ db_name: this.name }) + return this.db.checkSetup() + }) + } + + get(id?: string | undefined): Promise { + return tracer.trace("db.get", span => { + span?.addTags({ db_name: this.name, doc_id: id }) + return this.db.get(id) + }) + } + + getMultiple( + ids: string[], + opts?: { allowMissing?: boolean | undefined } | undefined + ): Promise { + return tracer.trace("db.getMultiple", span => { + span?.addTags({ + db_name: this.name, + num_docs: ids.length, + allow_missing: opts?.allowMissing, + }) + return this.db.getMultiple(ids, opts) + }) + } + + remove( + id: string | Document, + rev?: string | undefined + ): Promise { + return tracer.trace("db.remove", span => { + span?.addTags({ db_name: this.name, doc_id: id }) + return this.db.remove(id, rev) + }) + } + + put( + document: AnyDocument, + opts?: DatabasePutOpts | undefined + ): Promise { + return tracer.trace("db.put", span => { + span?.addTags({ db_name: this.name, doc_id: document._id }) + return this.db.put(document, opts) + }) + } + + bulkDocs(documents: AnyDocument[]): Promise { + return tracer.trace("db.bulkDocs", span => { + span?.addTags({ db_name: this.name, num_docs: documents.length }) + return this.db.bulkDocs(documents) + }) + } + + allDocs( + params: DatabaseQueryOpts + ): Promise> { + return tracer.trace("db.allDocs", span => { + span?.addTags({ db_name: this.name }) + return this.db.allDocs(params) + }) + } + + query( + viewName: string, + params: DatabaseQueryOpts + ): Promise> { + return tracer.trace("db.query", span => { + span?.addTags({ db_name: this.name, view_name: viewName }) + return this.db.query(viewName, params) + }) + } + + destroy(): Promise { + return tracer.trace("db.destroy", span => { + span?.addTags({ db_name: this.name }) + return this.db.destroy() + }) + } + + compact(): Promise { + return tracer.trace("db.compact", span => { + span?.addTags({ db_name: this.name }) + return this.db.compact() + }) + } + + dump(stream: Writable, opts?: DatabaseDumpOpts | undefined): Promise { + return tracer.trace("db.dump", span => { + span?.addTags({ db_name: this.name }) + return this.db.dump(stream, opts) + }) + } + + load(...args: any[]): Promise { + return tracer.trace("db.load", span => { + span?.addTags({ db_name: this.name }) + return this.db.load(...args) + }) + } + + createIndex(...args: any[]): Promise { + return tracer.trace("db.createIndex", span => { + span?.addTags({ db_name: this.name }) + return this.db.createIndex(...args) + }) + } + + deleteIndex(...args: any[]): Promise { + return tracer.trace("db.deleteIndex", span => { + span?.addTags({ db_name: this.name }) + return this.db.deleteIndex(...args) + }) + } + + getIndexes(...args: any[]): Promise { + return tracer.trace("db.getIndexes", span => { + span?.addTags({ db_name: this.name }) + return this.db.getIndexes(...args) + }) + } +} diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index ed882fe96a..138dbbd9e0 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -107,6 +107,7 @@ const environment = { ENCRYPTION_KEY: process.env.ENCRYPTION_KEY, API_ENCRYPTION_KEY: getAPIEncryptionKey(), COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", + COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984", COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID, diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index d04f48e5fc..7bf26f3688 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -33,6 +33,7 @@ export * as docUpdates from "./docUpdates" export * from "./utils/Duration" export { SearchParams } from "./db" export * as docIds from "./docIds" +export * as security from "./security" // Add context to tenancy for backwards compatibility // only do this for external usages to prevent internal // circular dependencies diff --git a/packages/backend-core/src/logging/pino/logger.ts b/packages/backend-core/src/logging/pino/logger.ts index 7c444a3a59..7a051e7f12 100644 --- a/packages/backend-core/src/logging/pino/logger.ts +++ b/packages/backend-core/src/logging/pino/logger.ts @@ -5,6 +5,8 @@ import { IdentityType } from "@budibase/types" import env from "../../environment" import * as context from "../../context" import * as correlation from "../correlation" +import tracer from "dd-trace" +import { formats } from "dd-trace/ext" import { localFileDestination } from "../system" @@ -115,6 +117,11 @@ if (!env.DISABLE_PINO_LOGGER) { correlationId: correlation.getId(), } + const span = tracer.scope().active() + if (span) { + tracer.inject(span.context(), formats.LOG, contextObject) + } + const mergingObject: any = { err: error, pid: process.pid, diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 9b44eace49..57ead0e809 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -260,12 +260,12 @@ export async function listAllObjects(bucketName: string, path: string) { } /** - * Generate a presigned url with a default TTL of 36 hours + * Generate a presigned url with a default TTL of 1 hour */ export function getPresignedUrl( bucketName: string, key: string, - durationSeconds: number = 129600 + durationSeconds: number = 3600 ) { const objectStore = ObjectStore(bucketName, { presigning: true }) const params = { diff --git a/packages/backend-core/src/queue/constants.ts b/packages/backend-core/src/queue/constants.ts index e1ffcfee36..eb4f21aced 100644 --- a/packages/backend-core/src/queue/constants.ts +++ b/packages/backend-core/src/queue/constants.ts @@ -3,4 +3,5 @@ export enum JobQueue { APP_BACKUP = "appBackupQueue", AUDIT_LOG = "auditLogQueue", SYSTEM_EVENT_QUEUE = "systemEventQueue", + APP_MIGRATION = "appMigration", } diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index a8add7ecb6..c05bbffbe9 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -15,6 +15,7 @@ function newJob(queue: string, message: any) { timestamp: Date.now(), queue: queue, data: message, + opts: {}, } } @@ -68,6 +69,10 @@ class InMemoryQueue { }) } + async isReady() { + return true + } + // simply puts a message to the queue and emits to the queue for processing /** * Simple function to replicate the add message functionality of Bull, putting diff --git a/packages/backend-core/src/queue/listeners.ts b/packages/backend-core/src/queue/listeners.ts index 42e3172364..063a01bd2f 100644 --- a/packages/backend-core/src/queue/listeners.ts +++ b/packages/backend-core/src/queue/listeners.ts @@ -87,6 +87,7 @@ enum QueueEventType { APP_BACKUP_EVENT = "app-backup-event", AUDIT_LOG_EVENT = "audit-log-event", SYSTEM_EVENT = "system-event", + APP_MIGRATION = "app-migration", } const EventTypeMap: { [key in JobQueue]: QueueEventType } = { @@ -94,6 +95,7 @@ const EventTypeMap: { [key in JobQueue]: QueueEventType } = { [JobQueue.APP_BACKUP]: QueueEventType.APP_BACKUP_EVENT, [JobQueue.AUDIT_LOG]: QueueEventType.AUDIT_LOG_EVENT, [JobQueue.SYSTEM_EVENT_QUEUE]: QueueEventType.SYSTEM_EVENT, + [JobQueue.APP_MIGRATION]: QueueEventType.APP_MIGRATION, } function logging(queue: Queue, jobQueue: JobQueue) { diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index 0657437a3b..b95dace5b2 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -47,7 +47,7 @@ export function createQueue( cleanupInterval = timers.set(cleanup, CLEANUP_PERIOD_MS) // fire off an initial cleanup cleanup().catch(err => { - console.error(`Unable to cleanup automation queue initially - ${err}`) + console.error(`Unable to cleanup ${jobQueue} initially - ${err}`) }) } return queue diff --git a/packages/backend-core/src/redis/redis.ts b/packages/backend-core/src/redis/redis.ts index 701e262091..d15453ba62 100644 --- a/packages/backend-core/src/redis/redis.ts +++ b/packages/backend-core/src/redis/redis.ts @@ -18,6 +18,7 @@ import { SelectableDatabase, getRedisConnectionDetails, } from "./utils" +import { logAlert } from "../logging" import * as timers from "../timers" const RETRY_PERIOD_MS = 2000 @@ -39,21 +40,16 @@ function pickClient(selectDb: number): any { return CLIENTS[selectDb] } -function connectionError( - selectDb: number, - timeout: NodeJS.Timeout, - err: Error | string -) { +function connectionError(timeout: NodeJS.Timeout, err: Error | string) { // manually shut down, ignore errors if (CLOSED) { return } - pickClient(selectDb).disconnect() CLOSED = true // always clear this on error clearTimeout(timeout) CONNECTED = false - console.error("Redis connection failed - " + err) + logAlert("Redis connection failed", err) setTimeout(() => { init() }, RETRY_PERIOD_MS) @@ -79,11 +75,7 @@ function init(selectDb = DEFAULT_SELECT_DB) { // start the timer - only allowed 5 seconds to connect timeout = setTimeout(() => { if (!CONNECTED) { - connectionError( - selectDb, - timeout, - "Did not successfully connect in timeout" - ) + connectionError(timeout, "Did not successfully connect in timeout") } }, STARTUP_TIMEOUT_MS) @@ -106,12 +98,13 @@ function init(selectDb = DEFAULT_SELECT_DB) { // allow the process to exit return } - connectionError(selectDb, timeout, err) + connectionError(timeout, err) }) client.on("error", (err: Error) => { - connectionError(selectDb, timeout, err) + connectionError(timeout, err) }) client.on("connect", () => { + console.log(`Connected to Redis DB: ${selectDb}`) clearTimeout(timeout) CONNECTED = true }) diff --git a/packages/backend-core/src/redis/redlockImpl.ts b/packages/backend-core/src/redis/redlockImpl.ts index 4de2516ab2..7009dc6f55 100644 --- a/packages/backend-core/src/redis/redlockImpl.ts +++ b/packages/backend-core/src/redis/redlockImpl.ts @@ -2,7 +2,6 @@ import Redlock from "redlock" import { getLockClient } from "./init" import { LockOptions, LockType } from "@budibase/types" import * as context from "../context" -import { logWarn } from "../logging" import { utils } from "@budibase/shared-core" import { Duration } from "../utils" @@ -137,7 +136,6 @@ export async function doWithLock( const result = await task() return { executed: true, result } } catch (e: any) { - logWarn(`lock type: ${opts.type} error`, e) // lock limit exceeded if (e.name === "LockError") { if (opts.type === LockType.TRY_ONCE) { diff --git a/packages/backend-core/src/security/auth.ts b/packages/backend-core/src/security/auth.ts new file mode 100644 index 0000000000..c90d9df09b --- /dev/null +++ b/packages/backend-core/src/security/auth.ts @@ -0,0 +1,24 @@ +import { env } from ".." + +export const PASSWORD_MIN_LENGTH = +(process.env.PASSWORD_MIN_LENGTH || 8) +export const PASSWORD_MAX_LENGTH = +(process.env.PASSWORD_MAX_LENGTH || 512) + +export function validatePassword( + password: string +): { valid: true } | { valid: false; error: string } { + if (!password || password.length < PASSWORD_MIN_LENGTH) { + return { + valid: false, + error: `Password invalid. Minimum ${PASSWORD_MIN_LENGTH} characters.`, + } + } + + if (password.length > PASSWORD_MAX_LENGTH) { + return { + valid: false, + error: `Password invalid. Maximum ${PASSWORD_MAX_LENGTH} characters.`, + } + } + + return { valid: true } +} diff --git a/packages/backend-core/src/security/index.ts b/packages/backend-core/src/security/index.ts new file mode 100644 index 0000000000..306751af96 --- /dev/null +++ b/packages/backend-core/src/security/index.ts @@ -0,0 +1 @@ +export * from "./auth" diff --git a/packages/backend-core/src/security/tests/auth.spec.ts b/packages/backend-core/src/security/tests/auth.spec.ts new file mode 100644 index 0000000000..b1835fdfb3 --- /dev/null +++ b/packages/backend-core/src/security/tests/auth.spec.ts @@ -0,0 +1,45 @@ +import { generator } from "../../../tests" +import { PASSWORD_MAX_LENGTH, validatePassword } from "../auth" + +describe("auth", () => { + describe("validatePassword", () => { + it("a valid password returns successful", () => { + expect(validatePassword("password")).toEqual({ valid: true }) + }) + + it.each([ + ["undefined", undefined], + ["null", null], + ["empty", ""], + ])("%s returns unsuccessful", (_, password) => { + expect(validatePassword(password as string)).toEqual({ + valid: false, + error: "Password invalid. Minimum 8 characters.", + }) + }) + + it.each([ + generator.word({ length: PASSWORD_MAX_LENGTH }), + generator.paragraph().substring(0, PASSWORD_MAX_LENGTH), + ])(`can use passwords up to 512 characters in length`, password => { + expect(validatePassword(password)).toEqual({ + valid: true, + }) + }) + + it.each([ + generator.word({ length: PASSWORD_MAX_LENGTH + 1 }), + generator + .paragraph({ sentences: 50 }) + .substring(0, PASSWORD_MAX_LENGTH + 1), + ])( + `passwords cannot have more than ${PASSWORD_MAX_LENGTH} characters`, + password => { + expect(validatePassword(password)).toEqual({ + valid: false, + error: "Password invalid. Maximum 512 characters.", + }) + } + ) + }) +}) diff --git a/packages/backend-core/src/tenancy/tenancy.ts b/packages/backend-core/src/tenancy/tenancy.ts index 3603ef3462..8835960ca5 100644 --- a/packages/backend-core/src/tenancy/tenancy.ts +++ b/packages/backend-core/src/tenancy/tenancy.ts @@ -39,7 +39,7 @@ const ALL_STRATEGIES = Object.values(TenantResolutionStrategy) export const getTenantIDFromCtx = ( ctx: BBContext, opts: GetTenantIdOptions -): string | null => { +): string | undefined => { // exit early if not multi-tenant if (!isMultiTenant()) { return DEFAULT_TENANT_ID @@ -144,5 +144,5 @@ export const getTenantIDFromCtx = ( ctx.throw(403, "Tenant id not set") } - return null + return undefined } diff --git a/packages/backend-core/src/tenancy/tests/tenancy.spec.ts b/packages/backend-core/src/tenancy/tests/tenancy.spec.ts index ebeaca074c..95dd76a6dd 100644 --- a/packages/backend-core/src/tenancy/tests/tenancy.spec.ts +++ b/packages/backend-core/src/tenancy/tests/tenancy.spec.ts @@ -157,12 +157,12 @@ describe("getTenantIDFromCtx", () => { TenantResolutionStrategy.PATH, ], } - expect(getTenantIDFromCtx(ctx, mockOpts)).toBeNull() + expect(getTenantIDFromCtx(ctx, mockOpts)).toBeUndefined() expect(ctx.throw).toBeCalledTimes(1) expect(ctx.throw).toBeCalledWith(403, "Tenant id not set") }) - it("returns null if allowNoTenant is true", () => { + it("returns undefined if allowNoTenant is true", () => { const ctx = createCtx({}) mockOpts = { allowNoTenant: true, @@ -172,7 +172,7 @@ describe("getTenantIDFromCtx", () => { TenantResolutionStrategy.PATH, ], } - expect(getTenantIDFromCtx(ctx, mockOpts)).toBeNull() + expect(getTenantIDFromCtx(ctx, mockOpts)).toBeUndefined() }) }) diff --git a/packages/backend-core/src/timers/timers.ts b/packages/backend-core/src/timers/timers.ts index 000be74821..9121c576cd 100644 --- a/packages/backend-core/src/timers/timers.ts +++ b/packages/backend-core/src/timers/timers.ts @@ -20,3 +20,41 @@ export function cleanup() { } intervals = [] } + +export class ExecutionTimeoutError extends Error { + public readonly name = "ExecutionTimeoutError" +} + +export class ExecutionTimeTracker { + static withLimit(limitMs: number) { + return new ExecutionTimeTracker(limitMs) + } + + constructor(readonly limitMs: number) {} + + private totalTimeMs = 0 + + track(f: () => T): T { + this.checkLimit() + const start = process.hrtime.bigint() + try { + return f() + } finally { + const end = process.hrtime.bigint() + this.totalTimeMs += Number(end - start) / 1e6 + this.checkLimit() + } + } + + get elapsedMS() { + return this.totalTimeMs + } + + checkLimit() { + if (this.totalTimeMs > this.limitMs) { + throw new ExecutionTimeoutError( + `Execution time limit of ${this.limitMs}ms exceeded: ${this.totalTimeMs}ms` + ) + } + } +} diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 326bed3cc5..3214b3ab63 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -2,7 +2,7 @@ import env from "../environment" import * as eventHelpers from "./events" import * as accountSdk from "../accounts" import * as cache from "../cache" -import { doInTenant, getGlobalDB, getIdentity, getTenantId } from "../context" +import { getGlobalDB, getIdentity, getTenantId } from "../context" import * as dbUtils from "../db" import { EmailUnavailableError, HTTPError } from "../errors" import * as platform from "../platform" @@ -27,6 +27,7 @@ import { } from "./utils" import { searchExistingEmails } from "./lookup" import { hash } from "../utils" +import { validatePassword } from "../security" type QuotaUpdateFn = ( change: number, @@ -110,6 +111,12 @@ export class UserDB { if (await UserDB.isPreventPasswordActions(user, account)) { throw new HTTPError("Password change is disabled for this user", 400) } + + const passwordValidation = validatePassword(password) + if (!passwordValidation.valid) { + throw new HTTPError(passwordValidation.error, 400) + } + hashedPassword = opts.hashPassword ? await hash(password) : password } else if (dbUser) { hashedPassword = dbUser.password diff --git a/packages/backend-core/src/utils/Duration.ts b/packages/backend-core/src/utils/Duration.ts index 3c7ef23b11..730b59d1dc 100644 --- a/packages/backend-core/src/utils/Duration.ts +++ b/packages/backend-core/src/utils/Duration.ts @@ -49,4 +49,8 @@ export class Duration { static fromDays(duration: number) { return Duration.from(DurationType.DAYS, duration) } + + static fromMilliseconds(duration: number) { + return Duration.from(DurationType.MILLISECONDS, duration) + } } diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index ee1ef6da0c..30cf55b149 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -31,8 +31,8 @@ export async function resolveAppUrl(ctx: Ctx) { const appUrl = ctx.path.split("/")[2] let possibleAppUrl = `/${appUrl.toLowerCase()}` - let tenantId: string | null = context.getTenantId() - if (env.MULTI_TENANCY) { + let tenantId: string | undefined = context.getTenantId() + if (!env.isDev() && env.MULTI_TENANCY) { // always use the tenant id from the subdomain in multi tenancy // this ensures the logged-in user tenant id doesn't overwrite // e.g. in the case of viewing a public app while already logged-in to another tenant @@ -41,7 +41,7 @@ export async function resolveAppUrl(ctx: Ctx) { }) } - // search prod apps for a url that matches + // search prod apps for an url that matches const apps: App[] = await context.doInTenant( tenantId, () => getAllApps({ dev: false }) as Promise @@ -96,7 +96,7 @@ export async function getAppIdFromCtx(ctx: Ctx) { } // look in the path - const pathId = parseAppIdFromUrl(ctx.path) + const pathId = parseAppIdFromUrlPath(ctx.path) if (!appId && pathId) { appId = confirmAppId(pathId) } @@ -116,18 +116,21 @@ export async function getAppIdFromCtx(ctx: Ctx) { // referer header is present from a builder redirect const referer = ctx.request.headers.referer if (!appId && referer?.includes(BUILDER_APP_PREFIX)) { - const refererId = parseAppIdFromUrl(ctx.request.headers.referer) + const refererId = parseAppIdFromUrlPath(ctx.request.headers.referer) appId = confirmAppId(refererId) } return appId } -function parseAppIdFromUrl(url?: string) { +function parseAppIdFromUrlPath(url?: string) { if (!url) { return } - return url.split("/").find(subPath => subPath.startsWith(APP_PREFIX)) + return url + .split("?")[0] // Remove any possible query string + .split("/") + .find(subPath => subPath.startsWith(APP_PREFIX)) } /** diff --git a/packages/backend-core/tests/core/utilities/structures/users.ts b/packages/backend-core/tests/core/utilities/structures/users.ts index 68ee29686c..8f4096d401 100644 --- a/packages/backend-core/tests/core/utilities/structures/users.ts +++ b/packages/backend-core/tests/core/utilities/structures/users.ts @@ -21,7 +21,7 @@ export const user = (userProps?: Partial>): User => { _id: userId, userId, email: newEmail(), - password: "test", + password: "password", roles: { app_test: "admin" }, firstName: generator.first(), lastName: generator.last(), diff --git a/packages/bbui/src/ActionButton/ActionButton.svelte b/packages/bbui/src/ActionButton/ActionButton.svelte index 427a98f888..0e6ec3d155 100644 --- a/packages/bbui/src/ActionButton/ActionButton.svelte +++ b/packages/bbui/src/ActionButton/ActionButton.svelte @@ -130,5 +130,6 @@ max-width: 150px; transform: translateX(-50%); text-align: center; + z-index: 1; } diff --git a/packages/bbui/src/DetailSummary/DetailSummary.svelte b/packages/bbui/src/DetailSummary/DetailSummary.svelte index daa9f3f5ca..2cbb6796f3 100644 --- a/packages/bbui/src/DetailSummary/DetailSummary.svelte +++ b/packages/bbui/src/DetailSummary/DetailSummary.svelte @@ -1,20 +1,17 @@ @@ -81,7 +78,7 @@ var(--spacing-xl); } .property-panel.no-title { - padding: var(--spacing-xl); + padding-top: var(--spacing-xl); } .show { diff --git a/packages/bbui/src/Markdown/MarkdownEditor.svelte b/packages/bbui/src/Markdown/MarkdownEditor.svelte index 888187c8da..2f18c9d634 100644 --- a/packages/bbui/src/Markdown/MarkdownEditor.svelte +++ b/packages/bbui/src/Markdown/MarkdownEditor.svelte @@ -19,7 +19,7 @@ // Ensure the value is updated if the value prop changes outside the editor's // control $: checkValue(value) - $: mde?.codemirror.on("change", debouncedUpdate) + $: mde?.codemirror.on("blur", update) $: if (readonly || disabled) { mde?.togglePreview() } @@ -30,21 +30,10 @@ } } - const debounce = (fn, interval) => { - let timeout - return () => { - clearTimeout(timeout) - timeout = setTimeout(fn, interval) - } - } - const update = () => { latestValue = mde.value() dispatch("change", latestValue) } - - // Debounce the update function to avoid spamming it constantly - const debouncedUpdate = debounce(update, 250) {#key height} diff --git a/packages/builder/package.json b/packages/builder/package.json index 3cc5612652..9472c51965 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "routify -b && vite build --emptyOutDir", "start": "routify -c rollup", - "dev:builder": "routify -c dev:vite", + "dev": "routify -c dev:vite", "dev:vite": "vite --host 0.0.0.0", "rollup": "rollup -c -w", "test": "vitest run", @@ -61,9 +61,9 @@ "@codemirror/theme-one-dark": "^6.1.2", "@codemirror/view": "^6.11.2", "@fontsource/source-sans-pro": "^5.0.3", - "@fortawesome/fontawesome-svg-core": "^6.2.1", - "@fortawesome/free-brands-svg-icons": "^6.2.1", - "@fortawesome/free-solid-svg-icons": "^6.2.1", + "@fortawesome/fontawesome-svg-core": "^6.4.2", + "@fortawesome/free-brands-svg-icons": "^6.4.2", + "@fortawesome/free-solid-svg-icons": "^6.4.2", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", "codemirror": "^5.59.0", @@ -78,25 +78,24 @@ "svelte-dnd-action": "^0.9.8", "svelte-loading-spinners": "^0.1.1", "svelte-portal": "1.0.0", - "yup": "0.29.2" + "yup": "^0.32.11" }, "devDependencies": { - "@babel/core": "^7.12.14", "@babel/plugin-transform-runtime": "^7.13.10", "@babel/preset-env": "^7.13.12", "@rollup/plugin-replace": "^5.0.3", "@roxi/routify": "2.18.12", - "@sveltejs/vite-plugin-svelte": "1.0.1", + "@sveltejs/vite-plugin-svelte": "1.4.0", "@testing-library/jest-dom": "5.17.0", "@testing-library/svelte": "^3.2.2", - "babel-jest": "29.6.2", + "babel-jest": "^29.6.2", "identity-obj-proxy": "^3.0.0", - "jest": "29.6.2", + "jest": "29.7.0", "jsdom": "^21.1.1", "ncp": "^2.0.0", - "svelte": "^3.48.0", + "svelte": "^3.49.0", "svelte-jester": "^1.3.2", - "vite": "^4.4.11", + "vite": "^4.5.0", "vite-plugin-static-copy": "^0.17.0", "vitest": "^0.29.2" }, @@ -115,7 +114,7 @@ } ] }, - "dev:builder": { + "dev": { "dependsOn": [ { "projects": [ diff --git a/packages/builder/src/builderStore/dataBinding.js b/packages/builder/src/builderStore/dataBinding.js index f14ac20792..85d5046d8c 100644 --- a/packages/builder/src/builderStore/dataBinding.js +++ b/packages/builder/src/builderStore/dataBinding.js @@ -510,9 +510,7 @@ const isContextCompatibleWithComponent = (context, component) => { return true } -/** - * Determines the correct category for a given binding. - */ +// Enrich binding category information for certain components const getComponentBindingCategory = (component, context, def) => { // Default category to component name let icon = def.icon @@ -520,14 +518,13 @@ const getComponentBindingCategory = (component, context, def) => { // Form block edge case if (component._component.endsWith("formblock")) { - let contextCategorySuffix = { - form: "Fields", - schema: "Row", + if (context.type === "form") { + category = `${component._instanceName} - Fields` + icon = "Form" + } else if (context.type === "schema") { + category = `${component._instanceName} - Row` + icon = "Data" } - category = `${component._instanceName} - ${ - contextCategorySuffix[context.type] - }` - icon = context.type === "form" ? "Form" : "Data" } return { diff --git a/packages/builder/src/builderStore/index.js b/packages/builder/src/builderStore/index.js index ece17cb46f..dd54dcf13e 100644 --- a/packages/builder/src/builderStore/index.js +++ b/packages/builder/src/builderStore/index.js @@ -8,6 +8,7 @@ import { derived, get } from "svelte/store" import { findComponent, findComponentPath } from "./componentUtils" import { RoleUtils } from "@budibase/frontend-core" import { createHistoryStore } from "builderStore/store/history" +import { cloneDeep } from "lodash/fp" export const store = getFrontendStore() export const automationStore = getAutomationStore() @@ -69,7 +70,14 @@ export const selectedComponent = derived( if (!$selectedScreen || !$store.selectedComponentId) { return null } - return findComponent($selectedScreen?.props, $store.selectedComponentId) + const selected = findComponent( + $selectedScreen?.props, + $store.selectedComponentId + ) + + const clone = selected ? cloneDeep(selected) : selected + store.actions.components.migrateSettings(clone) + return clone } ) diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index 2e22276e50..ce15524bd4 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -86,6 +86,7 @@ const INITIAL_FRONTEND_STATE = { selectedScreenId: null, selectedComponentId: null, selectedLayoutId: null, + hoverComponentId: null, // Client state selectedComponentInstance: null, @@ -113,7 +114,7 @@ export const getFrontendStore = () => { } let clone = cloneDeep(screen) const result = patchFn(clone) - + // An explicit false result means skip this change if (result === false) { return } @@ -602,6 +603,36 @@ export const getFrontendStore = () => { // Finally try an external table return validTables.find(table => table.sourceType === DB_TYPE_EXTERNAL) }, + migrateSettings: enrichedComponent => { + const componentPrefix = "@budibase/standard-components" + let migrated = false + + if (enrichedComponent?._component == `${componentPrefix}/formblock`) { + // Use default config if the 'buttons' prop has never been initialised + if (!("buttons" in enrichedComponent)) { + enrichedComponent["buttons"] = + Utils.buildFormBlockButtonConfig(enrichedComponent) + migrated = true + } else if (enrichedComponent["buttons"] == null) { + // Ignore legacy config if 'buttons' has been reset by 'resetOn' + const { _id, actionType, dataSource } = enrichedComponent + enrichedComponent["buttons"] = Utils.buildFormBlockButtonConfig({ + _id, + actionType, + dataSource, + }) + migrated = true + } + + // Ensure existing Formblocks position their buttons at the top. + if (!("buttonPosition" in enrichedComponent)) { + enrichedComponent["buttonPosition"] = "top" + migrated = true + } + } + + return migrated + }, enrichEmptySettings: (component, opts) => { if (!component?._component) { return @@ -673,7 +704,6 @@ export const getFrontendStore = () => { component[setting.key] = setting.defaultValue } } - // Validate non-empty settings else { if (setting.type === "dataProvider") { @@ -732,6 +762,9 @@ export const getFrontendStore = () => { useDefaultValues: true, }) + // Migrate nested component settings + store.actions.components.migrateSettings(instance) + // Add any extra properties the component needs let extras = {} if (definition.hasChildren) { @@ -855,7 +888,16 @@ export const getFrontendStore = () => { if (!component) { return false } - return patchFn(component, screen) + + // Mutates the fetched component with updates + const patchResult = patchFn(component, screen) + + // Mutates the component with any required settings updates + const migrated = store.actions.components.migrateSettings(component) + + // Returning an explicit false signifies that we should skip this + // update. If we migrated something, ensure we never skip. + return migrated ? null : patchResult } await store.actions.screens.patch(patchScreen, screenId) }, @@ -1257,11 +1299,14 @@ export const getFrontendStore = () => { const settings = getComponentSettings(component._component) const updatedSetting = settings.find(setting => setting.key === name) - const resetFields = settings.filter( - setting => name === setting.resetOn - ) - resetFields?.forEach(setting => { - component[setting.key] = null + // Reset dependent fields + settings.forEach(setting => { + const needsReset = + name === setting.resetOn || + (Array.isArray(setting.resetOn) && setting.resetOn.includes(name)) + if (needsReset) { + component[setting.key] = setting.defaultValue || null + } }) if ( @@ -1281,6 +1326,7 @@ export const getFrontendStore = () => { }) } component[name] = value + return true } }, requestEjectBlock: componentId => { @@ -1288,7 +1334,6 @@ export const getFrontendStore = () => { }, handleEjectBlock: async (componentId, ejectedDefinition) => { let nextSelectedComponentId - await store.actions.screens.patch(screen => { const block = findComponent(screen.props, componentId) const parent = findComponentParent(screen.props, componentId) diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte index 3fd2708186..851c5b39c9 100644 --- a/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte +++ b/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte @@ -69,7 +69,15 @@ on:change={e => onChange(e, field)} useLabel={false} /> -{:else if schema.type === "string" || schema.type === "number"} +{:else if schema.type === "bb_reference"} + onChange(e, field)} + useLabel={false} + /> +{:else if ["string", "number", "bigint", "barcodeqr"].includes(schema.type)} import { ActionButton, notifications } from "@budibase/bbui" import CreateEditRelationshipModal from "../../Datasources/CreateEditRelationshipModal.svelte" - import { datasources } from "../../../../stores/backend" + import { + datasources, + tables as tablesStore, + } from "../../../../stores/backend" import { createEventDispatcher } from "svelte" export let table const dispatch = createEventDispatcher() $: datasource = findDatasource(table?._id) - $: tables = datasource?.plus ? Object.values(datasource?.entities || {}) : [] + $: tables = datasource?.plus + ? $tablesStore.list.filter(tbl => tbl.sourceId === datasource._id) + : [] let modal @@ -28,7 +33,12 @@ } const onError = err => { - notifications.error(`Error saving relationship info: ${err}`) + if (err.err) { + err = err.err + } + notifications.error( + `Error saving relationship info: ${err?.message || JSON.stringify(err)}` + ) } diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 028300be9f..936eb614d6 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -85,6 +85,7 @@ let relationshipTableIdSecondary = null let table = $tables.selected + let confirmDeleteDialog let savingColumn let deleteColName @@ -171,7 +172,7 @@ } } } - if (!savingColumn) { + if (!savingColumn && !originalName) { let highestNumber = 0 Object.keys(table.schema).forEach(columnName => { const columnNumber = extractColumnNumber(columnName) diff --git a/packages/builder/src/components/backend/Datasources/ConfigEditor/stores/validation.js b/packages/builder/src/components/backend/Datasources/ConfigEditor/stores/validation.js index 08331b840d..2ec9539824 100644 --- a/packages/builder/src/components/backend/Datasources/ConfigEditor/stores/validation.js +++ b/packages/builder/src/components/backend/Datasources/ConfigEditor/stores/validation.js @@ -1,4 +1,4 @@ -import { string, number } from "yup" +import { string, number, object } from "yup" const propertyValidator = type => { if (type === "number") { @@ -9,6 +9,10 @@ const propertyValidator = type => { return string().email().nullable() } + if (type === "object") { + return object().nullable() + } + return string().nullable() } diff --git a/packages/builder/src/components/common/CodeEditor/CodeEditor.svelte b/packages/builder/src/components/common/CodeEditor/CodeEditor.svelte index 76d7a58ef1..a39634f9a3 100644 --- a/packages/builder/src/components/common/CodeEditor/CodeEditor.svelte +++ b/packages/builder/src/components/common/CodeEditor/CodeEditor.svelte @@ -53,6 +53,7 @@ export let value = "" export let placeholder = null export let autocompleteEnabled = true + export let autofocus = false // Export a function to expose caret position export const getCaretPosition = () => { @@ -241,6 +242,12 @@ }) } + $: { + if (autofocus && isEditorInitialised) { + editor.focus() + } + } + $: editorHeight = typeof height === "number" ? `${height}px` : height // Init when all elements are ready diff --git a/packages/builder/src/components/common/LinkedRowSelector.svelte b/packages/builder/src/components/common/LinkedRowSelector.svelte index bf851eadb6..56156190be 100644 --- a/packages/builder/src/components/common/LinkedRowSelector.svelte +++ b/packages/builder/src/components/common/LinkedRowSelector.svelte @@ -8,6 +8,8 @@ export let schema export let linkedRows = [] export let useLabel = true + export let linkedTableId + export let label const dispatch = createEventDispatcher() let rows = [] @@ -16,8 +18,8 @@ $: linkedIds = (Array.isArray(linkedRows) ? linkedRows : [])?.map( row => row?._id || row ) - $: label = capitalise(schema.name) - $: linkedTableId = schema.tableId + $: label = label || capitalise(schema.name) + $: linkedTableId = linkedTableId || schema.tableId $: linkedTable = $tables.list.find(table => table._id === linkedTableId) $: fetchRows(linkedTableId) @@ -57,7 +59,7 @@ {:else} row._id} diff --git a/packages/builder/src/components/common/NavItem.svelte b/packages/builder/src/components/common/NavItem.svelte index 35846525af..02cef82e80 100644 --- a/packages/builder/src/components/common/NavItem.svelte +++ b/packages/builder/src/components/common/NavItem.svelte @@ -23,6 +23,7 @@ export let showTooltip = false export let selectedBy = null export let compact = false + export let hovering = false const scrollApi = getContext("scroll") const dispatch = createEventDispatcher() @@ -61,6 +62,7 @@
- + { + e.stopPropagation() + }} + on:change={onToggle(item)} + text="" + value={item.active} + thin + />
diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js index 72fdbe4108..e7b1727b54 100644 --- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js +++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js @@ -114,7 +114,7 @@ const getColumns = ({ primary, sortable, updateSortable: newDraggableList => { - onChange(toGridFormat(newDraggableList.concat(primary))) + onChange(toGridFormat(newDraggableList.concat(primary || []))) }, update: newEntry => { const newDraggableList = draggableList.map(entry => { diff --git a/packages/builder/src/components/design/settings/controls/OptionsEditor/OptionsEditor.svelte b/packages/builder/src/components/design/settings/controls/OptionsEditor/OptionsEditor.svelte index c626081042..14144d006f 100644 --- a/packages/builder/src/components/design/settings/controls/OptionsEditor/OptionsEditor.svelte +++ b/packages/builder/src/components/design/settings/controls/OptionsEditor/OptionsEditor.svelte @@ -25,7 +25,6 @@
-
Define Options
diff --git a/packages/builder/src/components/design/settings/controls/PropertyControl.svelte b/packages/builder/src/components/design/settings/controls/PropertyControl.svelte index a6f3d1b218..c20dd9310b 100644 --- a/packages/builder/src/components/design/settings/controls/PropertyControl.svelte +++ b/packages/builder/src/components/design/settings/controls/PropertyControl.svelte @@ -24,6 +24,7 @@ export let propertyFocus = false export let info = null export let disableBindings = false + export let wide $: nullishValue = value == null || value === "" $: allBindings = getAllBindings(bindings, componentBindings, nested) @@ -78,7 +79,7 @@
@@ -104,6 +105,7 @@ {...props} on:drawerHide on:drawerShow + on:meta />
{#if info} @@ -146,15 +148,28 @@ .control { position: relative; } - .property-control.wide .control { - grid-column: 1 / -1; - } .text { font-size: var(--spectrum-global-dimension-font-size-75); color: var(--grey-6); grid-column: 2 / 2; } + + .property-control.wide .control { + flex: 1; + } + .property-control.wide { + grid-template-columns: unset; + display: flex; + flex-direction: column; + width: 100%; + } + .property-control.wide > * { + width: 100%; + } .property-control.wide .text { grid-column: 1 / -1; } + .property-control.wide .label { + margin-bottom: -8px; + } diff --git a/packages/builder/src/components/design/settings/controls/RelationshipFilterEditor.svelte b/packages/builder/src/components/design/settings/controls/RelationshipFilterEditor.svelte index 0010a22d15..7e63bf38df 100644 --- a/packages/builder/src/components/design/settings/controls/RelationshipFilterEditor.svelte +++ b/packages/builder/src/components/design/settings/controls/RelationshipFilterEditor.svelte @@ -32,4 +32,4 @@ $: schema = linkedTable?.schema - + diff --git a/packages/builder/src/components/design/settings/controls/ValidationEditor/ValidationDrawer.svelte b/packages/builder/src/components/design/settings/controls/ValidationEditor/ValidationDrawer.svelte index 0bff2ccce6..25c7651d35 100644 --- a/packages/builder/src/components/design/settings/controls/ValidationEditor/ValidationDrawer.svelte +++ b/packages/builder/src/components/design/settings/controls/ValidationEditor/ValidationDrawer.svelte @@ -12,7 +12,10 @@ } from "@budibase/bbui" import { currentAsset, selectedComponent } from "builderStore" import { findClosestMatchingComponent } from "builderStore/componentUtils" - import { getSchemaForDatasource } from "builderStore/dataBinding" + import { + getSchemaForDatasource, + getDatasourceForProvider, + } from "builderStore/dataBinding" import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte" import { generate } from "shortid" @@ -124,6 +127,12 @@ ], } + const resolveDatasource = (currentAsset, componentInstance, parent) => { + return ( + getDatasourceForProvider(currentAsset, parent || componentInstance) || {} + ) + } + $: dataSourceSchema = getDataSourceSchema($currentAsset, $selectedComponent) $: field = fieldName || $selectedComponent?.field $: schemaRules = parseRulesFromSchema(field, dataSourceSchema || {}) @@ -146,8 +155,8 @@ component._component.endsWith("/formblock") || component._component.endsWith("/tableblock") ) - - return getSchemaForDatasource(asset, formParent?.dataSource) + const dataSource = resolveDatasource(asset, component, formParent) + return getSchemaForDatasource(asset, dataSource) } const parseRulesFromSchema = (field, dataSourceSchema) => { @@ -164,7 +173,8 @@ // Required constraint if ( field === dataSourceSchema?.table?.primaryDisplay || - constraints.presence?.allowEmpty === false + constraints.presence?.allowEmpty === false || + constraints.presence === true ) { rules.push({ constraint: "required", diff --git a/packages/builder/src/pages/builder/admin/index.svelte b/packages/builder/src/pages/builder/admin/index.svelte index 9723c6b621..a9c9748216 100644 --- a/packages/builder/src/pages/builder/admin/index.svelte +++ b/packages/builder/src/pages/builder/admin/index.svelte @@ -38,7 +38,7 @@ $goto("../portal") } catch (error) { submitted = false - notifications.error("Failed to create admin user") + notifications.error(error.message || "Failed to create admin user") } } diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Relationships.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Relationships.svelte index 1a46ecb540..19f2486298 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Relationships.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Relationships.svelte @@ -15,7 +15,8 @@ let modal - $: tables = Object.values(datasource.entities) + $: tables = + $tablesStore.list.filter(tbl => tbl.sourceId === datasource._id) || [] $: relationships = getRelationships(tables) function getRelationships(tables) { @@ -43,14 +44,16 @@ }) }) - return Object.values(relatedColumns).map(({ from, to, through }) => { - return { - tables: `${from.tableName} ${through ? "↔" : "→"} ${to.tableName}`, - columns: `${from.name} to ${to.name}`, - from, - to, - } - }) + return Object.values(relatedColumns) + .filter(({ from, to }) => from && to) + .map(({ from, to, through }) => { + return { + tables: `${from.tableName} ${through ? "↔" : "→"} ${to.tableName}`, + columns: `${from.name} to ${to.name}`, + from, + to, + } + }) } const handleRowClick = ({ detail }) => { diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte index 6093d2a45e..27f0f7bf7a 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte @@ -35,17 +35,16 @@ const customSections = settings.filter( setting => setting.section && setting.tag === tag ) - let sections = [ - ...(generalSettings?.length - ? [ - { - name: "General", - settings: generalSettings, - }, - ] - : []), - ...(customSections || []), - ] + let sections = [] + if (generalSettings.length) { + sections.push({ + name: "General", + settings: generalSettings, + }) + } + if (customSections.length) { + sections = sections.concat(customSections) + } // Filter out settings which shouldn't be rendered sections.forEach(section => { @@ -151,7 +150,8 @@ {#if section.visible} {#if section.info}