From 9818379ed56bb72ccdf520a941eb716aa94b8440 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 11 Dec 2023 15:19:50 +0100 Subject: [PATCH 01/15] Update ref --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 62c4ed3846..b51dc2b913 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 62c4ed384670406576a2d52ce8dafd683cc0a571 +Subproject commit b51dc2b91349d70e235f1b8e25b2ce5b15576fd1 From 5bef8fc607120eaa2573b65a0f513eb66540ab5b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 11 Dec 2023 15:25:57 +0100 Subject: [PATCH 02/15] Remove develop references --- .github/workflows/budibase_ci.yml | 2 +- docs/CONTRIBUTING.md | 4 ++-- packages/account-portal | 2 +- scripts/install-contributor-dependencies.sh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 3efbb3988a..4b241c5165 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -246,7 +246,7 @@ jobs: if (submoduleCommit !== baseCommit) { console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.'); - console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/develop/docs/getting_started.md') + console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/master/docs/getting_started.md') process.exit(1); } else { console.log('All good, the submodule had been merged and setup correctly!') diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 77afd9453b..21e4b52a4b 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -84,7 +84,7 @@ Component libraries are collections of components as well as the definition of t - If the project diverges from your branch, please rebase instead of merging. This makes the commit graph easier to read. -- Once your work is completed, please raise a PR against the `develop` branch with some information about what has changed and why. +- Once your work is completed, please raise a PR against the `master` branch with some information about what has changed and why. ### Getting Started For Contributors @@ -246,7 +246,7 @@ From here - to develop a change in pro, you can follow the below flow: cd packages/pro # get the base branch you are working from (same as monorepo) git fetch -git checkout +git checkout master # create a branch, named the same as the branch in your monorepo git checkout -b ... make changes diff --git a/packages/account-portal b/packages/account-portal index b51dc2b913..0fa909b152 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit b51dc2b91349d70e235f1b8e25b2ce5b15576fd1 +Subproject commit 0fa909b15222b22882d6667b92c422d1057c9472 diff --git a/scripts/install-contributor-dependencies.sh b/scripts/install-contributor-dependencies.sh index 923dedd804..56d24c84f2 100755 --- a/scripts/install-contributor-dependencies.sh +++ b/scripts/install-contributor-dependencies.sh @@ -39,7 +39,7 @@ getDistro if [[ $OS == "Darwin" ]]; then echo "This script is not setup for your machine type:" $OS - echo "Please use the manual steps described in https://github.com/Budibase/budibase/blob/develop/docs/CONTRIBUTING.md#getting-started-for-contributors" + echo "Please use the manual steps described in https://github.com/Budibase/budibase/blob/master/docs/CONTRIBUTING.md#getting-started-for-contributors" exit 1 fi From 02f836d0cfa86d8ed3fb39c886f922b0ed5b211c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 11 Dec 2023 16:04:28 +0100 Subject: [PATCH 03/15] Update docs --- .all-contributorsrc | 194 ------------------------------------ .github/workflows/README.md | 115 ++------------------- packages/account-portal | 2 +- 3 files changed, 12 insertions(+), 299 deletions(-) delete mode 100644 .all-contributorsrc diff --git a/.all-contributorsrc b/.all-contributorsrc deleted file mode 100644 index 3a416f917e..0000000000 --- a/.all-contributorsrc +++ /dev/null @@ -1,194 +0,0 @@ -{ - "files": [ - "README.md" - ], - "imageSize": 100, - "commit": false, - "contributors": [ - { - "login": "shogunpurple", - "name": "Martin McKeaveney", - "avatar_url": "https://avatars1.githubusercontent.com/u/11256663?v=4", - "profile": "http://martinmck.com", - "contributions": [ - "code", - "doc", - "test", - "infra" - ] - }, - { - "login": "mike12345567", - "name": "Michael Drury", - "avatar_url": "https://avatars2.githubusercontent.com/u/4407001?v=4", - "profile": "http://www.michaeldrury.co.uk/", - "contributions": [ - "doc", - "code", - "test", - "infra" - ] - }, - { - "login": "aptkingston", - "name": "Andrew Kingston", - "avatar_url": "https://avatars3.githubusercontent.com/u/9075550?v=4", - "profile": "https://github.com/aptkingston", - "contributions": [ - "doc", - "code", - "test", - "design" - ] - }, - { - "login": "mjashanks", - "name": "Michael Shanks", - "avatar_url": "https://avatars3.githubusercontent.com/u/3524181?v=4", - "profile": "https://budibase.com/", - "contributions": [ - "doc", - "code", - "test" - ] - }, - { - "login": "kevmodrome", - "name": "Kevin Åberg Kultalahti", - "avatar_url": "https://avatars3.githubusercontent.com/u/534488?v=4", - "profile": "https://github.com/kevmodrome", - "contributions": [ - "doc", - "code", - "test" - ] - }, - { - "login": "joebudi", - "name": "Joe", - "avatar_url": "https://avatars2.githubusercontent.com/u/49767913?v=4", - "profile": "https://www.budibase.com/", - "contributions": [ - "doc", - "code", - "content", - "design" - ] - }, - { - "login": "Rory-Powell", - "name": "Rory Powell", - "avatar_url": "https://avatars.githubusercontent.com/u/8755148?v=4", - "profile": "https://github.com/Rory-Powell", - "contributions": [ - "code", - "doc", - "test" - ] - }, - { - "login": "PClmnt", - "name": "Peter Clement", - "avatar_url": "https://avatars.githubusercontent.com/u/5665926?v=4", - "profile": "https://github.com/PClmnt", - "contributions": [ - "code", - "doc", - "test" - ] - }, - { - "login": "Conor-Mack", - "name": "Conor_Mack", - "avatar_url": "https://avatars1.githubusercontent.com/u/36074859?v=4", - "profile": "https://github.com/Conor-Mack", - "contributions": [ - "code", - "test" - ] - }, - { - "login": "pngwn", - "name": "pngwn", - "avatar_url": "https://avatars1.githubusercontent.com/u/12937446?v=4", - "profile": "https://github.com/pngwn", - "contributions": [ - "code", - "test" - ] - }, - { - "login": "HugoLd", - "name": "HugoLd", - "avatar_url": "https://avatars0.githubusercontent.com/u/26521848?v=4", - "profile": "https://github.com/HugoLd", - "contributions": [ - "code" - ] - }, - { - "login": "victoriasloan", - "name": "victoriasloan", - "avatar_url": "https://avatars.githubusercontent.com/u/9913651?v=4", - "profile": "https://github.com/victoriasloan", - "contributions": [ - "code" - ] - }, - { - "login": "yashank09", - "name": "yashank09", - "avatar_url": "https://avatars.githubusercontent.com/u/37672190?v=4", - "profile": "https://github.com/yashank09", - "contributions": [ - "code" - ] - }, - { - "login": "SOVLOOKUP", - "name": "SOVLOOKUP", - "avatar_url": "https://avatars.githubusercontent.com/u/53158137?v=4", - "profile": "https://github.com/SOVLOOKUP", - "contributions": [ - "code" - ] - }, - { - "login": "seoulaja", - "name": "seoulaja", - "avatar_url": "https://avatars.githubusercontent.com/u/15101654?v=4", - "profile": "https://github.com/seoulaja", - "contributions": [ - "translation" - ] - }, - { - "login": "mslourens", - "name": "Maurits Lourens", - "avatar_url": "https://avatars.githubusercontent.com/u/1907152?v=4", - "profile": "https://github.com/mslourens", - "contributions": [ - "test", - "code" - ] - }, - { - "login": "Rory-Powell", - "name": "Rory Powell", - "avatar_url": "https://avatars.githubusercontent.com/u/8755148?v=4", - "profile": "https://github.com/Rory-Powell", - "contributions": [ - "infra", - "test", - "code" - ] - } - ], - "contributorsPerLine": 7, - "projectName": "budibase", - "projectOwner": "Budibase", - "repoType": "github", - "repoHost": "https://github.com", - "skipCi": true, - "commitConvention": "none" -} diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 9b75a2e73a..ea12a0d779 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -1,136 +1,43 @@ # Budibase CI Pipelines -Welcome to the budibase CI pipelines directory. This document details what each of the CI pipelines are for, and come common combinations. +Welcome to the Budibase CI pipelines directory. This document details what each of the CI pipelines are for, and come common combinations. ## All CI Pipelines -### Note - -- When running workflow dispatch jobs, ensure you always run them off the `master` branch. It defaults to `develop`, so double check before running any jobs. The exception to this case is the `deploy-release` job which requires the develop branch. - ### Standard CI Build Job (budibase_ci.yml) Triggers: -- PR or push to develop - PR or push to master -The standard CI Build job is what runs when you raise a PR to develop or master. +The standard CI Build job is what runs when you raise a PR to master. - Installs all dependencies, - builds the project - run the unit tests - Generate test coverage metrics with codecov - Run the integration tests +- Check that the pro and account portal submodules are pointing to the lastest master head -### Release Develop Job (release-develop.yml) +### Release Job (tag-release.yml) Triggers: -- Push to develop +- Manually triggered -The job responsible for building, tagging and pushing docker images out to the test and release environments. +This job is responsible for building and pushing all the production services, packages and images. This is done via [budibase-deploys](https://github.com/Budibase/budibase-deploys/actions/workflows/release.yml). -- Installs all dependencies -- builds the project -- run the unit tests -- publish the budibase JS packages under a prerelease tag to NPM -- build, tag and push docker images under the `develop` tag to docker hub +An input is required, indicating if the new version will be a `patch`, `minor` or `major` bump. -These images will then be pulled by the test and release environments, updating the latest automatically. Discord notifications are sent to the #infra channel when this occurs. - -### Release Job (release.yml) - -Triggers: - -- Push to master - -This job is responsible for building and pushing the latest code to NPM and docker hub, so that it can be deployed. - -- Installs all dependencies -- builds the project -- run the unit tests -- publish the budibase JS packages under a release tag to NPM (always incremented by patch versions) -- build, tag and push docker images under the `v.x.x.x` (the tag of the NPM release) tag to docker hub - -### Release Selfhost Job (release-selfhost.yml) - -Triggers: - -- Manual Workflow Dispatch Trigger - -This job is responsible for delivering the latest version of budibase to those that are self-hosting. - -This job relies on the release job to have run first, so the latest image is pushed to dockerhub. This job then will pull the latest version from `lerna.json` and try to find an image in dockerhub corresponding to that version. For example, if the version in `lerna.json` is `1.0.0`: - -- Pull the images for all budibase services tagged `v1.0.0` from dockerhub -- Tag these images as `latest` -- Push them back to dockerhub. This now means anyone who pulls `latest` (self hosters using docker-compose) will get the latest version. -- Build and release the budibase helm chart for kubernetes users -- Perform a github release with the latest version. You can see previous releases here (https://github.com/Budibase/budibase/releases) - -### Deploy Release (deploy-release.yml) - -Triggers: - -- Manual Workflow Dispatch Trigger - -This job is responsible for deploying to our release, cloud kubernetes environment. You must run the release job first, to ensure that the latest images have been built and pushed to docker hub. After kicking off this job, the following will occur: - -- Checks out the release branch -- Pulls the latest `values.yaml` from budibase infra, a private repo containing budibases infrastructure configuration -- Gets the latest budibase version from `lerna.json`, if it hasn't been specified in the workflow when you kicked it off -- Configures AWS Credentials -- Deploys the helm chart in the budibase repo to our preproduction EKS cluster, injecting the `values.yaml` we pulled from budibase-infra -- Fires off a discord webhook in the #infra channel to show that the deployment completely successfully. - -### Deploy Preprod (deploy-preprod.yml) - -Triggers: - -- Manual Workflow Dispatch Trigger - -This job is responsible for deploying to our preprod, cloud kubernetes environment. You must run the release job first, to ensure that the latest images have been built and pushed to docker hub. After kicking off this job, the following will occur: - -- Checks out the master branch -- Pulls the latest `values.yaml` from budibase infra, a private repo containing budibases infrastructure configuration -- Gets the latest budibase version from `lerna.json`, if it hasn't been specified in the workflow when you kicked it off -- Configures AWS Credentials -- Deploys the helm chart in the budibase repo to our preprod EKS cluster, injecting the `values.yaml` we pulled from budibase-infra -- Fires off a discord webhook in the #infra channel to show that the deployment completely successfully. - -### Deploy Production (deploy-cloud.yml) - -Triggers: - -- Manual Workflow Dispatch Trigger - -This job is responsible for deploying to our production, cloud kubernetes environment. You must run the release job first, to ensure that the latest images have been built and pushed to docker hub. You can also manually enter a version number for this job, so you can perform rollbacks or upgrade to a specific version. After kicking off this job, the following will occur: - -- Checks out the master branch -- Pulls the latest `values.yaml` from budibase infra, a private repo containing budibases infrastructure configuration -- Gets the latest budibase version from `lerna.json`, if it hasn't been specified in the workflow when you kicked it off -- Configures AWS Credentials -- Deploys the helm chart in the budibase repo to our production EKS cluster, injecting the `values.yaml` we pulled from budibase-infra -- Fires off a discord webhook in the #infra channel to show that the deployment completely successfully. +More documentation can be found in here: https://budibase.atlassian.net/wiki/spaces/DEVOPS/pages/347930625/Production+release ## Common Workflows ### Deploy Changes to Production (Release) -- Merge `develop` into `master` -- Wait for budibase CI job and release job to run -- Run cloud deploy job -- Run release selfhost job - -### Deploy Changes to Production (Hotfix) - -- Branch off `master` -- Perform your hotfix -- Merge back into `master` -- Wait for budibase CI job and release job to run -- Run cloud deploy job -- Run release selfhost job +- Merge your changes into `master` +- Run `tag-release.yml` +- Check the progress in [budibase-deploys](https://github.com/Budibase/budibase-deploys/actions/workflows/release.yml) ### Rollback A Bad Cloud Deployment diff --git a/packages/account-portal b/packages/account-portal index 0fa909b152..78ba8697c4 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 0fa909b15222b22882d6667b92c422d1057c9472 +Subproject commit 78ba8697c4156ae10cfba9dc563ddd486ab2f6fe From eb1eb738b8c7483d99cb1d2842747e9add4c8aa6 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 11 Dec 2023 16:40:11 +0100 Subject: [PATCH 04/15] Document rollback --- .github/workflows/README.md | 5 ++-- .github/workflows/force-release.yml | 46 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/force-release.yml diff --git a/.github/workflows/README.md b/.github/workflows/README.md index ea12a0d779..029dd5af42 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -41,6 +41,5 @@ More documentation can be found in here: https://budibase.atlassian.net/wiki/spa ### Rollback A Bad Cloud Deployment -- Kick off cloud deploy job -- Ensure you are running off master -- Enter the version number of the last known good version of budibase. For example `1.0.0` +Rollback documentation can be found in here. +https://budibase.atlassian.net/wiki/spaces/DEVOPS/pages/347930625/Production+release#Rollback diff --git a/.github/workflows/force-release.yml b/.github/workflows/force-release.yml new file mode 100644 index 0000000000..8a9d444f51 --- /dev/null +++ b/.github/workflows/force-release.yml @@ -0,0 +1,46 @@ +name: Forced release +concurrency: + group: tag-release + cancel-in-progress: false + +on: + workflow_dispatch: + +jobs: + ensure-is-master-tag: + name: Ensure is a master tag + runs-on: qa-arc-runner-set + steps: + - name: Checkout monorepo + uses: actions/checkout@v4 + with: + token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} + fetch-tags: true + fetch-depth: 0 + + - name: Fail if ref is not a tag + run: | + if ! git show-ref -q --verify "refs/tags/${{ github.ref_name }}" 2>/dev/null; then + echo "'${{ github.ref_name }}' is not a valid tag." + exit 1 + fi + - name: Fail if tag is not in master + run: | + if ! git merge-base --is-ancestor ${{ github.ref_name }} origin/master; then + echo "Tag is not in master. Release can only execute tags that are present on the master branch" + exit 1 + fi + + trigger-release: + needs: [ensure-is-master-tag] + runs-on: ubuntu-latest + steps: + - uses: peter-evans/repository-dispatch@v2 + with: + repository: budibase/budibase-deploys + event-type: release-prod + token: ${{ secrets.GH_ACCESS_TOKEN }} + client-payload: |- + { + "TAG": "${{ github.ref_name }}" + } From 59be83a2caa59a5c20e93cb677788f1c6143f519 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 11 Dec 2023 16:42:32 +0100 Subject: [PATCH 05/15] Update ref --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 78ba8697c4..e2a0fc337d 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 78ba8697c4156ae10cfba9dc563ddd486ab2f6fe +Subproject commit e2a0fc337dba5589033ffdbcb9d401ae09ad8716 From 686cf938f96e1f0cf475a60a102be7123b88f3e9 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 11 Dec 2023 17:00:10 +0100 Subject: [PATCH 06/15] Update ref --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index e2a0fc337d..1532b228de 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit e2a0fc337dba5589033ffdbcb9d401ae09ad8716 +Subproject commit 1532b228dea0bb248abdb26289e4468c5502f7af From 90c4485a90ae3fee21c7e54ca37a92be436d80d7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 11 Dec 2023 17:22:37 +0100 Subject: [PATCH 07/15] Update account-portal submodule ref --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 1532b228de..4b2ed4647f 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 1532b228dea0bb248abdb26289e4468c5502f7af +Subproject commit 4b2ed4647f09fb00f3bb32dbc5db9a22631146e6 From 6617243ce529f09fb8dcceec38012375922e6cd9 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 11 Dec 2023 17:23:02 +0000 Subject: [PATCH 08/15] Update global users search to account for numeric prefixing --- .../src/api/controllers/global/users.ts | 22 ++++++++++++++++--- .../src/api/routes/global/tests/users.spec.ts | 18 +++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 58979ec799..1c470bdba9 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -14,6 +14,7 @@ import { InviteUsersResponse, MigrationType, SaveUserResponse, + SearchQueryOperators, SearchUsersRequest, User, UserCtx, @@ -29,6 +30,7 @@ import { } from "@budibase/backend-core" import { checkAnyUserExists } from "../../../utilities/users" import { isEmailConfigured } from "../../../utilities/email" +import { removeKeyNumbering } from "@budibase/backend-core/src/db" const MAX_USERS_UPLOAD_LIMIT = 1000 @@ -185,9 +187,23 @@ export const getAppUsers = async (ctx: Ctx) => { export const search = async (ctx: Ctx) => { const body = ctx.request.body - // TODO: for now only one supported search key, string.email - if (body?.query && !userSdk.core.isSupportedUserSearch(body.query)) { - ctx.throw(501, "Can only search by string.email or equal._id") + // TODO: for now only two supported search keys; string.email and equal._id + if (body?.query) { + // Clean numeric prefixing. This will overwrite duplicate search fields, + // but this is fine because we only support a single custom search on + // email and id + for (let filters of Object.values(body.query)) { + if (filters && typeof filters === "object") { + for (let [field, value] of Object.entries(filters)) { + delete filters[field] + filters[removeKeyNumbering(field)] = value + } + } + } + // Validate we aren't trying to search on any illegal fields + if (!userSdk.core.isSupportedUserSearch(body.query)) { + ctx.throw(501, "Can only search by string.email or equal._id") + } } if (body.paginate === false) { diff --git a/packages/worker/src/api/routes/global/tests/users.spec.ts b/packages/worker/src/api/routes/global/tests/users.spec.ts index a85933255a..cb534a770a 100644 --- a/packages/worker/src/api/routes/global/tests/users.spec.ts +++ b/packages/worker/src/api/routes/global/tests/users.spec.ts @@ -590,6 +590,15 @@ describe("/api/global/users", () => { expect(response.body.data[0].email).toBe(user.email) }) + it("should be able to search by email with numeric prefixing", async () => { + const user = await config.createUser() + const response = await config.api.users.searchUsers({ + query: { string: { ["999:email"]: user.email } }, + }) + expect(response.body.data.length).toBe(1) + expect(response.body.data[0].email).toBe(user.email) + }) + it("should be able to search by _id", async () => { const user = await config.createUser() const response = await config.api.users.searchUsers({ @@ -599,6 +608,15 @@ describe("/api/global/users", () => { expect(response.body.data[0]._id).toBe(user._id) }) + it("should be able to search by _id with numeric prefixing", async () => { + const user = await config.createUser() + const response = await config.api.users.searchUsers({ + query: { equal: { ["1:_id"]: user._id } }, + }) + expect(response.body.data.length).toBe(1) + expect(response.body.data[0]._id).toBe(user._id) + }) + it("should throw an error when unimplemented options used", async () => { const user = await config.createUser() await config.api.users.searchUsers( From e16cc267815314c1780e506a706cad6cc8dafab5 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Mon, 11 Dec 2023 17:25:42 +0000 Subject: [PATCH 09/15] Lint --- packages/worker/src/api/controllers/global/users.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 1c470bdba9..2f9c78bf93 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -14,7 +14,6 @@ import { InviteUsersResponse, MigrationType, SaveUserResponse, - SearchQueryOperators, SearchUsersRequest, User, UserCtx, From c50e8fd05bda29b537158936a418dbfb2fc59f39 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 11 Dec 2023 17:43:56 +0000 Subject: [PATCH 10/15] Split withEnv, so the core env isn't touched in it. --- .../integrations/tests/googlesheets.spec.ts | 2 +- .../src/tests/utilities/TestConfiguration.ts | 31 +++++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 10ec7815d6..7007693086 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -39,7 +39,7 @@ describe("Google Sheets Integration", () => { let cleanupEnv: () => void beforeAll(() => { - cleanupEnv = config.setEnv({ + cleanupEnv = config.setCoreEnv({ GOOGLE_CLIENT_ID: "test", GOOGLE_CLIENT_SECRET: "test", }) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index afaad64723..51b5fda3d4 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -218,20 +218,45 @@ class TestConfiguration { */ setEnv(newEnvVars: Partial): () => void { const oldEnv = cloneDeep(env) - const oldCoreEnv = cloneDeep(coreEnv) let key: keyof typeof newEnvVars for (key in newEnvVars) { env._set(key, newEnvVars[key]) - coreEnv._set(key, newEnvVars[key]) } return () => { for (const [key, value] of Object.entries(oldEnv)) { env._set(key, value) } + } + } - for (const [key, value] of Object.entries(oldCoreEnv)) { + async withCoreEnv( + newEnvVars: Partial, + f: () => Promise + ) { + let cleanup = this.setCoreEnv(newEnvVars) + try { + await f() + } finally { + cleanup() + } + } + + /* + * Sets the environment variables to the given values and returns a function + * that can be called to reset the environment variables to their original values. + */ + setCoreEnv(newEnvVars: Partial): () => void { + const oldEnv = cloneDeep(env) + + let key: keyof typeof newEnvVars + for (key in newEnvVars) { + coreEnv._set(key, newEnvVars[key]) + } + + return () => { + for (const [key, value] of Object.entries(oldEnv)) { coreEnv._set(key, value) } } From 66f219d7787118cfeb17ccc730a3100606c91e79 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 12 Dec 2023 10:26:48 +0000 Subject: [PATCH 11/15] Lint and change status code to 400 when searching on invalid fields --- packages/worker/src/api/controllers/global/users.ts | 6 +++--- packages/worker/src/api/routes/global/tests/users.spec.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 2f9c78bf93..0520990f2f 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -26,10 +26,10 @@ import { migrations, platform, tenancy, + db, } from "@budibase/backend-core" import { checkAnyUserExists } from "../../../utilities/users" import { isEmailConfigured } from "../../../utilities/email" -import { removeKeyNumbering } from "@budibase/backend-core/src/db" const MAX_USERS_UPLOAD_LIMIT = 1000 @@ -195,13 +195,13 @@ export const search = async (ctx: Ctx) => { if (filters && typeof filters === "object") { for (let [field, value] of Object.entries(filters)) { delete filters[field] - filters[removeKeyNumbering(field)] = value + filters[db.removeKeyNumbering(field)] = value } } } // Validate we aren't trying to search on any illegal fields if (!userSdk.core.isSupportedUserSearch(body.query)) { - ctx.throw(501, "Can only search by string.email or equal._id") + ctx.throw(400, "Can only search by string.email or equal._id") } } diff --git a/packages/worker/src/api/routes/global/tests/users.spec.ts b/packages/worker/src/api/routes/global/tests/users.spec.ts index cb534a770a..1365173b21 100644 --- a/packages/worker/src/api/routes/global/tests/users.spec.ts +++ b/packages/worker/src/api/routes/global/tests/users.spec.ts @@ -623,7 +623,7 @@ describe("/api/global/users", () => { { query: { equal: { firstName: user.firstName } }, }, - { status: 501 } + { status: 400 } ) }) From 9d56039d0365e4e11cbbcaaec569548217fbd91b Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Tue, 12 Dec 2023 10:52:14 +0000 Subject: [PATCH 12/15] Revert TTL on signed urls (#12556) --- packages/backend-core/src/objectStore/objectStore.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 9b44eace49..57ead0e809 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -260,12 +260,12 @@ export async function listAllObjects(bucketName: string, path: string) { } /** - * Generate a presigned url with a default TTL of 36 hours + * Generate a presigned url with a default TTL of 1 hour */ export function getPresignedUrl( bucketName: string, key: string, - durationSeconds: number = 129600 + durationSeconds: number = 3600 ) { const objectStore = ObjectStore(bucketName, { presigning: true }) const params = { From 5511a21445003403c04b51d560cbe6db9e25d66c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 12 Dec 2023 11:59:19 +0100 Subject: [PATCH 13/15] Update account-portal submodule ref --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 4b2ed4647f..a0b13270c3 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 4b2ed4647f09fb00f3bb32dbc5db9a22631146e6 +Subproject commit a0b13270c36dd188e2a953d026b4560a1208008e From f7b7f3efdee911b7b162a8a12236c0e7b2288ace Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Tue, 12 Dec 2023 11:15:29 +0000 Subject: [PATCH 14/15] Error when searching global users using more than one filter per field --- .../src/api/controllers/global/users.ts | 6 +++- .../src/api/routes/global/tests/users.spec.ts | 30 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 0520990f2f..b0e3219656 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -195,7 +195,11 @@ export const search = async (ctx: Ctx) => { if (filters && typeof filters === "object") { for (let [field, value] of Object.entries(filters)) { delete filters[field] - filters[db.removeKeyNumbering(field)] = value + const cleanedField = db.removeKeyNumbering(field) + if (filters[cleanedField] !== undefined) { + ctx.throw(400, "Only 1 filter per field is supported") + } + filters[cleanedField] = value } } } diff --git a/packages/worker/src/api/routes/global/tests/users.spec.ts b/packages/worker/src/api/routes/global/tests/users.spec.ts index 1365173b21..c792de70a9 100644 --- a/packages/worker/src/api/routes/global/tests/users.spec.ts +++ b/packages/worker/src/api/routes/global/tests/users.spec.ts @@ -617,6 +617,36 @@ describe("/api/global/users", () => { expect(response.body.data[0]._id).toBe(user._id) }) + it("should throw an error when using multiple filters on the same field", async () => { + const user = await config.createUser() + await config.api.users.searchUsers( + { + query: { + string: { + ["1:email"]: user.email, + ["2:email"]: "something else", + }, + }, + }, + { status: 400 } + ) + }) + + it("should throw an error when using multiple filters on the same field without prefixes", async () => { + const user = await config.createUser() + await config.api.users.searchUsers( + { + query: { + string: { + ["_id"]: user.email, + ["999:_id"]: "something else", + }, + }, + }, + { status: 400 } + ) + }) + it("should throw an error when unimplemented options used", async () => { const user = await config.createUser() await config.api.users.searchUsers( From 0dca394a976cadaed76ee53f5814e0cbcadb6a20 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 12 Dec 2023 12:25:14 +0000 Subject: [PATCH 15/15] Bump version to 2.13.37 --- lerna.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lerna.json b/lerna.json index e0dc1a4f11..05183ad6d4 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.36", + "version": "2.13.37", "npmClient": "yarn", "packages": [ "packages/*", @@ -21,4 +21,4 @@ "loadEnvFiles": false } } -} +} \ No newline at end of file