From d7a8b5f2b776dd7cd6bd75c535fa3e4fd310d46d Mon Sep 17 00:00:00 2001 From: Dean Date: Tue, 26 Sep 2023 10:58:56 +0100 Subject: [PATCH 01/31] Updated the nav item body text to ensure it was truncated. The selected by Avatar was being pushed out of view. Hover text added --- packages/builder/src/components/common/NavItem.svelte | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/components/common/NavItem.svelte b/packages/builder/src/components/common/NavItem.svelte index a65da55c8f..2c8a862535 100644 --- a/packages/builder/src/components/common/NavItem.svelte +++ b/packages/builder/src/components/common/NavItem.svelte @@ -102,7 +102,7 @@ {/if}
- {text} + {text} {#if selectedBy} {/if} @@ -227,9 +227,6 @@ .text { font-weight: 600; font-size: 12px; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; flex: 1 1 auto; color: var(--spectrum-global-color-gray-900); order: 2; @@ -238,6 +235,11 @@ align-items: center; gap: 8px; } + .text span { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } .scrollable .text { flex: 0 0 auto; max-width: 160px; From f4e75f4a2be2f0f61faccc1763331a63f175b20b Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 11 Oct 2023 18:05:23 +0000 Subject: [PATCH 02/31] Bump version to 2.11.27 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 5708bb5f44..64d169472b 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.26", + "version": "2.11.27", "npmClient": "yarn", "packages": [ "packages/*" From d6f8662e952c9001c1712a07c484af820df49e4f Mon Sep 17 00:00:00 2001 From: Conor Webb <126772285+ConorWebb96@users.noreply.github.com> Date: Thu, 12 Oct 2023 08:29:40 +0100 Subject: [PATCH 03/31] Adding onchange actions to code scanner. (#12029) * Adding onchange actions to code scanner. * Removed unused code, set updateOnChange to false. --------- Co-authored-by: Michael Drury --- packages/client/manifest.json | 11 +++++++++++ .../src/components/app/forms/CodeScanner.svelte | 1 + 2 files changed, 12 insertions(+) diff --git a/packages/client/manifest.json b/packages/client/manifest.json index d987344956..47b935b420 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -3419,6 +3419,17 @@ "value": "custom" } }, + { + "type": "event", + "label": "On change", + "key": "onChange", + "context": [ + { + "label": "Field Value", + "key": "value" + } + ] + }, { "type": "validation/string", "label": "Validation", diff --git a/packages/client/src/components/app/forms/CodeScanner.svelte b/packages/client/src/components/app/forms/CodeScanner.svelte index 9895413446..04d6919157 100644 --- a/packages/client/src/components/app/forms/CodeScanner.svelte +++ b/packages/client/src/components/app/forms/CodeScanner.svelte @@ -128,6 +128,7 @@
{ dispatch("change", value) }} From ad0529cf64679f40373b1b498796a6fd9d6f4c1c Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 12 Oct 2023 07:30:01 +0000 Subject: [PATCH 04/31] Bump version to 2.11.28 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 64d169472b..e6fd167f16 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.27", + "version": "2.11.28", "npmClient": "yarn", "packages": [ "packages/*" From 025e660674e9b8b65f90b31fe23c52d0826d59e5 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 12 Oct 2023 07:39:03 +0000 Subject: [PATCH 05/31] Bump version to 2.11.29 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index e6fd167f16..c4315ee698 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.28", + "version": "2.11.29", "npmClient": "yarn", "packages": [ "packages/*" From eea31893cfba592550f573dcb44847fb101b6fc3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 12 Oct 2023 10:03:47 +0100 Subject: [PATCH 06/31] Trying to fix max-space size action as part of the single image release (required to create the space needed for both image builds. --- .github/workflows/release-singleimage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml index bd01ed786a..78b5cdd577 100644 --- a/.github/workflows/release-singleimage.yml +++ b/.github/workflows/release-singleimage.yml @@ -18,7 +18,7 @@ jobs: - name: Maximize build space uses: easimon/maximize-build-space@master with: - root-reserve-mb: 35000 + root-reserve-mb: 30000 swap-size-mb: 1024 remove-android: 'true' remove-dotnet: 'true' From f83df42bfc2781dd498e9e25a3e845e770d28a6c Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 12 Oct 2023 09:13:59 +0000 Subject: [PATCH 07/31] Bump version to 2.11.30 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index c4315ee698..95e10c444e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.29", + "version": "2.11.30", "npmClient": "yarn", "packages": [ "packages/*" From 78173c9d1e3436fbdc6b2274635044ab88032a5a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 12 Oct 2023 13:43:15 +0100 Subject: [PATCH 08/31] Some minor updates, attempting to maximise space to get build running again, having issues with network timeouts/buildx problems, attempting to restore full build. --- .github/workflows/release-singleimage.yml | 12 ------------ hosting/single/Dockerfile | 4 ++-- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml index 78b5cdd577..61ab9a4eb2 100644 --- a/.github/workflows/release-singleimage.yml +++ b/.github/workflows/release-singleimage.yml @@ -33,14 +33,6 @@ jobs: with: submodules: true token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - fetch-depth: 0 - - - name: Fail if tag is not in master - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then - echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" - exit 1 - fi - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 @@ -55,10 +47,6 @@ jobs: run: yarn - name: Update versions run: ./scripts/updateVersions.sh - - name: Runt Yarn Lint - run: yarn lint - - name: Update versions - run: ./scripts/updateVersions.sh - name: Run Yarn Build run: yarn build:docker:pre - name: Login to Docker Hub diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 9fdf2449d1..95e383edb0 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -12,14 +12,14 @@ RUN chmod +x /cleanup.sh WORKDIR /app ADD packages/server . COPY yarn.lock . -RUN yarn install --production=true +RUN yarn install --production=true --network-timeout 100000 RUN /cleanup.sh # build worker WORKDIR /worker ADD packages/worker . COPY yarn.lock . -RUN yarn install --production=true +RUN yarn install --production=true --network-timeout 100000 RUN /cleanup.sh FROM budibase/couchdb From 702ee4d504195aa716d6eac5b3333a2551ec13c8 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 12 Oct 2023 12:48:57 +0000 Subject: [PATCH 09/31] Bump version to 2.11.31 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 95e10c444e..51c702a2f6 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.30", + "version": "2.11.31", "npmClient": "yarn", "packages": [ "packages/*" From 61c675c3bf6d33dedd912f6940f0b2f691db3f8c Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 12 Oct 2023 14:32:16 +0100 Subject: [PATCH 10/31] removing all references to develop in budibase main repo CI --- .github/workflows/deploy-featurebranch.yml | 1 - .github/workflows/deploy-preprod.yml | 41 ------- .github/workflows/release-develop.yml | 124 --------------------- .github/workflows/release-master.yml | 12 +- .github/workflows/tag-prerelease.yml | 42 ------- 5 files changed, 3 insertions(+), 217 deletions(-) delete mode 100644 .github/workflows/deploy-preprod.yml delete mode 100644 .github/workflows/release-develop.yml delete mode 100644 .github/workflows/tag-prerelease.yml diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index 2c6302b56a..f1fb12c087 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -3,7 +3,6 @@ name: deploy-featurebranch on: pull_request: branches: - - develop - master jobs: diff --git a/.github/workflows/deploy-preprod.yml b/.github/workflows/deploy-preprod.yml deleted file mode 100644 index 9b7bca4770..0000000000 --- a/.github/workflows/deploy-preprod.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: "deploy-preprod" -on: - workflow_dispatch: - workflow_call: - -jobs: - deploy-to-legacy-preprod-env: - runs-on: ubuntu-latest - steps: - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Fail if tag is not in master - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then - echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" - exit 1 - fi - - - name: Get the latest budibase release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - - uses: passeidireto/trigger-external-workflow-action@main - env: - PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }} - with: - repository: budibase/budibase-deploys - event: budicloud-preprod-deploy - github_pat: ${{ secrets.GH_ACCESS_TOKEN }} - diff --git a/.github/workflows/release-develop.yml b/.github/workflows/release-develop.yml deleted file mode 100644 index bd727b7865..0000000000 --- a/.github/workflows/release-develop.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: Budibase Prerelease -concurrency: - group: release-prerelease - cancel-in-progress: false - -on: - push: - tags: - - "*-alpha.*" - workflow_dispatch: - -env: - # Posthog token used by ui at build time - # disable unless needed for testing - # POSTHOG_TOKEN: phc_uDYOfnFt6wAbBAXkC6STjcrTpAFiWIhqgFcsC1UVO5F - INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }} - PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - FEATURE_PREVIEW_URL: https://budirelease.live - -jobs: - release-images: - runs-on: ubuntu-latest - - steps: - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - fetch-depth: 0 - - - name: Fail if tag is not develop - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/develop; then - echo "Tag is not in develop" - exit 1 - fi - - - uses: actions/setup-node@v1 - with: - node-version: 18.x - - - run: yarn install --frozen-lockfile - - name: Update versions - run: ./scripts/updateVersions.sh - - run: yarn build - - run: yarn build:sdk - - - name: Publish budibase packages to NPM - env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - # setup the username and email. - git config --global user.name "Budibase Staging Release Bot" - git config --global user.email "<>" - git submodule foreach git commit -a -m 'Release process' - git commit -a -m 'Release process' - echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc - yarn release:develop - - - name: Build/release Docker images - run: | - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - yarn build:docker:develop - env: - DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - - release-helm-chart: - needs: [release-images] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Setup Helm - uses: azure/setup-helm@v1 - id: helm-install - - # due to helm repo index issue: https://github.com/helm/helm/issues/7363 - # we need to create new package in a different dir, merge the index and move the package back - - name: Build and release helm chart - run: | - git config user.name "Budibase Helm Bot" - git config user.email "<>" - git reset --hard - git fetch - mkdir sync - echo "Packaging chart to sync dir" - helm package charts/budibase --version 0.0.0-develop --app-version develop --destination sync - echo "Packaging successful" - git checkout gh-pages - echo "Indexing helm repo" - helm repo index --merge docs/index.yaml sync - mv -f sync/* docs - rm -rf sync - echo "Pushing new helm release" - git add -A - git commit -m "Helm Release: develop" - git push - - trigger-deploy-to-qa-env: - needs: [release-helm-chart] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Get the current budibase release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - - uses: passeidireto/trigger-external-workflow-action@main - env: - PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }} - with: - repository: budibase/budibase-deploys - event: budicloud-qa-deploy - github_pat: ${{ secrets.GH_ACCESS_TOKEN }} diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index b4991cbfbe..3af3a751ad 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -110,19 +110,13 @@ jobs: git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}" git push - deploy-to-legacy-preprod-env: - needs: [release-images] - uses: ./.github/workflows/deploy-preprod.yml - secrets: inherit - # Trigger deploy to new EKS preprod environment - trigger-deploy-to-preprod-env: + trigger-deploy-to-qa-env: needs: [release-helm-chart] runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - - name: Get the latest budibase release version + - name: Get the current budibase release version id: version run: | release_version=$(cat lerna.json | jq -r '.version') @@ -133,5 +127,5 @@ jobs: PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }} with: repository: budibase/budibase-deploys - event: budicloud-preprod-deploy + event: budicloud-qa-deploy github_pat: ${{ secrets.GH_ACCESS_TOKEN }} diff --git a/.github/workflows/tag-prerelease.yml b/.github/workflows/tag-prerelease.yml deleted file mode 100644 index f6446c55f5..0000000000 --- a/.github/workflows/tag-prerelease.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Tag prerelease -concurrency: - group: tag-prerelease - cancel-in-progress: false - -on: - push: - branches: - - develop - paths: - - ".aws/**" - - ".github/**" - - "charts/**" - - "packages/**" - - "scripts/**" - - "package.json" - - "yarn.lock" - workflow_dispatch: - -jobs: - tag-prerelease: - runs-on: ubuntu-latest - - steps: - - name: Fail if branch is not develop - if: github.ref != 'refs/heads/develop' - run: | - echo "Ref is not develop, you must run this job from develop." - exit 1 - - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - - - run: cd scripts && yarn - - name: Tag prerelease - run: | - cd scripts - # setup the username and email. - git config --global user.name "Budibase Staging Release Bot" - git config --global user.email "<>" - ./versionCommit.sh prerelease From 6bbce23910284a96b9fa8ea4addc6cd4e05f35c1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 12 Oct 2023 16:31:32 +0100 Subject: [PATCH 11/31] Updating user fetch functionality to send up lucene syntax for searching to global user endpoint. --- packages/frontend-core/src/api/user.js | 25 ++++++----- packages/frontend-core/src/fetch/UserFetch.js | 26 +++++++---- .../server/src/api/routes/tests/row.spec.ts | 4 +- .../src/api/routes/tests/viewV2.spec.ts | 13 +++--- packages/shared-core/src/filters.ts | 45 ++++++++++++------- packages/types/src/api/web/searchFilter.ts | 36 ++++++++++----- yarn.lock | 2 +- 7 files changed, 97 insertions(+), 54 deletions(-) diff --git a/packages/frontend-core/src/api/user.js b/packages/frontend-core/src/api/user.js index 6c616d7baf..92b3671c78 100644 --- a/packages/frontend-core/src/api/user.js +++ b/packages/frontend-core/src/api/user.js @@ -10,25 +10,30 @@ export const buildUserEndpoints = API => ({ /** * Gets a list of users in the current tenant. - * @param {string} page The page to retrieve - * @param {string} search The starts with string to search username/email by. + * @param {string} bookmark The page to retrieve + * @param {object} query search filters for lookup by user (all operators not supported). * @param {string} appId Facilitate app/role based user searching - * @param {boolean} paginated Allow the disabling of pagination + * @param {boolean} paginate Allow the disabling of pagination + * @param {number} limit How many users to retrieve in a single search */ - searchUsers: async ({ paginated, page, email, appId } = {}) => { + searchUsers: async ({ paginate, bookmark, query, appId, limit } = {}) => { const opts = {} - if (page) { - opts.page = page + if (bookmark) { + opts.bookmark = bookmark } - if (email) { - opts.email = email + if (query) { + opts.query = query } if (appId) { opts.appId = appId } - if (typeof paginated === "boolean") { - opts.paginated = paginated + if (typeof paginate === "boolean") { + opts.paginate = paginate } + if (limit) { + opts.limit = limit + } + console.log(opts) return await API.post({ url: `/api/global/users/search`, body: opts, diff --git a/packages/frontend-core/src/fetch/UserFetch.js b/packages/frontend-core/src/fetch/UserFetch.js index 5372d0ec33..2158b3ca57 100644 --- a/packages/frontend-core/src/fetch/UserFetch.js +++ b/packages/frontend-core/src/fetch/UserFetch.js @@ -1,6 +1,7 @@ import { get } from "svelte/store" import DataFetch from "./DataFetch.js" import { TableNames } from "../constants" +import { LuceneUtils } from "../utils" export default class UserFetch extends DataFetch { constructor(opts) { @@ -27,16 +28,25 @@ export default class UserFetch extends DataFetch { } async getData() { + const { limit, paginate } = this.options const { cursor, query } = get(this.store) + let finalQuery + // convert old format to new one - we now allow use of the lucene format + const { appId, paginated, ...rest} = query + if (!LuceneUtils.isLuceneFilter(query) && rest.email) { + finalQuery = { string: { email: rest.email }} + } else { + finalQuery = rest + } try { - // "query" normally contains a lucene query, but users uses a non-standard - // search endpoint so we use query uniquely here - const res = await this.API.searchUsers({ - page: cursor, - email: query.email, - appId: query.appId, - paginated: query.paginated, - }) + const opts = { + bookmark: cursor, + query: finalQuery, + appId: appId, + paginate: paginated || paginate, + limit, + } + const res = await this.API.searchUsers(opts) return { rows: res?.data || [], hasNextPage: res?.hasNextPage || false, diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 0a80253210..3ac8ceaf16 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -15,7 +15,7 @@ import { QuotaUsageType, RelationshipType, Row, - SaveTableRequest, + SaveTableRequest, SearchQueryOperators, SortOrder, SortType, StaticQuotaName, @@ -1141,7 +1141,7 @@ describe.each([ ) const createViewResponse = await config.createView({ - query: [{ operator: "equal", field: "age", value: 40 }], + query: [{ operator: SearchQueryOperators.EQUAL, field: "age", value: 40 }], schema: viewSchema, }) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 6d893c1c7f..7f4cf53ff1 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -3,6 +3,7 @@ import { CreateViewRequest, FieldSchema, FieldType, + SearchQueryOperators, SortOrder, SortType, Table, @@ -10,8 +11,8 @@ import { UpdateViewRequest, ViewV2, } from "@budibase/types" -import { generator } from "@budibase/backend-core/tests" -import { generateDatasourceID } from "../../../db/utils" +import {generator} from "@budibase/backend-core/tests" +import {generateDatasourceID} from "../../../db/utils" function priceTable(): Table { return { @@ -89,7 +90,7 @@ describe.each([ name: generator.name(), tableId: table._id!, primaryDisplay: generator.word(), - query: [{ operator: "equal", field: "field", value: "value" }], + query: [{ operator: SearchQueryOperators.EQUAL, field: "field", value: "value" }], sort: { field: "fieldToSort", order: SortOrder.DESCENDING, @@ -184,7 +185,7 @@ describe.each([ const tableId = table._id! await config.api.viewV2.update({ ...view, - query: [{ operator: "equal", field: "newField", value: "thatValue" }], + query: [{ operator: SearchQueryOperators.EQUAL, field: "newField", value: "thatValue" }], }) expect((await config.api.table.get(tableId)).views).toEqual({ @@ -207,7 +208,7 @@ describe.each([ primaryDisplay: generator.word(), query: [ { - operator: "equal", + operator: SearchQueryOperators.EQUAL, field: generator.word(), value: generator.word(), }, @@ -279,7 +280,7 @@ describe.each([ { ...view, tableId: generator.guid(), - query: [{ operator: "equal", field: "newField", value: "thatValue" }], + query: [{ operator: SearchQueryOperators.EQUAL, field: "newField", value: "thatValue" }], }, { expectStatus: 404 } ) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index ab43e86279..246b089af0 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -1,15 +1,16 @@ import { Datasource, + FieldSubtype, FieldType, - SortDirection, - SortType, SearchFilter, SearchQuery, SearchQueryFields, - FieldSubtype, + SearchQueryOperators, + SortDirection, + SortType, } from "@budibase/types" -import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" -import { deepGet } from "./helpers" +import {OperatorOptions, SqlNumberTypeRangeMap} from "./constants" +import {deepGet} from "./helpers" const HBS_REGEX = /{{([^{].*?)}}/g @@ -238,6 +239,18 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => { return query } +// type unknown +export const isLuceneFilter = (search: any) => { + if (typeof search !== "object") { + return false + } + const operators = Object.values(SearchQueryOperators) as string[] + const anySearchKey = Object.keys(search).find(key => { + return operators.includes(key) && typeof search[key] === "object" + }) + return !!anySearchKey +} + /** * Performs a client-side lucene search on an array of data * @param docs the data @@ -273,14 +286,14 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { } // Process a string match (fails if the value does not start with the string) - const stringMatch = match("string", (docValue: string, testValue: string) => { + const stringMatch = match(SearchQueryOperators.STRING, (docValue: string, testValue: string) => { return ( !docValue || !docValue?.toLowerCase().startsWith(testValue?.toLowerCase()) ) }) // Process a fuzzy match (treat the same as starts with when running locally) - const fuzzyMatch = match("fuzzy", (docValue: string, testValue: string) => { + const fuzzyMatch = match(SearchQueryOperators.FUZZY, (docValue: string, testValue: string) => { return ( !docValue || !docValue?.toLowerCase().startsWith(testValue?.toLowerCase()) ) @@ -288,7 +301,7 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { // Process a range match const rangeMatch = match( - "range", + SearchQueryOperators.RANGE, ( docValue: string | number | null, testValue: { low: number; high: number } @@ -304,7 +317,7 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { // Process an equal match (fails if the value is different) const equalMatch = match( - "equal", + SearchQueryOperators.EQUAL, (docValue: any, testValue: string | null) => { return testValue != null && testValue !== "" && docValue !== testValue } @@ -312,24 +325,24 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { // Process a not-equal match (fails if the value is the same) const notEqualMatch = match( - "notEqual", + SearchQueryOperators.NOT_EQUAL, (docValue: any, testValue: string | null) => { return testValue != null && testValue !== "" && docValue === testValue } ) // Process an empty match (fails if the value is not empty) - const emptyMatch = match("empty", (docValue: string | null) => { + const emptyMatch = match(SearchQueryOperators.EMPTY, (docValue: string | null) => { return docValue != null && docValue !== "" }) // Process a not-empty match (fails is the value is empty) - const notEmptyMatch = match("notEmpty", (docValue: string | null) => { + const notEmptyMatch = match(SearchQueryOperators.NOT_EMPTY, (docValue: string | null) => { return docValue == null || docValue === "" }) // Process an includes match (fails if the value is not included) - const oneOf = match("oneOf", (docValue: any, testValue: any) => { + const oneOf = match(SearchQueryOperators.ONE_OF, (docValue: any, testValue: any) => { if (typeof testValue === "string") { testValue = testValue.split(",") if (typeof docValue === "number") { @@ -339,19 +352,19 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { return !testValue?.includes(docValue) }) - const containsAny = match("containsAny", (docValue: any, testValue: any) => { + const containsAny = match(SearchQueryOperators.CONTAINS_ANY, (docValue: any, testValue: any) => { return !docValue?.includes(...testValue) }) const contains = match( - "contains", + SearchQueryOperators.CONTAINS, (docValue: string | any[], testValue: any[]) => { return !testValue?.every((item: any) => docValue?.includes(item)) } ) const notContains = match( - "notContains", + SearchQueryOperators.NOT_CONTAINS, (docValue: string | any[], testValue: any[]) => { return testValue?.every((item: any) => docValue?.includes(item)) } diff --git a/packages/types/src/api/web/searchFilter.ts b/packages/types/src/api/web/searchFilter.ts index 6980bc117b..ac3c446e36 100644 --- a/packages/types/src/api/web/searchFilter.ts +++ b/packages/types/src/api/web/searchFilter.ts @@ -10,43 +10,57 @@ export type SearchFilter = { externalType?: string } +export enum SearchQueryOperators { + STRING = "string", + FUZZY = "fuzzy", + RANGE = "range", + EQUAL = "equal", + NOT_EQUAL = "notEqual", + EMPTY = "empty", + NOT_EMPTY = "notEmpty", + ONE_OF = "oneOf", + CONTAINS = "contains", + NOT_CONTAINS = "notContains", + CONTAINS_ANY = "containsAny", +} + export type SearchQuery = { allOr?: boolean onEmptyFilter?: EmptyFilterOption - string?: { + [SearchQueryOperators.STRING]?: { [key: string]: string } - fuzzy?: { + [SearchQueryOperators.FUZZY]?: { [key: string]: string } - range?: { + [SearchQueryOperators.RANGE]?: { [key: string]: { high: number | string low: number | string } } - equal?: { + [SearchQueryOperators.EQUAL]?: { [key: string]: any } - notEqual?: { + [SearchQueryOperators.NOT_EQUAL]?: { [key: string]: any } - empty?: { + [SearchQueryOperators.EMPTY]?: { [key: string]: any } - notEmpty?: { + [SearchQueryOperators.NOT_EMPTY]?: { [key: string]: any } - oneOf?: { + [SearchQueryOperators.ONE_OF]?: { [key: string]: any[] } - contains?: { + [SearchQueryOperators.CONTAINS]?: { [key: string]: any[] } - notContains?: { + [SearchQueryOperators.NOT_CONTAINS]?: { [key: string]: any[] } - containsAny?: { + [SearchQueryOperators.CONTAINS_ANY]?: { [key: string]: any[] } } diff --git a/yarn.lock b/yarn.lock index 81c2815663..11d296b296 100644 --- a/yarn.lock +++ b/yarn.lock @@ -21750,7 +21750,7 @@ vlq@^0.2.2: resolved "https://registry.yarnpkg.com/vlq/-/vlq-0.2.3.tgz#8f3e4328cf63b1540c0d67e1b2778386f8975b26" integrity sha512-DRibZL6DsNhIgYQ+wNdWDL2SL3bKPlVrRiBqV5yuMm++op8W4kGFtaQfCs4KEJn0wBZcHVHJ3eoywX8983k1ow== -vm2@^3.9.19: +vm2@^3.9.19, vm2@^3.9.8: version "3.9.19" resolved "https://registry.yarnpkg.com/vm2/-/vm2-3.9.19.tgz#be1e1d7a106122c6c492b4d51c2e8b93d3ed6a4a" integrity sha512-J637XF0DHDMV57R6JyVsTak7nIL8gy5KH4r1HiwWLf/4GBbb5MKL5y7LpmF4A8E2nR6XmzpmMFQ7V7ppPTmUQg== From 16d551542e77e76077079716fffcc1ba7486329e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 12 Oct 2023 19:00:53 +0100 Subject: [PATCH 12/31] Frontend component of updating the API and accounting for default value not being in the paginated results of the relationship picker. --- packages/backend-core/src/users/users.ts | 44 ++++++--- .../app/forms/RelationshipField.svelte | 37 ++++++-- packages/frontend-core/src/api/user.js | 1 - packages/frontend-core/src/fetch/UserFetch.js | 6 +- .../server/src/api/routes/tests/row.spec.ts | 7 +- .../src/api/routes/tests/viewV2.spec.ts | 28 ++++-- packages/shared-core/src/filters.ts | 90 ++++++++++--------- packages/types/src/api/web/user.ts | 7 +- .../src/api/controllers/global/users.ts | 7 +- 9 files changed, 153 insertions(+), 74 deletions(-) diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index a7e1389920..71ac460112 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -14,13 +14,14 @@ import { } from "../db" import { BulkDocsResponse, + ContextUser, + SearchQuery, + SearchQueryOperators, SearchUsersRequest, User, - ContextUser, } from "@budibase/types" -import { getGlobalDB } from "../context" import * as context from "../context" -import { user as userCache } from "../cache" +import { getGlobalDB } from "../context" type GetOpts = { cleanup?: boolean } @@ -39,6 +40,27 @@ function removeUserPassword(users: User | User[]) { return users } +export const isSupportedUserSearch = (query: SearchQuery) => { + const allowed = [ + { op: SearchQueryOperators.STRING, key: "email" }, + { op: SearchQueryOperators.EQUAL, key: "_id" }, + ] + for (let [key, operation] of Object.entries(query)) { + if (typeof operation !== "object") { + return false + } + const fields = Object.keys(operation || {}) + const allowedOperation = allowed.find( + allow => + allow.op === key && fields.length === 1 && fields[0] === allow.key + ) + if (!allowedOperation && fields.length > 0) { + return false + } + } + return true +} + export const bulkGetGlobalUsersById = async ( userIds: string[], opts?: GetOpts @@ -211,8 +233,8 @@ export const searchGlobalUsersByEmail = async ( const PAGE_LIMIT = 8 export const paginatedUsers = async ({ - page, - email, + bookmark, + query, appId, }: SearchUsersRequest = {}) => { const db = getGlobalDB() @@ -222,18 +244,20 @@ export const paginatedUsers = async ({ limit: PAGE_LIMIT + 1, } // add a startkey if the page was specified (anchor) - if (page) { - opts.startkey = page + if (bookmark) { + opts.startkey = bookmark } // property specifies what to use for the page/anchor let userList: User[], property = "_id", getKey - if (appId) { + if (query?.equal?._id) { + userList = [await getById(query.equal._id)] + } else if (appId) { userList = await searchGlobalUsersByApp(appId, opts) getKey = (doc: any) => getGlobalUserByAppPage(appId, doc) - } else if (email) { - userList = await searchGlobalUsersByEmail(email, opts) + } else if (query?.string?.email) { + userList = await searchGlobalUsersByEmail(query?.string?.email, opts) property = "email" } else { // no search, query allDocs diff --git a/packages/client/src/components/app/forms/RelationshipField.svelte b/packages/client/src/components/app/forms/RelationshipField.svelte index 52faf46615..5dbf51e169 100644 --- a/packages/client/src/components/app/forms/RelationshipField.svelte +++ b/packages/client/src/components/app/forms/RelationshipField.svelte @@ -4,6 +4,7 @@ import { getContext } from "svelte" import Field from "./Field.svelte" import { FieldTypes } from "../../../constants" + import { onMount } from "svelte" const { API } = getContext("sdk") @@ -25,6 +26,7 @@ let tableDefinition let searchTerm let open + let hasFetchedDefault, fetchedDefault $: type = datasourceType === "table" ? FieldTypes.LINK : FieldTypes.BB_REFERENCE @@ -75,8 +77,8 @@ } } - $: enrichedOptions = enrichOptions(optionsObj, $fetch.rows) - const enrichOptions = (optionsObj, fetchResults) => { + $: enrichedOptions = enrichOptions(optionsObj, $fetch.rows, fetchedDefault) + const enrichOptions = (optionsObj, fetchResults, fetchedDefault) => { const result = (fetchResults || [])?.reduce((accumulator, row) => { if (!accumulator[row._id]) { accumulator[row._id] = row @@ -84,7 +86,11 @@ return accumulator }, optionsObj) - return Object.values(result) + const final = Object.values(result) + if (fetchedDefault && !final.find(row => row._id === fetchedDefault._id)) { + final.push(fetchedDefault) + } + return final } $: { // We don't want to reorder while the dropdown is open, to avoid UX jumps @@ -105,16 +111,17 @@ } } - $: fetchRows(searchTerm, primaryDisplay) + $: fetchRows(searchTerm, primaryDisplay, hasFetchedDefault) - const fetchRows = (searchTerm, primaryDisplay) => { + const fetchRows = async (searchTerm, primaryDisplay, gotDefault) => { const allRowsFetched = $fetch.loaded && !Object.keys($fetch.query?.string || {}).length && !$fetch.hasNextPage - // Don't request until we have the primary display - if (!allRowsFetched && primaryDisplay) { - fetch.update({ + const shouldFetch = !defaultValue ? !allRowsFetched : gotDefault + // Don't request until we have the primary display or default value has been fetched + if (shouldFetch && primaryDisplay) { + await fetch.update({ query: { string: { [primaryDisplay]: searchTerm } }, }) } @@ -171,6 +178,20 @@ fetch.nextPage() } } + + onMount(async () => { + // the pagination might not include the default row + if (defaultValue) { + await fetch.update({ + query: { equal: { _id: defaultValue }} + }) + const fetched = $fetch.rows?.[0] + if (fetched) { + fetchedDefault = { ...fetched } + } + } + hasFetchedDefault = true + }) ({ if (limit) { opts.limit = limit } - console.log(opts) return await API.post({ url: `/api/global/users/search`, body: opts, diff --git a/packages/frontend-core/src/fetch/UserFetch.js b/packages/frontend-core/src/fetch/UserFetch.js index 2158b3ca57..b1478c3a6d 100644 --- a/packages/frontend-core/src/fetch/UserFetch.js +++ b/packages/frontend-core/src/fetch/UserFetch.js @@ -32,9 +32,9 @@ export default class UserFetch extends DataFetch { const { cursor, query } = get(this.store) let finalQuery // convert old format to new one - we now allow use of the lucene format - const { appId, paginated, ...rest} = query - if (!LuceneUtils.isLuceneFilter(query) && rest.email) { - finalQuery = { string: { email: rest.email }} + const { appId, paginated, ...rest } = query + if (!LuceneUtils.hasFilters(query) && rest.email) { + finalQuery = { string: { email: rest.email } } } else { finalQuery = rest } diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 3ac8ceaf16..4c2e7a7494 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -15,7 +15,8 @@ import { QuotaUsageType, RelationshipType, Row, - SaveTableRequest, SearchQueryOperators, + SaveTableRequest, + SearchQueryOperators, SortOrder, SortType, StaticQuotaName, @@ -1141,7 +1142,9 @@ describe.each([ ) const createViewResponse = await config.createView({ - query: [{ operator: SearchQueryOperators.EQUAL, field: "age", value: 40 }], + query: [ + { operator: SearchQueryOperators.EQUAL, field: "age", value: 40 }, + ], schema: viewSchema, }) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 7f4cf53ff1..40060aef48 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -11,8 +11,8 @@ import { UpdateViewRequest, ViewV2, } from "@budibase/types" -import {generator} from "@budibase/backend-core/tests" -import {generateDatasourceID} from "../../../db/utils" +import { generator } from "@budibase/backend-core/tests" +import { generateDatasourceID } from "../../../db/utils" function priceTable(): Table { return { @@ -90,7 +90,13 @@ describe.each([ name: generator.name(), tableId: table._id!, primaryDisplay: generator.word(), - query: [{ operator: SearchQueryOperators.EQUAL, field: "field", value: "value" }], + query: [ + { + operator: SearchQueryOperators.EQUAL, + field: "field", + value: "value", + }, + ], sort: { field: "fieldToSort", order: SortOrder.DESCENDING, @@ -185,7 +191,13 @@ describe.each([ const tableId = table._id! await config.api.viewV2.update({ ...view, - query: [{ operator: SearchQueryOperators.EQUAL, field: "newField", value: "thatValue" }], + query: [ + { + operator: SearchQueryOperators.EQUAL, + field: "newField", + value: "thatValue", + }, + ], }) expect((await config.api.table.get(tableId)).views).toEqual({ @@ -280,7 +292,13 @@ describe.each([ { ...view, tableId: generator.guid(), - query: [{ operator: SearchQueryOperators.EQUAL, field: "newField", value: "thatValue" }], + query: [ + { + operator: SearchQueryOperators.EQUAL, + field: "newField", + value: "thatValue", + }, + ], }, { expectStatus: 404 } ) diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 246b089af0..1839a53525 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -9,8 +9,8 @@ import { SortDirection, SortType, } from "@budibase/types" -import {OperatorOptions, SqlNumberTypeRangeMap} from "./constants" -import {deepGet} from "./helpers" +import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" +import { deepGet } from "./helpers" const HBS_REGEX = /{{([^{].*?)}}/g @@ -239,18 +239,6 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => { return query } -// type unknown -export const isLuceneFilter = (search: any) => { - if (typeof search !== "object") { - return false - } - const operators = Object.values(SearchQueryOperators) as string[] - const anySearchKey = Object.keys(search).find(key => { - return operators.includes(key) && typeof search[key] === "object" - }) - return !!anySearchKey -} - /** * Performs a client-side lucene search on an array of data * @param docs the data @@ -286,18 +274,26 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { } // Process a string match (fails if the value does not start with the string) - const stringMatch = match(SearchQueryOperators.STRING, (docValue: string, testValue: string) => { - return ( - !docValue || !docValue?.toLowerCase().startsWith(testValue?.toLowerCase()) - ) - }) + const stringMatch = match( + SearchQueryOperators.STRING, + (docValue: string, testValue: string) => { + return ( + !docValue || + !docValue?.toLowerCase().startsWith(testValue?.toLowerCase()) + ) + } + ) // Process a fuzzy match (treat the same as starts with when running locally) - const fuzzyMatch = match(SearchQueryOperators.FUZZY, (docValue: string, testValue: string) => { - return ( - !docValue || !docValue?.toLowerCase().startsWith(testValue?.toLowerCase()) - ) - }) + const fuzzyMatch = match( + SearchQueryOperators.FUZZY, + (docValue: string, testValue: string) => { + return ( + !docValue || + !docValue?.toLowerCase().startsWith(testValue?.toLowerCase()) + ) + } + ) // Process a range match const rangeMatch = match( @@ -332,29 +328,41 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => { ) // Process an empty match (fails if the value is not empty) - const emptyMatch = match(SearchQueryOperators.EMPTY, (docValue: string | null) => { - return docValue != null && docValue !== "" - }) + const emptyMatch = match( + SearchQueryOperators.EMPTY, + (docValue: string | null) => { + return docValue != null && docValue !== "" + } + ) // Process a not-empty match (fails is the value is empty) - const notEmptyMatch = match(SearchQueryOperators.NOT_EMPTY, (docValue: string | null) => { - return docValue == null || docValue === "" - }) + const notEmptyMatch = match( + SearchQueryOperators.NOT_EMPTY, + (docValue: string | null) => { + return docValue == null || docValue === "" + } + ) // Process an includes match (fails if the value is not included) - const oneOf = match(SearchQueryOperators.ONE_OF, (docValue: any, testValue: any) => { - if (typeof testValue === "string") { - testValue = testValue.split(",") - if (typeof docValue === "number") { - testValue = testValue.map((item: string) => parseFloat(item)) + const oneOf = match( + SearchQueryOperators.ONE_OF, + (docValue: any, testValue: any) => { + if (typeof testValue === "string") { + testValue = testValue.split(",") + if (typeof docValue === "number") { + testValue = testValue.map((item: string) => parseFloat(item)) + } } + return !testValue?.includes(docValue) } - return !testValue?.includes(docValue) - }) + ) - const containsAny = match(SearchQueryOperators.CONTAINS_ANY, (docValue: any, testValue: any) => { - return !docValue?.includes(...testValue) - }) + const containsAny = match( + SearchQueryOperators.CONTAINS_ANY, + (docValue: any, testValue: any) => { + return !docValue?.includes(...testValue) + } + ) const contains = match( SearchQueryOperators.CONTAINS, @@ -446,7 +454,7 @@ export const hasFilters = (query?: SearchQuery) => { if (skipped.includes(key) || typeof value !== "object") { continue } - if (Object.keys(value).length !== 0) { + if (Object.keys(value || {}).length !== 0) { return true } } diff --git a/packages/types/src/api/web/user.ts b/packages/types/src/api/web/user.ts index 85e2d89ad1..a1e039cfd7 100644 --- a/packages/types/src/api/web/user.ts +++ b/packages/types/src/api/web/user.ts @@ -1,4 +1,5 @@ import { User } from "../../documents" +import { SearchQuery } from "./searchFilter" export interface SaveUserResponse { _id: string @@ -51,10 +52,10 @@ export interface InviteUsersResponse { } export interface SearchUsersRequest { - page?: string - email?: string + bookmark?: string + query?: SearchQuery appId?: string - paginated?: boolean + paginate?: boolean } export interface CreateAdminUserRequest { diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 822a16d33e..8de3a1444e 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -197,7 +197,12 @@ export const getAppUsers = async (ctx: Ctx) => { export const search = async (ctx: Ctx) => { const body = ctx.request.body - if (body.paginated === false) { + // TODO: for now only one supported search key, string.email + if (body?.query && !userSdk.core.isSupportedUserSearch(body.query)) { + ctx.throw(501, "Can only search by string.email or equal._id") + } + + if (body.paginate === false) { await getAppUsers(ctx) } else { const paginated = await userSdk.core.paginatedUsers(body) From 535a6e1a472f29f17f32474c04483fb5514d6ca6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 12 Oct 2023 19:07:04 +0100 Subject: [PATCH 13/31] Minor cleanup. --- .../app/forms/RelationshipField.svelte | 36 ++++++++----------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/packages/client/src/components/app/forms/RelationshipField.svelte b/packages/client/src/components/app/forms/RelationshipField.svelte index 5dbf51e169..f1324e57af 100644 --- a/packages/client/src/components/app/forms/RelationshipField.svelte +++ b/packages/client/src/components/app/forms/RelationshipField.svelte @@ -26,7 +26,7 @@ let tableDefinition let searchTerm let open - let hasFetchedDefault, fetchedDefault + let fetchedDefault $: type = datasourceType === "table" ? FieldTypes.LINK : FieldTypes.BB_REFERENCE @@ -111,20 +111,26 @@ } } - $: fetchRows(searchTerm, primaryDisplay, hasFetchedDefault) + $: fetchRows(searchTerm, primaryDisplay, defaultValue) - const fetchRows = async (searchTerm, primaryDisplay, gotDefault) => { + const fetchRows = async (searchTerm, primaryDisplay, defaultVal) => { const allRowsFetched = $fetch.loaded && !Object.keys($fetch.query?.string || {}).length && !$fetch.hasNextPage - const shouldFetch = !defaultValue ? !allRowsFetched : gotDefault // Don't request until we have the primary display or default value has been fetched - if (shouldFetch && primaryDisplay) { - await fetch.update({ - query: { string: { [primaryDisplay]: searchTerm } }, - }) + if (allRowsFetched || !primaryDisplay) { + return } + if (defaultVal) { + await fetch.update({ + query: {equal: {_id: defaultValue}} + }) + fetchedDefault = $fetch.rows?.[0] + } + await fetch.update({ + query: { string: { [primaryDisplay]: searchTerm } }, + }) } const flatten = values => { @@ -178,20 +184,6 @@ fetch.nextPage() } } - - onMount(async () => { - // the pagination might not include the default row - if (defaultValue) { - await fetch.update({ - query: { equal: { _id: defaultValue }} - }) - const fetched = $fetch.rows?.[0] - if (fetched) { - fetchedDefault = { ...fetched } - } - } - hasFetchedDefault = true - }) Date: Thu, 12 Oct 2023 19:07:22 +0100 Subject: [PATCH 14/31] Removing un-used function. --- .../client/src/components/app/forms/RelationshipField.svelte | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/client/src/components/app/forms/RelationshipField.svelte b/packages/client/src/components/app/forms/RelationshipField.svelte index f1324e57af..3608c15a3c 100644 --- a/packages/client/src/components/app/forms/RelationshipField.svelte +++ b/packages/client/src/components/app/forms/RelationshipField.svelte @@ -4,7 +4,6 @@ import { getContext } from "svelte" import Field from "./Field.svelte" import { FieldTypes } from "../../../constants" - import { onMount } from "svelte" const { API } = getContext("sdk") From bd182d5b3b9477858784b68cc57fc10ab6badc34 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 12 Oct 2023 19:19:05 +0100 Subject: [PATCH 15/31] Test cases for updated API. --- .../src/api/routes/global/tests/users.spec.ts | 30 +++++++++++++++++++ packages/worker/src/tests/api/users.ts | 10 +++++++ 2 files changed, 40 insertions(+) diff --git a/packages/worker/src/api/routes/global/tests/users.spec.ts b/packages/worker/src/api/routes/global/tests/users.spec.ts index e4504eccfe..a446d10ed0 100644 --- a/packages/worker/src/api/routes/global/tests/users.spec.ts +++ b/packages/worker/src/api/routes/global/tests/users.spec.ts @@ -544,6 +544,36 @@ describe("/api/global/users", () => { }) }) + describe("POST /api/global/users/search", () => { + it("should be able to search by email", async () => { + const user = await config.createUser() + const response = await config.api.users.searchUsers({ + query: { string: { email: user.email } }, + }) + expect(response.body.data.length).toBe(1) + expect(response.body.data[0].email).toBe(user.email) + }) + + it("should be able to search by _id", async () => { + const user = await config.createUser() + const response = await config.api.users.searchUsers({ + query: { equal: { _id: user._id } }, + }) + expect(response.body.data.length).toBe(1) + expect(response.body.data[0]._id).toBe(user._id) + }) + + it("should throw an error when unimplemented options used", async () => { + const user = await config.createUser() + await config.api.users.searchUsers( + { + query: { equal: { firstName: user.firstName } }, + }, + 501 + ) + }) + }) + describe("DELETE /api/global/users/:userId", () => { it("should be able to destroy a basic user", async () => { const user = await config.createUser() diff --git a/packages/worker/src/tests/api/users.ts b/packages/worker/src/tests/api/users.ts index e96209eca6..b2a19bcb28 100644 --- a/packages/worker/src/tests/api/users.ts +++ b/packages/worker/src/tests/api/users.ts @@ -4,6 +4,7 @@ import { InviteUsersRequest, User, CreateAdminUserRequest, + SearchQuery, } from "@budibase/types" import structures from "../structures" import { generator } from "@budibase/backend-core/tests" @@ -133,6 +134,15 @@ export class UserAPI extends TestAPI { .expect(status ? status : 200) } + searchUsers = ({ query }: { query?: SearchQuery }, status = 200) => { + return this.request + .post("/api/global/users/search") + .set(this.config.defaultHeaders()) + .send({ query }) + .expect("Content-Type", /json/) + .expect(status ? status : 200) + } + getUser = (userId: string, opts?: TestAPIOpts) => { return this.request .get(`/api/global/users/${userId}`) From 2e8e21e5d26eef1040a03aa4f01fe2ac3495c614 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 12 Oct 2023 19:20:03 +0100 Subject: [PATCH 16/31] Linting. --- .../client/src/components/app/forms/RelationshipField.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/components/app/forms/RelationshipField.svelte b/packages/client/src/components/app/forms/RelationshipField.svelte index 3608c15a3c..1511e75c5d 100644 --- a/packages/client/src/components/app/forms/RelationshipField.svelte +++ b/packages/client/src/components/app/forms/RelationshipField.svelte @@ -123,7 +123,7 @@ } if (defaultVal) { await fetch.update({ - query: {equal: {_id: defaultValue}} + query: { equal: { _id: defaultValue } }, }) fetchedDefault = $fetch.rows?.[0] } From 18363b7b60bb98c5cc00d4988cbb41c09b093e46 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 13 Oct 2023 09:50:45 +0100 Subject: [PATCH 17/31] Make sure to stop postgres container after the postgres integration tests have run. --- packages/server/src/integration-test/postgres.spec.ts | 4 ++++ packages/server/src/integrations/tests/utils/postgres.ts | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index bcd1c14389..84c19f8bbc 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -43,6 +43,10 @@ describe("postgres integrations", () => { ) }) + afterAll(async () => { + await databaseTestProviders.postgres.stopContainer() + }) + beforeEach(async () => { async function createAuxTable(prefix: string) { return await config.createTable({ diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts index 036e81bbd8..b749551721 100644 --- a/packages/server/src/integrations/tests/utils/postgres.ts +++ b/packages/server/src/integrations/tests/utils/postgres.ts @@ -36,3 +36,10 @@ export async function getDsConfig(): Promise { }, } } + +export async function stopContainer() { + if (container) { + await container.stop() + container = undefined + } +} From d3366a535bb6fda5992cf006f33a85312d77e24f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 13 Oct 2023 10:22:29 +0100 Subject: [PATCH 18/31] PR comments. --- packages/backend-core/src/users/users.ts | 6 +++++- .../app/forms/RelationshipField.svelte | 16 +++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 71ac460112..b087a6b538 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -50,11 +50,15 @@ export const isSupportedUserSearch = (query: SearchQuery) => { return false } const fields = Object.keys(operation || {}) + // this filter doesn't contain options - ignore + if (fields.length === 0) { + continue + } const allowedOperation = allowed.find( allow => allow.op === key && fields.length === 1 && fields[0] === allow.key ) - if (!allowedOperation && fields.length > 0) { + if (!allowedOperation) { return false } } diff --git a/packages/client/src/components/app/forms/RelationshipField.svelte b/packages/client/src/components/app/forms/RelationshipField.svelte index 1511e75c5d..960ad0eaec 100644 --- a/packages/client/src/components/app/forms/RelationshipField.svelte +++ b/packages/client/src/components/app/forms/RelationshipField.svelte @@ -25,7 +25,6 @@ let tableDefinition let searchTerm let open - let fetchedDefault $: type = datasourceType === "table" ? FieldTypes.LINK : FieldTypes.BB_REFERENCE @@ -76,8 +75,8 @@ } } - $: enrichedOptions = enrichOptions(optionsObj, $fetch.rows, fetchedDefault) - const enrichOptions = (optionsObj, fetchResults, fetchedDefault) => { + $: enrichedOptions = enrichOptions(optionsObj, $fetch.rows) + const enrichOptions = (optionsObj, fetchResults) => { const result = (fetchResults || [])?.reduce((accumulator, row) => { if (!accumulator[row._id]) { accumulator[row._id] = row @@ -85,11 +84,7 @@ return accumulator }, optionsObj) - const final = Object.values(result) - if (fetchedDefault && !final.find(row => row._id === fetchedDefault._id)) { - final.push(fetchedDefault) - } - return final + return Object.values(result) } $: { // We don't want to reorder while the dropdown is open, to avoid UX jumps @@ -125,7 +120,10 @@ await fetch.update({ query: { equal: { _id: defaultValue } }, }) - fetchedDefault = $fetch.rows?.[0] + const defaultRow = $fetch.rows?.[0] + if (defaultRow) { + optionsObj[defaultRow._id] = defaultRow + } } await fetch.update({ query: { string: { [primaryDisplay]: searchTerm } }, From 7f5adeefdebf23059bcc6892a2456a6cb6b91261 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 13 Oct 2023 09:23:11 +0000 Subject: [PATCH 19/31] Bump version to 2.11.32 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 51c702a2f6..8764e90ee7 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.31", + "version": "2.11.32", "npmClient": "yarn", "packages": [ "packages/*" From 563baa53c3d8ebfd0cbf567584e773e33a0407bb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 13 Oct 2023 11:28:23 +0100 Subject: [PATCH 20/31] Fix var usage. --- .../client/src/components/app/forms/RelationshipField.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/components/app/forms/RelationshipField.svelte b/packages/client/src/components/app/forms/RelationshipField.svelte index 960ad0eaec..81efbfd2bd 100644 --- a/packages/client/src/components/app/forms/RelationshipField.svelte +++ b/packages/client/src/components/app/forms/RelationshipField.svelte @@ -118,7 +118,7 @@ } if (defaultVal) { await fetch.update({ - query: { equal: { _id: defaultValue } }, + query: { equal: { _id: defaultVal } }, }) const defaultRow = $fetch.rows?.[0] if (defaultRow) { From ccfc98120dcad1df7ca02b6b90f479a989b4ef8f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 13 Oct 2023 11:44:15 +0100 Subject: [PATCH 21/31] Adding check to not re-fetch default value. --- .../client/src/components/app/forms/RelationshipField.svelte | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/client/src/components/app/forms/RelationshipField.svelte b/packages/client/src/components/app/forms/RelationshipField.svelte index 81efbfd2bd..da21d8610c 100644 --- a/packages/client/src/components/app/forms/RelationshipField.svelte +++ b/packages/client/src/components/app/forms/RelationshipField.svelte @@ -25,6 +25,7 @@ let tableDefinition let searchTerm let open + let hasFetchedDefaultValue $: type = datasourceType === "table" ? FieldTypes.LINK : FieldTypes.BB_REFERENCE @@ -116,7 +117,7 @@ if (allRowsFetched || !primaryDisplay) { return } - if (defaultVal) { + if (defaultVal && !hasFetchedDefaultValue) { await fetch.update({ query: { equal: { _id: defaultVal } }, }) @@ -124,6 +125,7 @@ if (defaultRow) { optionsObj[defaultRow._id] = defaultRow } + hasFetchedDefaultValue = true } await fetch.update({ query: { string: { [primaryDisplay]: searchTerm } }, From 03371005c143ccd347dec298712e76f7a42e3b1e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 13 Oct 2023 12:23:28 +0100 Subject: [PATCH 22/31] PR comments - addressing API.searchUser updates. --- .../src/pages/builder/portal/account/auditLogs/index.svelte | 5 ++++- .../portal/users/groups/_components/EditUserPicker.svelte | 5 ++++- .../src/components/grid/cells/BBReferenceCell.svelte | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte b/packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte index a6dba59196..a5b0b19c9b 100644 --- a/packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte +++ b/packages/builder/src/pages/builder/portal/account/auditLogs/index.svelte @@ -123,7 +123,10 @@ prevUserSearch = search try { userPageInfo.loading() - await users.search({ userPage, email: search }) + await users.search({ + bookmark: userPage, + query: { string: { email: search } }, + }) userPageInfo.fetched($users.hasNextPage, $users.nextPage) } catch (error) { notifications.error("Error getting user list") diff --git a/packages/builder/src/pages/builder/portal/users/groups/_components/EditUserPicker.svelte b/packages/builder/src/pages/builder/portal/users/groups/_components/EditUserPicker.svelte index da4b12f7f9..cc524f1acf 100644 --- a/packages/builder/src/pages/builder/portal/users/groups/_components/EditUserPicker.svelte +++ b/packages/builder/src/pages/builder/portal/users/groups/_components/EditUserPicker.svelte @@ -31,7 +31,10 @@ prevSearch = search try { pageInfo.loading() - await users.search({ page, email: search }) + await users.search({ + bookmark: page, + query: { string: { email: search } }, + }) pageInfo.fetched($users.hasNextPage, $users.nextPage) } catch (error) { notifications.error("Error getting user list") diff --git a/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte b/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte index 4e76c264a1..48b1279346 100644 --- a/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte @@ -27,7 +27,7 @@ const email = Object.values(searchParams.query.string)[0] const results = await API.searchUsers({ - email, + query: { string: { email } }, }) // Mapping to the expected data within RelationshipCell From c5360e1dcb3fe2028b14627252e660dd1fabaf72 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 13 Oct 2023 12:53:18 +0100 Subject: [PATCH 23/31] Don't need to put into options obj, reaction does work. --- .../client/src/components/app/forms/RelationshipField.svelte | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/client/src/components/app/forms/RelationshipField.svelte b/packages/client/src/components/app/forms/RelationshipField.svelte index da21d8610c..2c86a8abca 100644 --- a/packages/client/src/components/app/forms/RelationshipField.svelte +++ b/packages/client/src/components/app/forms/RelationshipField.svelte @@ -121,10 +121,6 @@ await fetch.update({ query: { equal: { _id: defaultVal } }, }) - const defaultRow = $fetch.rows?.[0] - if (defaultRow) { - optionsObj[defaultRow._id] = defaultRow - } hasFetchedDefaultValue = true } await fetch.update({ From f64800f9f423940145662d709fbfcabb97371c6b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 13 Oct 2023 13:05:07 +0100 Subject: [PATCH 24/31] Final PR comment. --- .../client/src/components/app/forms/RelationshipField.svelte | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/client/src/components/app/forms/RelationshipField.svelte b/packages/client/src/components/app/forms/RelationshipField.svelte index 2c86a8abca..544a1a8434 100644 --- a/packages/client/src/components/app/forms/RelationshipField.svelte +++ b/packages/client/src/components/app/forms/RelationshipField.svelte @@ -25,7 +25,6 @@ let tableDefinition let searchTerm let open - let hasFetchedDefaultValue $: type = datasourceType === "table" ? FieldTypes.LINK : FieldTypes.BB_REFERENCE @@ -117,11 +116,10 @@ if (allRowsFetched || !primaryDisplay) { return } - if (defaultVal && !hasFetchedDefaultValue) { + if (defaultVal && !optionsObj[defaultVal]) { await fetch.update({ query: { equal: { _id: defaultVal } }, }) - hasFetchedDefaultValue = true } await fetch.update({ query: { string: { [primaryDisplay]: searchTerm } }, From 7cef9bf597e0759609d2c8b8b2d06cfbe7ddafec Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 13 Oct 2023 12:21:43 +0000 Subject: [PATCH 25/31] Bump version to 2.11.33 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 8764e90ee7..4894f679f3 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.32", + "version": "2.11.33", "npmClient": "yarn", "packages": [ "packages/*" From d26789a330d423a765cc9a5cf8ac81a2e8e6be2e Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Fri, 13 Oct 2023 18:04:23 +0100 Subject: [PATCH 26/31] Bug warnings stale bot (#12053) * Bug warnings * Lower operations per run --- .github/workflows/stale_bot.yml | 43 +++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/.github/workflows/stale_bot.yml b/.github/workflows/stale_bot.yml index f87d561db9..49e3473e63 100644 --- a/.github/workflows/stale_bot.yml +++ b/.github/workflows/stale_bot.yml @@ -2,7 +2,7 @@ name: Close stale issues and PRs # https://github.com/actions/stale on: workflow_dispatch: schedule: - - cron: '*/30 * * * *' # Every 30 mins + - cron: "*/30 * * * *" # Every 30 mins jobs: stale: @@ -10,20 +10,37 @@ jobs: steps: - uses: actions/stale@v8 with: - # stale rules - days-before-stale: 60 + operations-per-run: 1 + # stale rules for PRs days-before-pr-stale: 7 stale-issue-label: stale - stale-issue-message: "This issue has been automatically marked as stale because it has not had any activity for 60 days." - - # close rules - # days after being marked as stale to close - days-before-close: 30 - close-issue-label: closed-stale - close-issue-message: This issue has been automatically closed it has not had any activity in 90 days." - days-before-pr-close: 7 - - # exemptions exempt-pr-labels: pinned,security,roadmap + days-before-pr-close: 7 + - uses: actions/stale@v8 + with: + operations-per-run: 3 + # stale rules for high priority bugs + days-before-stale: 30 + only-issue-labels: bug,High priority + stale-issue-label: warn + + - uses: actions/stale@v8 + with: + operations-per-run: 3 + # stale rules for medium priority bugs + days-before-stale: 90 + only-issue-labels: bug,Medium priority + stale-issue-label: warn + + - uses: actions/stale@v8 + with: + operations-per-run: 3 + # stale rules for all bugs + days-before-stale: 180 + stale-issue-label: stale + only-issue-labels: bug + stale-issue-message: "This issue has been automatically marked as stale because it has not had any activity for six months." + + days-before-close: 30 From 6754c93ee410fc2e4fa00d632114920ee0976954 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 13 Oct 2023 17:04:44 +0000 Subject: [PATCH 27/31] Bump version to 2.11.34 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 4894f679f3..0dd6abdc46 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.33", + "version": "2.11.34", "npmClient": "yarn", "packages": [ "packages/*" From c269573f074c4c9b66bdb0277abc9d6b37126a83 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 16 Oct 2023 11:00:41 +0100 Subject: [PATCH 28/31] removing auto tag of release --- .github/workflows/tag-release.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/tag-release.yml b/.github/workflows/tag-release.yml index 191c3ad9ef..eaf71ae61a 100644 --- a/.github/workflows/tag-release.yml +++ b/.github/workflows/tag-release.yml @@ -4,17 +4,6 @@ concurrency: cancel-in-progress: false on: - push: - branches: - - master - paths: - - ".aws/**" - - ".github/**" - - "charts/**" - - "packages/**" - - "scripts/**" - - "package.json" - - "yarn.lock" workflow_dispatch: inputs: versioning: From f2dbb4106d4bebae5e05ce19d795819820f0fa91 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 16 Oct 2023 11:04:33 +0100 Subject: [PATCH 29/31] update CI job to remove references to develop --- .github/workflows/budibase_ci.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index c8bc31f41d..840d580892 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -10,7 +10,6 @@ on: push: branches: - master - - develop pull_request: workflow_dispatch: @@ -262,11 +261,7 @@ jobs: branch="${{ github.base_ref || github.ref_name }}" echo "Running on branch '$branch' (base_ref=${{ github.base_ref }}, ref_name=${{ github.head_ref }})" - if [[ $branch == "master" ]]; then - base_commit=$(git rev-parse origin/master) - elif [[ $branch == "develop" ]]; then - base_commit=$(git rev-parse origin/develop) - fi + base_commit=$(git rev-parse origin/master) if [[ ! -z $base_commit ]]; then echo "target_branch=$branch" From 4337463402d39d718d02b28c1feef73fd46e1ccb Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 16 Oct 2023 11:06:54 +0100 Subject: [PATCH 30/31] remove develop reference in feature branch close job --- .github/workflows/close-featurebranch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/close-featurebranch.yml b/.github/workflows/close-featurebranch.yml index 0ec3b43598..47de15b58b 100644 --- a/.github/workflows/close-featurebranch.yml +++ b/.github/workflows/close-featurebranch.yml @@ -4,7 +4,7 @@ on: pull_request: types: [closed] branches: - - develop + - master jobs: release: From b22f7804dba26582acd9a2c2006e290cc0873a7e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 16 Oct 2023 12:21:45 +0100 Subject: [PATCH 31/31] allow close-featurebranch to be done through workflow dispatch --- .github/workflows/close-featurebranch.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/close-featurebranch.yml b/.github/workflows/close-featurebranch.yml index 47de15b58b..bb64f5ce20 100644 --- a/.github/workflows/close-featurebranch.yml +++ b/.github/workflows/close-featurebranch.yml @@ -5,6 +5,12 @@ on: types: [closed] branches: - master + workflow_dispatch: + inputs: + BRANCH: + type: string + description: Which featurebranch branch to destroy? + required: true jobs: release: @@ -13,7 +19,7 @@ jobs: - uses: actions/checkout@v3 - uses: passeidireto/trigger-external-workflow-action@main env: - PAYLOAD_BRANCH: ${{ github.head_ref }} + PAYLOAD_BRANCH: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.BRANCH || github.head_ref }} PAYLOAD_PR_NUMBER: ${{ github.ref }} with: repository: budibase/budibase-deploys