From b07382fa2e0efb0009bf98f14f3140ebd07da591 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Sat, 4 Nov 2023 11:04:09 +0100 Subject: [PATCH 01/16] WIP - Migrating worker --- packages/types/src/sdk/licensing/plan.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/types/src/sdk/licensing/plan.ts b/packages/types/src/sdk/licensing/plan.ts index 1604dfb8af..5ac8b1c9f6 100644 --- a/packages/types/src/sdk/licensing/plan.ts +++ b/packages/types/src/sdk/licensing/plan.ts @@ -7,7 +7,9 @@ export enum PlanType { /** @deprecated */ PREMIUM = "premium", PREMIUM_PLUS = "premium_plus", + /** @deprecated */ BUSINESS = "business", + ENTERPRISE_BASIC = "enterprise_basic", ENTERPRISE = "enterprise", } From 5088caf0f944c82cad3b292df3c06896a0583893 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Mon, 6 Nov 2023 08:37:53 +0100 Subject: [PATCH 02/16] Update pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 3820c0c93a..34cce46d9c 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 3820c0c93a3e448e10a60a9feb5396844b537ca8 +Subproject commit 34cce46d9c67f03aa66272e079e1d460a731225e From b03719ff1c72509e7edd75f7a1eb82695c658c80 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Wed, 8 Nov 2023 17:34:28 +0100 Subject: [PATCH 03/16] Update pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 34cce46d9c..cbd3c7e333 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 34cce46d9c67f03aa66272e079e1d460a731225e +Subproject commit cbd3c7e33389c976dc73934667beb65d60c605f3 From 8436a8d22055d47cf5f331cc4e65259ae5d87e5d Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 9 Nov 2023 09:54:05 +0100 Subject: [PATCH 04/16] Day passes are optional --- packages/types/src/sdk/licensing/quota.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts index 85700f167b..2d33c14182 100644 --- a/packages/types/src/sdk/licensing/quota.ts +++ b/packages/types/src/sdk/licensing/quota.ts @@ -61,7 +61,7 @@ export type PlanQuotas = { [key in PlanType]: Quotas | undefined } export type MonthlyQuotas = { [MonthlyQuotaName.QUERIES]: Quota [MonthlyQuotaName.AUTOMATIONS]: Quota - [MonthlyQuotaName.DAY_PASSES]: Quota + [MonthlyQuotaName.DAY_PASSES]?: Quota } export type StaticQuotas = { From e1ca1d67322779eae0857028ea715a899933da14 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 9 Nov 2023 09:55:09 +0100 Subject: [PATCH 05/16] Update pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index cbd3c7e333..ab17824872 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit cbd3c7e33389c976dc73934667beb65d60c605f3 +Subproject commit ab17824872a08010886abdc5893b14d3430efe00 From 311ad6b6a5010435fd56778151f85356e4ee01aa Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 9 Nov 2023 10:10:04 +0100 Subject: [PATCH 06/16] Update pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index ab17824872..18f558efb0 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit ab17824872a08010886abdc5893b14d3430efe00 +Subproject commit 18f558efb0f2d4a021a46d6236d12d93f6ce474f From 1f85b56a99400b3cda5735379a98a359279180a5 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 9 Nov 2023 11:44:36 +0100 Subject: [PATCH 07/16] Revert "Day passes are optional" This reverts commit 8436a8d22055d47cf5f331cc4e65259ae5d87e5d. --- packages/types/src/sdk/licensing/quota.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts index 2d33c14182..85700f167b 100644 --- a/packages/types/src/sdk/licensing/quota.ts +++ b/packages/types/src/sdk/licensing/quota.ts @@ -61,7 +61,7 @@ export type PlanQuotas = { [key in PlanType]: Quotas | undefined } export type MonthlyQuotas = { [MonthlyQuotaName.QUERIES]: Quota [MonthlyQuotaName.AUTOMATIONS]: Quota - [MonthlyQuotaName.DAY_PASSES]?: Quota + [MonthlyQuotaName.DAY_PASSES]: Quota } export type StaticQuotas = { From 5ddc3faf22a7100cf83a286ae372967b83a268bb Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 9 Nov 2023 11:45:38 +0100 Subject: [PATCH 08/16] Upgrade pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 18f558efb0..837bbc7b33 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 18f558efb0f2d4a021a46d6236d12d93f6ce474f +Subproject commit 837bbc7b3317c5681fa22f2b92a8d1a7d39c3c95 From 14c6e1bc4a4cf9c3f68cda4b300e8e511e15e73d Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Thu, 9 Nov 2023 12:00:21 +0100 Subject: [PATCH 09/16] Update pro submodule --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 837bbc7b33..e202f415d9 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 837bbc7b3317c5681fa22f2b92a8d1a7d39c3c95 +Subproject commit e202f415d9fa540d08cc2ba6e27394fbc22f357b From ee91d6753c438229affc681e13cf54b4d8ab1760 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Thu, 9 Nov 2023 11:55:10 +0000 Subject: [PATCH 10/16] Encode query string --- .../builder/src/components/integration/RestQueryViewer.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/integration/RestQueryViewer.svelte b/packages/builder/src/components/integration/RestQueryViewer.svelte index e6913b0953..9634cd9746 100644 --- a/packages/builder/src/components/integration/RestQueryViewer.svelte +++ b/packages/builder/src/components/integration/RestQueryViewer.svelte @@ -404,7 +404,7 @@ datasource = $datasources.list.find(ds => ds._id === query?.datasourceId) const datasourceUrl = datasource?.config.url const qs = query?.fields.queryString - breakQs = restUtils.breakQueryString(qs) + breakQs = restUtils.breakQueryString(encodeURI(qs)) breakQs = runtimeToReadableMap(mergedBindings, breakQs) const path = query.fields.path @@ -652,7 +652,7 @@
- {#if !response && Object.keys(schema).length === 0} + {#if !response && Object.keys(schema || {}).length === 0} Response
From 08b96ad2131dacd72e7042cd5c98110afc8f1fe0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 9 Nov 2023 12:55:50 +0100 Subject: [PATCH 11/16] Remove unnused pipelines --- .github/workflows/deploy-cloud.yaml | 48 ------ .github/workflows/release-master.yml | 178 ---------------------- .github/workflows/release-selfhost.yml | 125 --------------- .github/workflows/release-singleimage.yml | 86 ----------- 4 files changed, 437 deletions(-) delete mode 100644 .github/workflows/deploy-cloud.yaml delete mode 100644 .github/workflows/release-master.yml delete mode 100644 .github/workflows/release-selfhost.yml delete mode 100644 .github/workflows/release-singleimage.yml diff --git a/.github/workflows/deploy-cloud.yaml b/.github/workflows/deploy-cloud.yaml deleted file mode 100644 index 389b10f7d3..0000000000 --- a/.github/workflows/deploy-cloud.yaml +++ /dev/null @@ -1,48 +0,0 @@ -name: Budibase Deploy Production - -on: - workflow_dispatch: - inputs: - version: - description: Budibase release version. For example - 1.0.0 - required: false - -jobs: - release: - runs-on: ubuntu-latest - - steps: - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Fail if tag is not in master - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then - echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" - exit 1 - fi - - - name: Get the latest budibase release version - id: version - run: | - if [ -z "${{ github.event.inputs.version }}" ]; then - release_version=$(cat lerna.json | jq -r '.version') - else - release_version=${{ github.event.inputs.version }} - fi - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - - uses: passeidireto/trigger-external-workflow-action@main - env: - PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }} - with: - repository: budibase/budibase-deploys - event: budicloud-prod-deploy - github_pat: ${{ secrets.GH_ACCESS_TOKEN }} diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml deleted file mode 100644 index 2edb470405..0000000000 --- a/.github/workflows/release-master.yml +++ /dev/null @@ -1,178 +0,0 @@ -name: Budibase Release -concurrency: - group: release - cancel-in-progress: false - -on: - push: - tags: - - "[0-9]+.[0-9]+.[0-9]+" - # Exclude all pre-releases - - "!*[0-9]+.[0-9]+.[0-9]+-*" - -env: - # Posthog token used by ui at build time - POSTHOG_TOKEN: phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU - INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }} - PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - -jobs: - release-images: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - fetch-depth: 0 - - - name: Fail if tag is not in master - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then - echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" - exit 1 - fi - - - uses: actions/setup-node@v1 - with: - node-version: 18.x - cache: yarn - - - run: yarn install --frozen-lockfile - - name: Update versions - run: ./scripts/updateVersions.sh - - run: yarn lint - - run: yarn build - - run: yarn build:sdk - - - name: Publish budibase packages to NPM - env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default - git config --global user.name "Budibase Release Bot" - git config --global user.email "<>" - git submodule foreach git commit -a -m 'Release process' - git commit -a -m 'Release process' - echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc - yarn release - - - name: "Get Current tag" - id: currenttag - run: | - version=$(./scripts/getCurrentVersion.sh) - echo "Using tag $version" - echo "version=$version" >> "$GITHUB_OUTPUT" - - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - - name: Docker login - run: | - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - env: - DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - - - name: Build worker docker - uses: docker/build-push-action@v5 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - build-args: | - BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} - tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} - file: ./packages/worker/Dockerfile.v2 - cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest - cache-to: type=inline - env: - IMAGE_NAME: budibase/worker - IMAGE_TAG: ${{ steps.currenttag.outputs.version }} - BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }} - - - name: Build server docker - uses: docker/build-push-action@v5 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - build-args: | - BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} - tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} - file: ./packages/server/Dockerfile.v2 - cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest - cache-to: type=inline - env: - IMAGE_NAME: budibase/apps - IMAGE_TAG: ${{ steps.currenttag.outputs.version }} - BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }} - - - name: Build proxy docker - uses: docker/build-push-action@v5 - with: - context: ./hosting/proxy - push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} - file: ./hosting/proxy/Dockerfile - cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest - cache-to: type=inline - env: - IMAGE_NAME: budibase/proxy - IMAGE_TAG: ${{ steps.currenttag.outputs.version }} - - release-helm-chart: - needs: [release-images] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Setup Helm - uses: azure/setup-helm@v1 - id: helm-install - - - name: Get the latest budibase release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - # due to helm repo index issue: https://github.com/helm/helm/issues/7363 - # we need to create new package in a different dir, merge the index and move the package back - - name: Build and release helm chart - run: | - git config user.name "Budibase Helm Bot" - git config user.email "<>" - git reset --hard - git fetch - mkdir sync - echo "Packaging chart to sync dir" - helm package charts/budibase --version 0.0.0-master --app-version "$RELEASE_VERSION" --destination sync - echo "Packaging successful" - git checkout gh-pages - echo "Indexing helm repo" - helm repo index --merge docs/index.yaml sync - mv -f sync/* docs - rm -rf sync - echo "Pushing new helm release" - git add -A - git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}" - git push - - trigger-deploy-to-qa-env: - needs: [release-helm-chart] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - uses: peter-evans/repository-dispatch@v2 - with: - repository: budibase/budibase-deploys - event-type: budicloud-qa-deploy - token: ${{ secrets.GH_ACCESS_TOKEN }} - client-payload: |- - { - "VERSION": "${{ github.ref_name }}", - "REF_NAME": "${{ github.ref_name}}" - } diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml deleted file mode 100644 index d2689a0ea0..0000000000 --- a/.github/workflows/release-selfhost.yml +++ /dev/null @@ -1,125 +0,0 @@ -name: Budibase Release Selfhost - -on: - workflow_dispatch: - -jobs: - release: - runs-on: ubuntu-latest - - steps: - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - fetch-depth: 0 - - - name: Fail if tag is not in master - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then - echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" - exit 1 - fi - - - name: Use Node.js 18.x - uses: actions/setup-node@v1 - with: - node-version: 18.x - - - name: Get the latest budibase release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - - name: Tag and release Docker images (Self Host) - run: | - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - - release_tag=${{ env.RELEASE_VERSION }} - - # Pull apps and worker images - docker pull budibase/apps:$release_tag - docker pull budibase/worker:$release_tag - docker pull budibase/proxy:$release_tag - - # Tag apps and worker images - docker tag budibase/apps:$release_tag budibase/apps:$SELFHOST_TAG - docker tag budibase/worker:$release_tag budibase/worker:$SELFHOST_TAG - docker tag budibase/proxy:$release_tag budibase/proxy:$SELFHOST_TAG - - # Push images - docker push budibase/apps:$SELFHOST_TAG - docker push budibase/worker:$SELFHOST_TAG - docker push budibase/proxy:$SELFHOST_TAG - env: - DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - SELFHOST_TAG: latest - - - name: Bootstrap and build (CLI) - run: | - yarn - yarn build - - - name: Build OpenAPI spec - run: | - pushd packages/server - yarn - yarn specs - popd - - - name: Setup Helm - uses: azure/setup-helm@v1 - id: helm-install - - # due to helm repo index issue: https://github.com/helm/helm/issues/7363 - # we need to create new package in a different dir, merge the index and move the package back - - name: Build and release helm chart - run: | - git config user.name "Budibase Helm Bot" - git config user.email "<>" - git reset --hard - git fetch - mkdir sync - echo "Packaging chart to sync dir" - helm package charts/budibase --version "$RELEASE_VERSION" --app-version "$RELEASE_VERSION" --destination sync - echo "Packaging successful" - git checkout gh-pages - echo "Indexing helm repo" - helm repo index --merge docs/index.yaml sync - mv -f sync/* docs - rm -rf sync - echo "Pushing new helm release" - git add -A - git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}" - git push - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Perform Github Release - uses: softprops/action-gh-release@v1 - with: - name: ${{ env.RELEASE_VERSION }} - tag_name: ${{ env.RELEASE_VERSION }} - generate_release_notes: true - files: | - packages/cli/build/cli-win.exe - packages/cli/build/cli-linux - packages/cli/build/cli-macos - packages/server/specs/openapi.yaml - packages/server/specs/openapi.json - - - name: Discord Webhook Action - uses: tsickert/discord-webhook@v4.0.0 - with: - webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }} - content: "Self Host Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Self Host." - embed-title: ${{ env.RELEASE_VERSION }} diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml deleted file mode 100644 index 16b1da186a..0000000000 --- a/.github/workflows/release-singleimage.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Deploy Budibase Single Container Image to DockerHub - -on: - workflow_dispatch: - -env: - CI: true - PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - REGISTRY_URL: registry.hub.docker.com -jobs: - build: - name: "build" - runs-on: ubuntu-latest - strategy: - matrix: - node-version: [18.x] - steps: - - name: Maximize build space - uses: easimon/maximize-build-space@master - with: - root-reserve-mb: 30000 - swap-size-mb: 1024 - remove-android: "true" - remove-dotnet: "true" - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - name: "Checkout" - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - name: Setup QEMU - uses: docker/setup-qemu-action@v1 - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - name: Run Yarn - run: yarn - - name: Update versions - run: ./scripts/updateVersions.sh - - name: Run Yarn Build - run: yarn build - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_API_KEY }} - - name: Get the latest release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo $release_version - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - name: Tag and release Budibase service docker image - uses: docker/build-push-action@v2 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - build-args: BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} - tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }} - file: ./hosting/single/Dockerfile.v2 - env: - BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }} - - name: Tag and release Budibase Azure App Service docker image - uses: docker/build-push-action@v2 - with: - context: . - push: true - platforms: linux/amd64 - build-args: | - TARGETBUILD=aas - BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} - tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }} - file: ./hosting/single/Dockerfile.v2 - env: - BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }} From a26f2e83e49a1acc24f3b262666ec273e7c4d470 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 9 Nov 2023 14:45:23 +0000 Subject: [PATCH 12/16] Some of the new table getters did not account for table IDs not being found, adding a new function which properly accounts for IDs being missing and manages it correctly, rather than leaving tables in an undefined state. --- packages/backend-core/src/db/couch/DatabaseImpl.ts | 13 ++++++++++++- .../server/src/api/controllers/row/internal.ts | 11 ++++------- packages/server/src/api/controllers/row/utils.ts | 14 +------------- packages/server/src/db/linkedRows/index.ts | 6 ++---- packages/server/src/db/utils.ts | 10 ---------- packages/server/src/sdk/app/tables/getters.ts | 14 +++++++++----- packages/server/src/utilities/global.ts | 6 ++---- packages/types/src/sdk/db.ts | 3 ++- 8 files changed, 32 insertions(+), 45 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 1ace60ed5b..96b946fe16 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -109,7 +109,7 @@ export class DatabaseImpl implements Database { } } - async get(id?: string): Promise { + async get(id?: string): Promise { const db = await this.checkSetup() if (!id) { throw new Error("Unable to get doc without a valid _id.") @@ -117,6 +117,17 @@ export class DatabaseImpl implements Database { return this.updateOutput(() => db.get(id)) } + async getMultiple(ids: string[]): Promise { + // get unique + ids = [...new Set(ids)] + const response = await this.allDocs({ + keys: ids, + include_docs: true, + }) + const rows = response.rows.filter(row => row.error !== "not_found") + return rows.map(row => row.doc!) + } + async remove(idOrDoc: string | Document, rev?: string) { const db = await this.checkSetup() let _id: string diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index fe7d94547a..0edbe4ea86 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -1,9 +1,5 @@ import * as linkRows from "../../../db/linkedRows" -import { - generateRowID, - getMultiIDParams, - InternalTables, -} from "../../../db/utils" +import { generateRowID, InternalTables } from "../../../db/utils" import * as userController from "../user" import { cleanupAttachments, @@ -240,8 +236,9 @@ export async function fetchEnrichedRow(ctx: UserCtx) { const linkVals = links as LinkDocumentValue[] // look up the actual rows based on the ids - const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id)) - let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc!) + let linkedRows = await db.getMultiple( + linkVals.map(linkVal => linkVal.id) + ) // get the linked tables const linkTableIds = getLinkedTableIDs(table as Table) diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index cd311fdf0f..ed6ccd4c53 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -1,21 +1,9 @@ import { InternalTables } from "../../../db/utils" import * as userController from "../user" import { context } from "@budibase/backend-core" -import { - Ctx, - FieldType, - ManyToOneRelationshipFieldMetadata, - OneToManyRelationshipFieldMetadata, - Row, - SearchFilters, - Table, - UserCtx, -} from "@budibase/types" -import { FieldTypes, NoEmptyFilterStrings } from "../../../constants" -import sdk from "../../../sdk" +import { Ctx, Row, UserCtx } from "@budibase/types" import validateJs from "validate.js" -import { cloneDeep } from "lodash/fp" validateJs.extend(validateJs.validators.datetime, { parse: function (value: string) { diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 7324fa1d94..24ff47ead3 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -8,7 +8,7 @@ import { getLinkedTable, } from "./linkUtils" import flatten from "lodash/flatten" -import { getMultiIDParams, USER_METDATA_PREFIX } from "../utils" +import { USER_METDATA_PREFIX } from "../utils" import partition from "lodash/partition" import { getGlobalUsersFromMetadata } from "../../utilities/global" import { processFormulas } from "../../utilities/rowProcessor" @@ -79,9 +79,7 @@ async function getFullLinkedDocs(links: LinkDocumentValue[]) { const db = context.getAppDB() const linkedRowIds = links.map(link => link.id) const uniqueRowIds = [...new Set(linkedRowIds)] - let dbRows = (await db.allDocs(getMultiIDParams(uniqueRowIds))).rows.map( - row => row.doc! - ) + let dbRows = await db.getMultiple(uniqueRowIds) // convert the unique db rows back to a full list of linked rows const linked = linkedRowIds .map(id => dbRows.find(row => row && row._id === id)) diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index 715db552c9..a5569f8166 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -283,16 +283,6 @@ export function generatePluginID(name: string) { return `${DocumentType.PLUGIN}${SEPARATOR}${name}` } -/** - * This can be used with the db.allDocs to get a list of IDs - */ -export function getMultiIDParams(ids: string[]) { - return { - keys: ids, - include_docs: true, - } -} - /** * Generates a new view ID. * @returns The new view ID which the view doc can be stored under. diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index a7074f95b2..a09d686b6d 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -1,5 +1,5 @@ import { context } from "@budibase/backend-core" -import { getMultiIDParams, getTableParams } from "../../../db/utils" +import { getTableParams } from "../../../db/utils" import { breakExternalTableId, isExternalTableID, @@ -17,6 +17,9 @@ import datasources from "../datasources" import sdk from "../../../sdk" export function processTable(table: Table): Table { + if (!table) { + return table + } if (table._id && isExternalTableID(table._id)) { return { ...table, @@ -73,6 +76,9 @@ export async function getExternalTable( tableName: string ): Promise { const entities = await getExternalTablesInDatasource(datasourceId) + if (!entities[tableName]) { + throw new Error(`Unable to find table named "${tableName}"`) + } return processTable(entities[tableName]) } @@ -124,10 +130,8 @@ export async function getTables(tableIds: string[]): Promise { } if (internalTableIds.length) { const db = context.getAppDB() - const internalTableDocs = await db.allDocs
( - getMultiIDParams(internalTableIds) - ) - tables = tables.concat(internalTableDocs.rows.map(row => row.doc!)) + const internalTables = await db.getMultiple
(internalTableIds) + tables = tables.concat(internalTables) } return processTables(tables) } diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts index cdc2d84513..762d1fb7a0 100644 --- a/packages/server/src/utilities/global.ts +++ b/packages/server/src/utilities/global.ts @@ -1,4 +1,4 @@ -import { getMultiIDParams, getGlobalIDFromUserMetadataID } from "../db/utils" +import { getGlobalIDFromUserMetadataID } from "../db/utils" import { roles, db as dbCore, @@ -96,9 +96,7 @@ export async function getRawGlobalUsers(userIds?: string[]): Promise { const db = tenancy.getGlobalDB() let globalUsers: User[] if (userIds) { - globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map( - row => row.doc! - ) + globalUsers = await db.getMultiple(userIds) } else { globalUsers = ( await db.allDocs( diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index 26807d99ce..9ea854b3aa 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -122,7 +122,8 @@ export interface Database { exists(): Promise checkSetup(): Promise> - get(id?: string): Promise + get(id?: string): Promise + getMultiple(ids: string[]): Promise remove( id: string | Document, rev?: string From 37e34c8ed2e79c1c5981a95e6402736643d94d1e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 9 Nov 2023 14:53:14 +0000 Subject: [PATCH 13/16] Adding the ability to fail on getMultiple if needed. --- packages/backend-core/src/db/couch/DatabaseImpl.ts | 14 ++++++++++++-- packages/types/src/sdk/db.ts | 5 ++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 96b946fe16..fa505847af 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -117,14 +117,24 @@ export class DatabaseImpl implements Database { return this.updateOutput(() => db.get(id)) } - async getMultiple(ids: string[]): Promise { + async getMultiple( + ids: string[], + opts?: { failIfMissing?: boolean } + ): Promise { // get unique ids = [...new Set(ids)] const response = await this.allDocs({ keys: ids, include_docs: true, }) - const rows = response.rows.filter(row => row.error !== "not_found") + const NOT_FOUND = "not_found" + const rows = response.rows.filter(row => row.error !== NOT_FOUND) + // some were filtered out - means some missing + if (opts?.failIfMissing && rows.length !== response.rows.length) { + const missing = response.rows.filter(row => row.error === NOT_FOUND) + const missingIds = missing.map(row => row.key).join(", ") + throw new Error(`Unable to get documents: ${missingIds}`) + } return rows.map(row => row.doc!) } diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index 9ea854b3aa..ff5742bcfc 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -123,7 +123,10 @@ export interface Database { exists(): Promise checkSetup(): Promise> get(id?: string): Promise - getMultiple(ids: string[]): Promise + getMultiple( + ids: string[], + opts?: { failIfMissing?: boolean } + ): Promise remove( id: string | Document, rev?: string From dde446286de6922d79953881b11abb1033d33e0e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 9 Nov 2023 17:08:14 +0000 Subject: [PATCH 14/16] Switching getMultiple to default to failure if not all entries found, then updating usages. --- packages/backend-core/src/db/couch/DatabaseImpl.ts | 5 +++-- packages/server/src/api/controllers/row/internal.ts | 3 ++- packages/server/src/db/linkedRows/index.ts | 2 +- packages/server/src/sdk/app/tables/getters.ts | 4 +++- packages/server/src/utilities/global.ts | 2 +- packages/types/src/sdk/db.ts | 2 +- 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index fa505847af..6a1e575ac9 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -119,7 +119,7 @@ export class DatabaseImpl implements Database { async getMultiple( ids: string[], - opts?: { failIfMissing?: boolean } + opts?: { allowMissing?: boolean } ): Promise { // get unique ids = [...new Set(ids)] @@ -129,8 +129,9 @@ export class DatabaseImpl implements Database { }) const NOT_FOUND = "not_found" const rows = response.rows.filter(row => row.error !== NOT_FOUND) + const someMissing = rows.length !== response.rows.length // some were filtered out - means some missing - if (opts?.failIfMissing && rows.length !== response.rows.length) { + if (!opts?.allowMissing && someMissing) { const missing = response.rows.filter(row => row.error === NOT_FOUND) const missingIds = missing.map(row => row.key).join(", ") throw new Error(`Unable to get documents: ${missingIds}`) diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index 0edbe4ea86..0907c22f0e 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -237,7 +237,8 @@ export async function fetchEnrichedRow(ctx: UserCtx) { // look up the actual rows based on the ids let linkedRows = await db.getMultiple( - linkVals.map(linkVal => linkVal.id) + linkVals.map(linkVal => linkVal.id), + { allowMissing: true } ) // get the linked tables diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 24ff47ead3..7af3f9392f 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -79,7 +79,7 @@ async function getFullLinkedDocs(links: LinkDocumentValue[]) { const db = context.getAppDB() const linkedRowIds = links.map(link => link.id) const uniqueRowIds = [...new Set(linkedRowIds)] - let dbRows = await db.getMultiple(uniqueRowIds) + let dbRows = await db.getMultiple(uniqueRowIds, { allowMissing: true }) // convert the unique db rows back to a full list of linked rows const linked = linkedRowIds .map(id => dbRows.find(row => row && row._id === id)) diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index a09d686b6d..72a6ab61f1 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -130,7 +130,9 @@ export async function getTables(tableIds: string[]): Promise { } if (internalTableIds.length) { const db = context.getAppDB() - const internalTables = await db.getMultiple
(internalTableIds) + const internalTables = await db.getMultiple
(internalTableIds, { + allowMissing: true, + }) tables = tables.concat(internalTables) } return processTables(tables) diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts index 762d1fb7a0..bbb84c1882 100644 --- a/packages/server/src/utilities/global.ts +++ b/packages/server/src/utilities/global.ts @@ -96,7 +96,7 @@ export async function getRawGlobalUsers(userIds?: string[]): Promise { const db = tenancy.getGlobalDB() let globalUsers: User[] if (userIds) { - globalUsers = await db.getMultiple(userIds) + globalUsers = await db.getMultiple(userIds, { allowMissing: true }) } else { globalUsers = ( await db.allDocs( diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index ff5742bcfc..7613ac6aeb 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -125,7 +125,7 @@ export interface Database { get(id?: string): Promise getMultiple( ids: string[], - opts?: { failIfMissing?: boolean } + opts?: { allowMissing?: boolean } ): Promise remove( id: string | Document, From b68607b0485844e004a2a481300048421d0ef33d Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Thu, 9 Nov 2023 18:21:28 +0000 Subject: [PATCH 15/16] Redis query command doesn't accept spaced values (#12357) * Handle string phrase with spaces value * Unit test --- packages/server/src/integrations/redis.ts | 14 +++++++++++++- .../server/src/integrations/tests/redis.spec.ts | 17 +++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/redis.ts b/packages/server/src/integrations/redis.ts index 879a790550..6a6331ccd4 100644 --- a/packages/server/src/integrations/redis.ts +++ b/packages/server/src/integrations/redis.ts @@ -165,10 +165,22 @@ class RedisIntegration { // commands split line by line const commands = query.json.trim().split("\n") let pipelineCommands = [] + let tokenised // process each command separately for (let command of commands) { - const tokenised = command.trim().split(" ") + const valueToken = command.trim().match(/".*"/) + if (valueToken?.[0]) { + tokenised = [ + ...command + .substring(0, command.indexOf(valueToken[0]) - 1) + .trim() + .split(" "), + valueToken?.[0], + ] + } else { + tokenised = command.trim().split(" ") + } // Pipeline only accepts lower case commands tokenised[0] = tokenised[0].toLowerCase() pipelineCommands.push(tokenised) diff --git a/packages/server/src/integrations/tests/redis.spec.ts b/packages/server/src/integrations/tests/redis.spec.ts index 9521d58a51..942da99530 100644 --- a/packages/server/src/integrations/tests/redis.spec.ts +++ b/packages/server/src/integrations/tests/redis.spec.ts @@ -85,4 +85,21 @@ describe("Redis Integration", () => { ["get", "foo"], ]) }) + + it("calls the pipeline method with double quoted phrase values", async () => { + const body = { + json: 'SET foo "What a wonderful world!"\nGET foo', + } + + // ioredis-mock doesn't support pipelines + config.integration.client.pipeline = jest.fn(() => ({ + exec: jest.fn(() => [[]]), + })) + + await config.integration.command(body) + expect(config.integration.client.pipeline).toHaveBeenCalledWith([ + ["set", "foo", '"What a wonderful world!"'], + ["get", "foo"], + ]) + }) }) From 20895cf4266f3accc30aba8bfd5678793195de04 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 9 Nov 2023 18:22:06 +0000 Subject: [PATCH 16/16] Adding test case. --- packages/server/src/sdk/tests/tables.spec.ts | 39 ++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 packages/server/src/sdk/tests/tables.spec.ts diff --git a/packages/server/src/sdk/tests/tables.spec.ts b/packages/server/src/sdk/tests/tables.spec.ts new file mode 100644 index 0000000000..0e3cd73cfd --- /dev/null +++ b/packages/server/src/sdk/tests/tables.spec.ts @@ -0,0 +1,39 @@ +import TestConfig from "../../tests/utilities/TestConfiguration" +import { basicTable } from "../../tests/utilities/structures" +import { Table } from "@budibase/types" +import sdk from "../" + +describe("tables", () => { + const config = new TestConfig() + let table: Table + + beforeAll(async () => { + await config.init() + table = await config.api.table.create(basicTable()) + }) + + describe("getTables", () => { + it("should be able to retrieve tables", async () => { + await config.doInContext(config.appId, async () => { + const tables = await sdk.tables.getTables([table._id!]) + expect(tables.length).toBe(1) + expect(tables[0]._id).toBe(table._id) + expect(tables[0].name).toBe(table.name) + }) + }) + + it("shouldn't fail when retrieving tables that don't exist", async () => { + await config.doInContext(config.appId, async () => { + const tables = await sdk.tables.getTables(["unknown"]) + expect(tables.length).toBe(0) + }) + }) + + it("should de-duplicate the IDs", async () => { + await config.doInContext(config.appId, async () => { + const tables = await sdk.tables.getTables([table._id!, table._id!]) + expect(tables.length).toBe(1) + }) + }) + }) +})